diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cba85e41..1bc0028a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,7 +56,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.17 + go-version: 1.18 - name: Import GPG key uses: crazy-max/ghaction-import-gpg@v3 @@ -67,7 +67,7 @@ jobs: - name: Install goreleaser uses: goreleaser/goreleaser-action@v2 with: - version: v1.4.1 + version: v1.8.3 install-only: true - name: Run goreleaser diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d67581df..573e51ad 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,7 @@ jobs: - name: Install Go uses: actions/setup-go@v2 with: - go-version: 1.17 + go-version: 1.18 - name: Lint if: ${{ matrix.os == 'ubuntu-20.04' }} diff --git a/.gitignore b/.gitignore index b7fae06f..009071e0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ dist +helpers vendor **/*.tgz /gitlab-ci-pipelines-exporter.yml diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..f3f758ec --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,45 @@ +linters: + enable-all: true + disable: + # Deprecated ones + - golint + - interfacer + - maligned + - scopelint + + # We don't want these ones + - exhaustivestruct + - forcetypeassert + - gochecknoglobals + - godox + - goerr113 + - ireturn + - nakedret + - testpackage + - varnamelen + + # TODO + - tagliatelle + - promlinter + - paralleltest + - gocognit + - gomoddirectives + - forbidigo + - goconst + - gomnd + - lll + - dupl + +linters-settings: + funlen: + lines: -1 # (disabled) + statements: 100 + + cyclop: + max-complexity: 20 + + lll: + line-length: 140 + + nestif: + min-complexity: 18 \ No newline at end of file diff --git a/.goreleaser.pre.yml b/.goreleaser.pre.yml index 2ba3092c..542e8ec1 100644 --- a/.goreleaser.pre.yml +++ b/.goreleaser.pre.yml @@ -1,6 +1,7 @@ before: hooks: - - go mod download + - make man-pages + - make autocomplete-scripts builds: - main: ./cmd/gitlab-ci-pipelines-exporter @@ -18,16 +19,49 @@ builds: goarm: [6, 7] flags: - -trimpath - ignore: - - { goos: darwin, goarch: 386 } - - { goos: darwin, goarch: arm } - - { goos: windows, goarch: arm } + +universal_binaries: + - {} archives: - name_template: '{{ .ProjectName }}_edge_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' format_overrides: - goos: windows format: zip + files: + - README.md + - LICENSE + - helpers/* + +nfpms: + - maintainer: &author Maxime VISONNEAU + description: &description GitLab CI pipelines exporter (prometheus/open-metrics) + license: &license Apache-2.0 + homepage: &homepage https://github.com/mvisonneau/gitlab-ci-pipelines-exporter + vendor: *author + file_name_template: '{{ .ProjectName }}_{{ .Tag }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' + bindir: /usr/local/bin + formats: + - apk + - deb + - rpm + contents: + - src: ./helpers/autocomplete/bash + dst: /usr/share/bash-completion/completions/{{ .ProjectName }} + file_info: + mode: 0644 + - src: ./helpers/autocomplete/zsh + dst: /usr/share/zsh/vendor-completions/_{{ .ProjectName }} + file_info: + mode: 0644 + - src: ./helpers/manpages/{{ .ProjectName }}.1.gz + dst: /usr/share/man/man1/{{ .ProjectName }}.1.gz + file_info: + mode: 0644 + - src: ./LICENSE + dst: /usr/share/doc/{{ .ProjectName }}/copyright + file_info: + mode: 0644 release: disable: true @@ -36,7 +70,7 @@ snapcrafts: - summary: GitLab CI pipelines exporter (prometheus/open-metrics) description: Monitor GitLab CI pipelines related metrics. license: Apache-2.0 - confinement: strict + confinement: classic grade: devel apps: gitlab-ci-pipelines-exporter: diff --git a/.goreleaser.yml b/.goreleaser.yml index 96300df9..9a521364 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,6 +1,7 @@ before: hooks: - - go mod download + - make man-pages + - make autocomplete-scripts builds: - main: ./cmd/gitlab-ci-pipelines-exporter @@ -18,16 +19,19 @@ builds: goarm: [6, 7] flags: - -trimpath - ignore: - - { goos: darwin, goarch: 386 } - - { goos: darwin, goarch: arm } - - { goos: windows, goarch: arm } + +universal_binaries: + - {} archives: - name_template: '{{ .ProjectName }}_edge_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' format_overrides: - goos: windows format: zip + files: + - README.md + - LICENSE + - helpers/* nfpms: - maintainer: &author Maxime VISONNEAU @@ -36,17 +40,44 @@ nfpms: homepage: &homepage https://github.com/mvisonneau/gitlab-ci-pipelines-exporter vendor: *author file_name_template: '{{ .ProjectName }}_{{ .Tag }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' + bindir: /usr/local/bin formats: + - apk - deb - rpm + contents: + - src: ./helpers/autocomplete/bash + dst: /usr/share/bash-completion/completions/{{ .ProjectName }} + file_info: + mode: 0644 + - src: ./helpers/autocomplete/zsh + dst: /usr/share/zsh/vendor-completions/_{{ .ProjectName }} + file_info: + mode: 0644 + - src: ./helpers/manpages/{{ .ProjectName }}.1.gz + dst: /usr/share/man/man1/{{ .ProjectName }}.1.gz + file_info: + mode: 0644 + - src: ./LICENSE + dst: /usr/share/doc/{{ .ProjectName }}/copyright + file_info: + mode: 0644 brews: - description: *description homepage: *homepage + license: *license folder: Formula tap: owner: mvisonneau name: homebrew-tap + test: | + system "#{bin}/{{ .ProjectName }} -v" + install: |- + bin.install "{{ .ProjectName }}" + bash_completion.install "./helpers/autocomplete/bash" => "{{ .ProjectName }}" + zsh_completion.install "./helpers/autocomplete/zsh" => "_{{ .ProjectName }}" + man1.install "./helpers/manpages/{{ .ProjectName }}.1.gz" scoop: description: *description diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cb05afa..b5fcc036 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,19 @@ and this project adheres to [0ver](https://0ver.org) (more or less). ## [Unreleased] +### Added + +- Now passing a `context.Context` to most functional calls +- Aggregated already used linters and added new ones through the implementation of `golangci` +- Release `.apk` packages for Alpine linux +- Added man pages and autocompletion scripts (bash & zsh) to `.apk`, `.deb`, `.rpm` & `homebrew` packages +- Release "fat" binaries (arm64 + amd64 combined) for MacOS under `_all` suffix + ### Changed - Fixed a config issue preventing the arm deb/rpm packages to be released correctly +- Upgraded golang to **v1.18** +- Upgraded most dependencies to their lastest versions ## [v0.5.3] - 2022-02-11 diff --git a/Makefile b/Makefile index 4d751b37..2d2e0068 100644 --- a/Makefile +++ b/Makefile @@ -5,43 +5,16 @@ REPOSITORY := mvisonneau/$(NAME) .PHONY: setup setup: ## Install required libraries/tools for build tasks - @command -v gofumpt 2>&1 >/dev/null || go install mvdan.cc/gofumpt@v0.2.1 - @command -v gosec 2>&1 >/dev/null || go install github.com/securego/gosec/v2/cmd/gosec@v2.9.6 - @command -v ineffassign 2>&1 >/dev/null || go install github.com/gordonklaus/ineffassign@v0.0.0-20210914165742-4cc7213b9bc8 - @command -v misspell 2>&1 >/dev/null || go install github.com/client9/misspell/cmd/misspell@v0.3.4 - @command -v revive 2>&1 >/dev/null || go install github.com/mgechev/revive@v1.1.3 + @command -v gofumpt 2>&1 >/dev/null || go install mvdan.cc/gofumpt@v0.3.1 + @command -v golangci-lint 2>&1 >/dev/null || go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.45.2 .PHONY: fmt fmt: setup ## Format source code gofumpt -w $(FILES) .PHONY: lint -lint: revive vet gofumpt ineffassign misspell gosec ## Run all lint related tests against the codebase - -.PHONY: revive -revive: setup ## Test code syntax with revive - revive -config .revive.toml $(FILES) - -.PHONY: vet -vet: ## Test code syntax with go vet - go vet ./... - -.PHONY: gofumpt -gofumpt: setup ## Test code syntax with gofumpt - gofumpt -d $(FILES) > gofumpt.out - @if [ -s gofumpt.out ]; then cat gofumpt.out; rm gofumpt.out; exit 1; else rm gofumpt.out; fi - -.PHONY: ineffassign -ineffassign: setup ## Test code syntax for ineffassign - ineffassign ./... - -.PHONY: misspell -misspell: setup ## Test code with misspell - misspell -error $(FILES) - -.PHONY: gosec -gosec: setup ## Test code for security vulnerabilities - gosec ./... +lint: setup ## Run all lint related tests upon the codebase + golangci-lint run -v --fast .PHONY: test test: ## Run the tests against the codebase @@ -88,7 +61,7 @@ dev-env: ## Build a local development environment using Docker -v $(shell pwd):/go/src/github.com/mvisonneau/$(NAME) \ -w /go/src/github.com/mvisonneau/$(NAME) \ -p 8080:8080 \ - golang:1.17 \ + golang:1.18 \ /bin/bash -c 'make setup; make install; bash' .PHONY: is-git-dirty @@ -96,6 +69,19 @@ is-git-dirty: ## Tests if git is in a dirty state @git status --porcelain @test $(shell git status --porcelain | grep -c .) -eq 0 +.PHONY: man-pages +man-pages: ## Generates man pages + rm -rf helpers/manpages + mkdir -p helpers/manpages + go run ./cmd/tools/man | gzip -c -9 >helpers/manpages/$(NAME).1.gz + +.PHONY: autocomplete-scripts +autocomplete-scripts: ## Download CLI autocompletion scripts + rm -rf helpers/autocomplete + mkdir -p helpers/autocomplete + curl -sL https://raw.githubusercontent.com/urfave/cli/v2.5.0/autocomplete/bash_autocomplete > helpers/autocomplete/bash + curl -sL https://raw.githubusercontent.com/urfave/cli/v2.5.0/autocomplete/zsh_autocomplete > helpers/autocomplete/zsh + .PHONY: all all: lint test build coverage ## Test, builds and ship package for all supported platforms diff --git a/cmd/tools/man/main.go b/cmd/tools/man/main.go new file mode 100644 index 00000000..66f32cf9 --- /dev/null +++ b/cmd/tools/man/main.go @@ -0,0 +1,14 @@ +package main + +import ( + "fmt" + "time" + + "github.com/mvisonneau/gitlab-ci-pipelines-exporter/internal/cli" +) + +var version = "devel" + +func main() { + fmt.Println(cli.NewApp(version, time.Now()).ToMan()) +} diff --git a/go.mod b/go.mod index 90ceb5e6..97fecc66 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/mvisonneau/gitlab-ci-pipelines-exporter -go 1.17 +go 1.18 require ( github.com/alecthomas/chroma v0.10.0 @@ -15,7 +15,6 @@ require ( github.com/google/uuid v1.3.0 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb github.com/imdario/mergo v0.3.12 - github.com/muesli/termenv v0.11.1-0.20220212125758-44cd13922739 github.com/mvisonneau/go-helpers v0.0.1 github.com/openlyinc/pointy v1.1.2 github.com/paulbellamy/ratecounter v0.2.0 @@ -23,10 +22,10 @@ require ( github.com/prometheus/client_golang v1.12.1 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.1 - github.com/urfave/cli/v2 v2.4.0 + github.com/urfave/cli/v2 v2.5.0 github.com/vmihailenco/msgpack/v5 v5.3.5 github.com/vmihailenco/taskq/v3 v3.2.8 - github.com/xanzy/go-gitlab v0.63.0 + github.com/xanzy/go-gitlab v0.64.0 github.com/xeonx/timeago v1.0.0-rc4 go.uber.org/ratelimit v0.2.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b @@ -39,9 +38,9 @@ require ( github.com/bsm/redislock v0.7.2 // indirect github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/charmbracelet/harmonica v0.1.0 // indirect + github.com/charmbracelet/harmonica v0.2.0 // indirect github.com/containerd/console v1.0.3 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -51,9 +50,9 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.0 // indirect + github.com/hashicorp/go-retryablehttp v0.7.1 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/klauspost/compress v1.15.1 // indirect + github.com/klauspost/compress v1.15.2 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-isatty v0.0.14 // indirect @@ -61,21 +60,22 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.11.1-0.20220212125758-44cd13922739 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.33.0 // indirect + github.com/prometheus/common v0.34.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 // indirect - golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect - golang.org/x/net v0.0.0-20220403103023-749bd193bc2b // indirect - golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect - golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64 // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect + golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect + golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect + golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect + golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect + golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 // indirect golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect + golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect diff --git a/go.sum b/go.sum index 2a38c0d4..6efcbe21 100644 --- a/go.sum +++ b/go.sum @@ -48,14 +48,10 @@ github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9or github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aws/aws-sdk-go v1.42.7 h1:Ee7QC4Y/eGebVGO/5IGN3fSXXSrheesZYYj2pYJG7Zk= -github.com/aws/aws-sdk-go v1.42.7/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bsm/ginkgo v1.16.4/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk= -github.com/bsm/gomega v1.13.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk= -github.com/bsm/redislock v0.7.1/go.mod h1:TSF3xUotaocycoHjVAp535/bET+ZmvrtcyNrXc0Whm8= github.com/bsm/redislock v0.7.2 h1:jggqOio8JyX9FJBKIfjF3fTxAu/v7zC5mAID9LveqG4= github.com/bsm/redislock v0.7.2/go.mod h1:kS2g0Yvlymc9Dz8V3iVYAtLAaSVruYbAFdYBDrmC5WU= github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3 h1:IHZ1Le1ejzkmS7Si7dIzJvYDWe+BIoNmqMnfWHBZSVw= @@ -69,8 +65,9 @@ github.com/charmbracelet/bubbles v0.10.3/go.mod h1:jOA+DUF1rjZm7gZHcNyIVW+YrBPAL github.com/charmbracelet/bubbletea v0.19.3/go.mod h1:VuXF2pToRxDUHcBUcPmCRUHRvFATM4Ckb/ql1rBl3KA= github.com/charmbracelet/bubbletea v0.20.0 h1:/b8LEPgCbNr7WWZ2LuE/BV1/r4t5PyYJtDb+J3vpwxc= github.com/charmbracelet/bubbletea v0.20.0/go.mod h1:zpkze1Rioo4rJELjRyGlm9T2YNou1Fm4LIJQSa5QMEM= -github.com/charmbracelet/harmonica v0.1.0 h1:lFKeSd6OAckQ/CEzPVd2mqj+YMEubQ/3FM2IYY3xNm0= github.com/charmbracelet/harmonica v0.1.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= +github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ= +github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= github.com/charmbracelet/lipgloss v0.4.0/go.mod h1:vmdkHvce7UzX6xkyf4cca8WlwdQ5RQr8fzta+xl7BOM= github.com/charmbracelet/lipgloss v0.5.0 h1:lulQHuVeodSgDez+3rGiuxlPVXSnhth442DATR2/8t8= github.com/charmbracelet/lipgloss v0.5.0/go.mod h1:EZLha/HbzEt7cYqdFPovlqy5FZPj0xFhg5SaqxScmgs= @@ -82,8 +79,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creasty/defaults v1.6.0 h1:ltuE9cfphUtlrBeomuu8PEyISTXnxqkBIoQfXgv7BSc= github.com/creasty/defaults v1.6.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= @@ -122,7 +119,6 @@ github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/j github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/validator/v10 v10.10.1 h1:uA0+amWMiglNZKZ9FJRKUAe9U3RX91eVn1JYXMWt7ig= github.com/go-playground/validator/v10 v10.10.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= -github.com/go-redis/redis/v8 v8.1.0/go.mod h1:isLoQT/NFSP7V67lyvM9GmdvLdyZ7pEhsXvvyQtnQTo= github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= @@ -185,7 +181,6 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -196,9 +191,8 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= -github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4= -github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= +github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= @@ -207,15 +201,10 @@ github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb h1:tsEKRC3P github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac h1:w5wltlINIIqRTqQ64dASrCo0fM7k9nosPbKCZnkL0W0= -github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac/go.mod h1:gyMTRVO+ZkEy7wQDyD++okPsBN2q127EpuShhHMWG54= -github.com/jeffh/go.bdd v0.0.0-20120717032931-88f798ee0c74/go.mod h1:qNa9FlAfO0U/qNkzYBMH1JKYRMzC+sP9IcyV4U18l98= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -226,9 +215,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.2 h1:3WH+AG7s2+T8o3nrM/8u2rdqUEcQhmga7smjrT41nAw= +github.com/klauspost/compress v1.15.2/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -280,18 +268,12 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= -github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/openlyinc/pointy v1.1.2 h1:LywVV2BWC5Sp5v7FoP4bUD+2Yn5k0VNeRbU5vq9jUMY= github.com/openlyinc/pointy v1.1.2/go.mod h1:w2Sytx+0FVuMKn37xpXIAyBNhFNBIJGR/v2m7ik1WtM= github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs= @@ -318,8 +300,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.33.0 h1:rHgav/0a6+uYgGdNt3jwz8FNSesO/Hsang3O0T9A5SE= -github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= +github.com/prometheus/common v0.34.0 h1:RBmGO9d/FVjqHT0yUGQwBJhkwKV+wPCn7KGpvfab0uE= +github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -337,7 +319,6 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -354,14 +335,14 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/urfave/cli/v2 v2.4.0 h1:m2pxjjDFgDxSPtO8WSdbndj17Wu2y8vOT86wE/tjr+I= -github.com/urfave/cli/v2 v2.4.0/go.mod h1:NX9W0zmTvedE5oDoOMs2RTC8RvdK98NTYZE5LbaEYPg= +github.com/urfave/cli/v2 v2.5.0 h1:2sqblaW62ebcTIEvwb8eRvDfNHeBAeKxfhdynaanhug= +github.com/urfave/cli/v2 v2.5.0/go.mod h1:oDzoM7pVwz6wHn5ogWgFUU1s4VJayeQS+aEZDqXIEJs= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= -github.com/xanzy/go-gitlab v0.63.0 h1:a9fXpKWykUS6dowapFej/2Wjf4aOAEFC1q2ZIcz4IpI= -github.com/xanzy/go-gitlab v0.63.0/go.mod h1:F0QEXwmqiBUxCgJm8fE9S+1veX4XC9Z4cfaAbqwk4YM= +github.com/xanzy/go-gitlab v0.64.0 h1:rMgQdW9S1w3qvNAH2LYpFd2xh7KNLk+JWJd7sorNuTc= +github.com/xanzy/go-gitlab v0.64.0/go.mod h1:F0QEXwmqiBUxCgJm8fE9S+1veX4XC9Z4cfaAbqwk4YM= github.com/xeonx/timeago v1.0.0-rc4 h1:9rRzv48GlJC0vm+iBpLcWAr8YbETyN9Vij+7h2ammz4= github.com/xeonx/timeago v1.0.0-rc4/go.mod h1:qDLrYEFynLO7y5Ho7w3GwgtYgpy5UfhcXIIQvMKVDkA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -375,7 +356,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/otel v0.11.0/go.mod h1:G8UCk+KooF2HLkgo8RHX9epABH/aRGYET7gQOqBVdB0= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= @@ -387,8 +367,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o= -golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -399,7 +379,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20200908183739-ae8ad444f925/go.mod h1:1phAWC201xIgDyaFpmDeZkgf70Q4Pd/CNqfRtVPtxNw= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -420,7 +399,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -454,22 +432,20 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220403103023-749bd193bc2b h1:vI32FkLJNAWtGD4BwkThwEy6XS7ZLLMHkSkYfF8M0W0= -golang.org/x/net v0.0.0-20220403103023-749bd193bc2b/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a h1:qfl7ob3DIEs3Ml9oLuPwY2N04gymzAW04WsUQHIClgM= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -514,7 +490,6 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -531,12 +506,13 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64 h1:D1v9ucDTYBtbz5vNuBbAhIMAGhQhJ6Ym5ah3maMVNX4= -golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210422114643-f5beecf764ed/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 h1:EH1Deb8WZJ0xc0WK//leUHXcX9aLE5SymusoTmMZye8= +golang.org/x/term v0.0.0-20220411215600-e5f449aeb171/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -548,8 +524,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 h1:M73Iuj3xbbb9Uk1DYhzydthsj6oOd6l9bpuFcNoUvTs= -golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w= +golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -613,7 +589,6 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -692,7 +667,6 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 46e531a3..bfbde092 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -9,7 +9,7 @@ import ( "github.com/urfave/cli/v2" ) -// Run handles the instanciation of the CLI application +// Run handles the instanciation of the CLI application. func Run(version string, args []string) { err := NewApp(version, time.Now()).Run(args) if err != nil { @@ -18,7 +18,7 @@ func Run(version string, args []string) { } } -// NewApp configures the CLI application +// NewApp configures the CLI application. func NewApp(version string, start time.Time) (app *cli.App) { app = cli.NewApp() app.Name = "gitlab-ci-pipelines-exporter" diff --git a/internal/cmd/run.go b/internal/cmd/run.go index 01fff769..5fe635e3 100644 --- a/internal/cmd/run.go +++ b/internal/cmd/run.go @@ -15,7 +15,7 @@ import ( "github.com/urfave/cli/v2" ) -// Run launches the exporter +// Run launches the exporter. func Run(cliCtx *cli.Context) (int, error) { cfg, err := configure(cliCtx) if err != nil { @@ -54,7 +54,7 @@ func Run(cliCtx *cli.Context) (int, error) { } // health endpoints - health := c.HealthCheckHandler() + health := c.HealthCheckHandler(ctx) mux.HandleFunc("/health/live", health.LiveEndpoint) mux.HandleFunc("/health/ready", health.ReadyEndpoint) @@ -106,5 +106,6 @@ func Run(cliCtx *cli.Context) (int, error) { } log.Info("stopped!") + return 0, nil } diff --git a/internal/cmd/run_test.go b/internal/cmd/run_test.go index ca401b4f..8856ddf9 100644 --- a/internal/cmd/run_test.go +++ b/internal/cmd/run_test.go @@ -16,7 +16,9 @@ import ( func TestRunInvalidConfigFile(t *testing.T) { ctx, flags := NewTestContext() + flags.String("config", "path_does_not_exist", "") + exitCode, err := Run(ctx) assert.Equal(t, 1, exitCode) assert.Error(t, err) diff --git a/internal/cmd/utils.go b/internal/cmd/utils.go index 283d9e07..0ce9ebaf 100644 --- a/internal/cmd/utils.go +++ b/internal/cmd/utils.go @@ -9,10 +9,9 @@ import ( "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" "github.com/mvisonneau/go-helpers/logger" - "github.com/vmihailenco/taskq/v3" - log "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" + "github.com/vmihailenco/taskq/v3" ) var start time.Time @@ -71,6 +70,7 @@ func configure(ctx *cli.Context) (cfg config.Config, err error) { func parseGlobalFlags(ctx *cli.Context) (cfg config.Global, err error) { cfg.InternalMonitoringListenerAddress, err = url.Parse(ctx.String("internal-monitoring-listener-address")) + return } @@ -88,7 +88,7 @@ func exit(exitCode int, err error) cli.ExitCoder { return cli.NewExitError("", exitCode) } -// ExecWrapper gracefully logs and exits our `run` functions +// ExecWrapper gracefully logs and exits our `run` functions. func ExecWrapper(f func(ctx *cli.Context) (int, error)) cli.ActionFunc { return func(ctx *cli.Context) error { return exit(f(ctx)) @@ -114,6 +114,7 @@ func configCliOverrides(ctx *cli.Context, cfg *config.Config) { func assertStringVariableDefined(ctx *cli.Context, k string) { if len(ctx.String(k)) == 0 { _ = cli.ShowAppHelp(ctx) + log.Errorf("'--%s' must be set!", k) os.Exit(2) } diff --git a/internal/cmd/utils_test.go b/internal/cmd/utils_test.go index e5d9f7ac..1844340d 100644 --- a/internal/cmd/utils_test.go +++ b/internal/cmd/utils_test.go @@ -28,10 +28,14 @@ func NewTestContext() (ctx *cli.Context, flags *flag.FlagSet) { } func TestConfigure(t *testing.T) { - var cfg config.Config - var err error + var ( + cfg config.Config + err error + ) + f, err := ioutil.TempFile(".", "test-*.yml") assert.NoError(t, err) + defer os.Remove(f.Name()) // Webhook endpoint enabled @@ -44,17 +48,20 @@ func TestConfigure(t *testing.T) { // Undefined gitlab-token flags.String("gitlab-token", "", "") + _, err = configure(ctx) assert.Error(t, err) // Valid configuration flags.Set("gitlab-token", "secret") + cfg, err = configure(ctx) assert.NoError(t, err) assert.Equal(t, "secret", cfg.Gitlab.Token) // Invalid config file syntax ioutil.WriteFile(f.Name(), []byte("["), 0o644) + cfg, err = configure(ctx) assert.Error(t, err) @@ -72,6 +79,7 @@ server: // Defining the webhook secret token flags.String("webhook-secret-token", "secret", "") + cfg, err = configure(ctx) assert.NoError(t, err) assert.Equal(t, "secret", cfg.Server.Webhook.SecretToken) diff --git a/pkg/config/config.go b/pkg/config/config.go index ced70498..54ff83b0 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -11,7 +11,7 @@ import ( var validate *validator.Validate -// Config represents all the parameters required for the app to be configured properly +// Config represents all the parameters required for the app to be configured properly. type Config struct { // Global .. Global Global `yaml:",omitempty"` @@ -44,7 +44,7 @@ type Config struct { Wildcards []Wildcard `validate:"unique,at-least-1-project-or-wildcard,dive" yaml:"wildcards"` } -// Log holds runtime logging configuration +// Log holds runtime logging configuration. type Log struct { // Log level Level string `default:"info" validate:"required,oneof=trace debug info warning error fatal panic"` @@ -173,8 +173,7 @@ type GarbageCollect struct { } `yaml:"metrics"` } -// UnmarshalYAML allows us to correctly hydrate our configuration using some -// custom logic +// UnmarshalYAML allows us to correctly hydrate our configuration using some custom logic. func (c *Config) UnmarshalYAML(v *yaml.Node) (err error) { type localConfig struct { Log Log `yaml:"log"` @@ -191,6 +190,7 @@ func (c *Config) UnmarshalYAML(v *yaml.Node) (err error) { _cfg := localConfig{} defaults.MustSet(&_cfg) + if err = v.Decode(&_cfg); err != nil { return } @@ -208,6 +208,7 @@ func (c *Config) UnmarshalYAML(v *yaml.Node) (err error) { if err = n.Decode(&p); err != nil { return } + c.Projects = append(c.Projects, p) } @@ -216,6 +217,7 @@ func (c *Config) UnmarshalYAML(v *yaml.Node) (err error) { if err = n.Decode(&w); err != nil { return } + c.Wildcards = append(c.Wildcards, w) } @@ -227,19 +229,22 @@ func (c Config) ToYAML() string { c.Global = Global{} c.Server.Webhook.SecretToken = "*******" c.Gitlab.Token = "*******" + b, err := yaml.Marshal(c) if err != nil { panic(err) } + return string(b) } -// Validate will throw an error if the Config parameters are whether incomplete or incorrects +// Validate will throw an error if the Config parameters are whether incomplete or incorrects. func (c Config) Validate() error { if validate == nil { validate = validator.New() _ = validate.RegisterValidation("at-least-1-project-or-wildcard", ValidateAtLeastOneProjectOrWildcard) } + return validate.Struct(c) } @@ -250,7 +255,7 @@ type SchedulerConfig struct { IntervalSeconds int } -// Log returns some logging fields to showcase the configuration to the enduser +// Log returns some logging fields to showcase the configuration to the enduser. func (sc SchedulerConfig) Log() log.Fields { onInit, scheduled := "no", "no" if sc.OnInit { @@ -268,25 +273,28 @@ func (sc SchedulerConfig) Log() log.Fields { } // ValidateAtLeastOneProjectOrWildcard implements validator.Func -// assess that we have at least one projet or wildcard configured +// assess that we have at least one projet or wildcard configured. func ValidateAtLeastOneProjectOrWildcard(v validator.FieldLevel) bool { return v.Parent().FieldByName("Projects").Len() > 0 || v.Parent().FieldByName("Wildcards").Len() > 0 } -// New returns a new config with the default parameters +// New returns a new config with the default parameters. func New() (c Config) { defaults.MustSet(&c) + return } -// NewProject returns a new project with the config default parameters +// NewProject returns a new project with the config default parameters. func (c Config) NewProject() (p Project) { p.ProjectParameters = c.ProjectDefaults + return } -// NewWildcard returns a new wildcard with the config default parameters +// NewWildcard returns a new wildcard with the config default parameters. func (c Config) NewWildcard() (w Wildcard) { w.ProjectParameters = c.ProjectDefaults + return } diff --git a/pkg/config/global.go b/pkg/config/global.go index fbc28b22..93666f03 100644 --- a/pkg/config/global.go +++ b/pkg/config/global.go @@ -4,7 +4,7 @@ import ( "net/url" ) -// Global is used for globally shared exporter config +// Global is used for globally shared exporter config. type Global struct { // InternalMonitoringListenerAddress can be used to access // some metrics related to the exporter internals diff --git a/pkg/config/parser.go b/pkg/config/parser.go index d2a19431..9811694a 100644 --- a/pkg/config/parser.go +++ b/pkg/config/parser.go @@ -8,19 +8,21 @@ import ( "gopkg.in/yaml.v3" ) -// Format represents the format of the config file +// Format represents the format of the config file. type Format uint8 const ( - // FormatYAML represents a Config written in yaml format + // FormatYAML represents a Config written in yaml format. FormatYAML Format = iota ) // ParseFile reads the content of a file and attempt to unmarshal it -// into a Config +// into a Config. func ParseFile(filename string) (c Config, err error) { - var t Format - var fileBytes []byte + var ( + t Format + fileBytes []byte + ) // Figure out what type of config file we provided t, err = GetTypeFromFileExtension(filename) @@ -38,11 +40,8 @@ func ParseFile(filename string) (c Config, err error) { return Parse(t, fileBytes) } -// Parse unmarshal provided bytes with given ConfigType into a Config object -func Parse(f Format, bytes []byte) (Config, error) { - cfg := Config{} - var err error - +// Parse unmarshal provided bytes with given ConfigType into a Config object. +func Parse(f Format, bytes []byte) (cfg Config, err error) { switch f { case FormatYAML: err = yaml.Unmarshal(bytes, &cfg) @@ -56,18 +55,18 @@ func Parse(f Format, bytes []byte) (Config, error) { cfg.Gitlab.HealthURL = fmt.Sprintf("%s/-/health", cfg.Gitlab.URL) } - return cfg, err + return } // GetTypeFromFileExtension returns the ConfigType based upon the extension of -// the file +// the file. func GetTypeFromFileExtension(filename string) (f Format, err error) { - ext := filepath.Ext(filename) - switch ext { + switch ext := filepath.Ext(filename); ext { case ".yml", ".yaml": f = FormatYAML default: err = fmt.Errorf("unsupported config type '%s', expected .y(a)ml", ext) } + return } diff --git a/pkg/config/project.go b/pkg/config/project.go index 9f5c0570..251d69c6 100644 --- a/pkg/config/project.go +++ b/pkg/config/project.go @@ -4,9 +4,9 @@ import ( "github.com/creasty/defaults" ) -// ProjectParameters for the fetching configuration of Projects and Wildcards +// ProjectParameters for the fetching configuration of Projects and Wildcards. type ProjectParameters struct { - // From handles ProjectPullParameters configuration + // From handles ProjectPullParameters configuration. Pull ProjectPull `yaml:"pull"` // Whether or not to export all pipeline/job statuses (being 0) or solely the one of the last job (being 1). @@ -65,33 +65,33 @@ type ProjectPullRefsBranches struct { // ProjectPullRefsTags .. type ProjectPullRefsTags struct { - // Monitor pipelines related to project tags + // Monitor pipelines related to project tags. Enabled bool `default:"true" yaml:"enabled"` - // Filter for tags to include + // Filter for tags to include. Regexp string `default:".*" yaml:"regexp"` - // Only keep most 'n' recently updated tags + // Only keep most 'n' recently updated tags. MostRecent uint `default:"0" yaml:"most_recent"` // If the most recent pipeline for the tag was last updated at - // at time greater than this value the metrics won't be exported + // at time greater than this value the metrics won't be exported. MaxAgeSeconds uint `default:"0" yaml:"max_age_seconds"` - // Prevent exporting metrics for deleted tags + // Prevent exporting metrics for deleted tags. ExcludeDeleted bool `default:"true" yaml:"exclude_deleted"` } // ProjectPullRefsMergeRequests .. type ProjectPullRefsMergeRequests struct { - // Monitor pipelines related to project merge requests + // Monitor pipelines related to project merge requests. Enabled bool `yaml:"enabled"` - // Only keep most 'n' recently updated merge requests + // Only keep most 'n' recently updated merge requests. MostRecent uint `default:"0" yaml:"most_recent"` // If the most recent pipeline for the merge request was last updated at - // at time greater than this value the metrics won't be exported + // at time greater than this value the metrics won't be exported. MaxAgeSeconds uint `default:"0" yaml:"max_age_seconds"` } @@ -103,55 +103,56 @@ type ProjectPullPipeline struct { // ProjectPullPipelineJobs .. type ProjectPullPipelineJobs struct { - // Enabled set to true will pull pipeline jobs related metrics + // Enabled set to true will pull pipeline jobs related metrics. Enabled bool `default:"false" yaml:"enabled"` - // Pull pipeline jobs from child/downstream pipelines + // Pull pipeline jobs from child/downstream pipelines. FromChildPipelines ProjectPullPipelineJobsFromChildPipelines `yaml:"from_child_pipelines"` - // Configure the export of the runner description which ran the job + // Configure the export of the runner description which ran the job. RunnerDescription ProjectPullPipelineJobsRunnerDescription `yaml:"runner_description"` } // ProjectPullPipelineJobsFromChildPipelines .. type ProjectPullPipelineJobsFromChildPipelines struct { - // Enabled set to true will pull pipeline jobs from child/downstream pipelines related metrics + // Enabled set to true will pull pipeline jobs from child/downstream pipelines related metrics. Enabled bool `default:"true" yaml:"enabled"` } // ProjectPullPipelineJobsRunnerDescription .. type ProjectPullPipelineJobsRunnerDescription struct { - // Enabled set to true will export the description of the runner which ran the job + // Enabled set to true will export the description of the runner which ran the job. Enabled bool `default:"true" yaml:"enabled"` - // Regular expression to be able to reduce the cardinality of the exported value when necessary + // Regular expression to be able to reduce the cardinality of the exported value when necessary. AggregationRegexp string `default:"shared-runners-manager-(\\d*)\\.gitlab\\.com" yaml:"aggregation_regexp"` } // ProjectPullPipelineVariables .. type ProjectPullPipelineVariables struct { - // Enabled set to true will attempt to retrieve variables included in the pipeline + // Enabled set to true will attempt to retrieve variables included in the pipeline. Enabled bool `default:"false" yaml:"enabled"` - // Regexp to filter pipeline variables values to fetch + // Regexp to filter pipeline variables values to fetch. Regexp string `default:".*" yaml:"regexp"` } -// Project holds information about a GitLab project +// Project holds information about a GitLab project. type Project struct { - // ProjectParameters holds parameters specific to this project + // ProjectParameters holds parameters specific to this project. ProjectParameters `yaml:",inline"` - // Name is actually what is commonly referred as path_with_namespace on GitLab + // Name is actually what is commonly referred as path_with_namespace on GitLab. Name string `yaml:"name"` } // Projects .. type Projects []Project -// NewProject returns a new project composed with the default parameters +// NewProject returns a new project composed with the default parameters. func NewProject(name string) (p Project) { defaults.MustSet(&p) p.Name = name + return } diff --git a/pkg/config/wildcard.go b/pkg/config/wildcard.go index dd3bad4a..eb4c4d79 100644 --- a/pkg/config/wildcard.go +++ b/pkg/config/wildcard.go @@ -4,10 +4,10 @@ import ( "github.com/creasty/defaults" ) -// Wildcard is a specific handler to dynamically search projects +// Wildcard is a specific handler to dynamically search projects. type Wildcard struct { // ProjectParameters holds parameters specific to the projects which - // will be discovered using this wildcard + // will be discovered using this wildcard. ProjectParameters `yaml:",inline"` Search string `yaml:"search"` @@ -25,8 +25,9 @@ type WildcardOwner struct { // Wildcards .. type Wildcards []Wildcard -// NewWildcard returns a new wildcard with the default parameters +// NewWildcard returns a new wildcard with the default parameters. func NewWildcard() (w Wildcard) { defaults.MustSet(&w) + return } diff --git a/pkg/controller/collectors.go b/pkg/controller/collectors.go index e32b724d..126f1e16 100644 --- a/pkg/controller/collectors.go +++ b/pkg/controller/collectors.go @@ -11,7 +11,7 @@ var ( statusesList = [...]string{"created", "waiting_for_resource", "preparing", "pending", "running", "success", "failed", "canceled", "skipped", "manual", "scheduled"} ) -// NewInternalCollectorCurrentlyQueuedTasksCount returns a new collector for the gcpe_currently_queued_tasks_count metric +// NewInternalCollectorCurrentlyQueuedTasksCount returns a new collector for the gcpe_currently_queued_tasks_count metric. func NewInternalCollectorCurrentlyQueuedTasksCount() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -22,7 +22,7 @@ func NewInternalCollectorCurrentlyQueuedTasksCount() prometheus.Collector { ) } -// NewInternalCollectorEnvironmentsCount returns a new collector for the gcpe_environments_count metric +// NewInternalCollectorEnvironmentsCount returns a new collector for the gcpe_environments_count metric. func NewInternalCollectorEnvironmentsCount() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -33,7 +33,7 @@ func NewInternalCollectorEnvironmentsCount() prometheus.Collector { ) } -// NewInternalCollectorExecutedTasksCount returns a new collector for the gcpe_executed_tasks_count metric +// NewInternalCollectorExecutedTasksCount returns a new collector for the gcpe_executed_tasks_count metric. func NewInternalCollectorExecutedTasksCount() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -44,7 +44,7 @@ func NewInternalCollectorExecutedTasksCount() prometheus.Collector { ) } -// NewInternalCollectorGitLabAPIRequestsCount returns a new collector for the gcpe_gitlab_api_requests_count metric +// NewInternalCollectorGitLabAPIRequestsCount returns a new collector for the gcpe_gitlab_api_requests_count metric. func NewInternalCollectorGitLabAPIRequestsCount() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -55,7 +55,7 @@ func NewInternalCollectorGitLabAPIRequestsCount() prometheus.Collector { ) } -// NewInternalCollectorGitLabAPIRequestsRemaining returns a new collector for the gcpe_gitlab_api_requests_remaining metric +// NewInternalCollectorGitLabAPIRequestsRemaining returns a new collector for the gcpe_gitlab_api_requests_remaining metric. func NewInternalCollectorGitLabAPIRequestsRemaining() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -66,7 +66,7 @@ func NewInternalCollectorGitLabAPIRequestsRemaining() prometheus.Collector { ) } -// NewInternalCollectorGitLabAPIRequestsLimit returns a new collector for the gcpe_gitlab_api_requests_limit metric +// NewInternalCollectorGitLabAPIRequestsLimit returns a new collector for the gcpe_gitlab_api_requests_limit metric. func NewInternalCollectorGitLabAPIRequestsLimit() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -77,7 +77,7 @@ func NewInternalCollectorGitLabAPIRequestsLimit() prometheus.Collector { ) } -// NewInternalCollectorMetricsCount returns a new collector for the gcpe_metrics_count metric +// NewInternalCollectorMetricsCount returns a new collector for the gcpe_metrics_count metric. func NewInternalCollectorMetricsCount() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -88,7 +88,7 @@ func NewInternalCollectorMetricsCount() prometheus.Collector { ) } -// NewInternalCollectorProjectsCount returns a new collector for the gcpe_projects_count metric +// NewInternalCollectorProjectsCount returns a new collector for the gcpe_projects_count metric. func NewInternalCollectorProjectsCount() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -99,7 +99,7 @@ func NewInternalCollectorProjectsCount() prometheus.Collector { ) } -// NewInternalCollectorRefsCount returns a new collector for the gcpe_refs_count metric +// NewInternalCollectorRefsCount returns a new collector for the gcpe_refs_count metric. func NewInternalCollectorRefsCount() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -110,7 +110,7 @@ func NewInternalCollectorRefsCount() prometheus.Collector { ) } -// NewCollectorCoverage returns a new collector for the gitlab_ci_pipeline_coverage metric +// NewCollectorCoverage returns a new collector for the gitlab_ci_pipeline_coverage metric. func NewCollectorCoverage() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -121,7 +121,7 @@ func NewCollectorCoverage() prometheus.Collector { ) } -// NewCollectorDurationSeconds returns a new collector for the gitlab_ci_pipeline_duration_seconds metric +// NewCollectorDurationSeconds returns a new collector for the gitlab_ci_pipeline_duration_seconds metric. func NewCollectorDurationSeconds() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -132,7 +132,7 @@ func NewCollectorDurationSeconds() prometheus.Collector { ) } -// NewCollectorQueuedDurationSeconds returns a new collector for the gitlab_ci_pipeline_queued_duration_seconds metric +// NewCollectorQueuedDurationSeconds returns a new collector for the gitlab_ci_pipeline_queued_duration_seconds metric. func NewCollectorQueuedDurationSeconds() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -143,7 +143,7 @@ func NewCollectorQueuedDurationSeconds() prometheus.Collector { ) } -// NewCollectorEnvironmentBehindCommitsCount returns a new collector for the gitlab_ci_environment_behind_commits_count metric +// NewCollectorEnvironmentBehindCommitsCount returns a new collector for the gitlab_ci_environment_behind_commits_count metric. func NewCollectorEnvironmentBehindCommitsCount() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -154,7 +154,7 @@ func NewCollectorEnvironmentBehindCommitsCount() prometheus.Collector { ) } -// NewCollectorEnvironmentBehindDurationSeconds returns a new collector for the gitlab_ci_environment_behind_duration_seconds metric +// NewCollectorEnvironmentBehindDurationSeconds returns a new collector for the gitlab_ci_environment_behind_duration_seconds metric. func NewCollectorEnvironmentBehindDurationSeconds() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -165,7 +165,7 @@ func NewCollectorEnvironmentBehindDurationSeconds() prometheus.Collector { ) } -// NewCollectorEnvironmentDeploymentCount returns a new collector for the gitlab_ci_environment_deployment_count metric +// NewCollectorEnvironmentDeploymentCount returns a new collector for the gitlab_ci_environment_deployment_count metric. func NewCollectorEnvironmentDeploymentCount() prometheus.Collector { return prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -176,7 +176,7 @@ func NewCollectorEnvironmentDeploymentCount() prometheus.Collector { ) } -// NewCollectorEnvironmentDeploymentDurationSeconds returns a new collector for the gitlab_ci_environment_deployment_duration_seconds metric +// NewCollectorEnvironmentDeploymentDurationSeconds returns a new collector for the gitlab_ci_environment_deployment_duration_seconds metric. func NewCollectorEnvironmentDeploymentDurationSeconds() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -187,7 +187,7 @@ func NewCollectorEnvironmentDeploymentDurationSeconds() prometheus.Collector { ) } -// NewCollectorEnvironmentDeploymentJobID returns a new collector for the gitlab_ci_environment_deployment_id metric +// NewCollectorEnvironmentDeploymentJobID returns a new collector for the gitlab_ci_environment_deployment_id metric. func NewCollectorEnvironmentDeploymentJobID() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -198,7 +198,7 @@ func NewCollectorEnvironmentDeploymentJobID() prometheus.Collector { ) } -// NewCollectorEnvironmentDeploymentStatus returns a new collector for the gitlab_ci_environment_deployment_status metric +// NewCollectorEnvironmentDeploymentStatus returns a new collector for the gitlab_ci_environment_deployment_status metric. func NewCollectorEnvironmentDeploymentStatus() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -209,7 +209,7 @@ func NewCollectorEnvironmentDeploymentStatus() prometheus.Collector { ) } -// NewCollectorEnvironmentDeploymentTimestamp returns a new collector for the gitlab_ci_environment_deployment_timestamp metric +// NewCollectorEnvironmentDeploymentTimestamp returns a new collector for the gitlab_ci_environment_deployment_timestamp metric. func NewCollectorEnvironmentDeploymentTimestamp() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -220,7 +220,7 @@ func NewCollectorEnvironmentDeploymentTimestamp() prometheus.Collector { ) } -// NewCollectorEnvironmentInformation returns a new collector for the gitlab_ci_environment_information metric +// NewCollectorEnvironmentInformation returns a new collector for the gitlab_ci_environment_information metric. func NewCollectorEnvironmentInformation() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -231,7 +231,7 @@ func NewCollectorEnvironmentInformation() prometheus.Collector { ) } -// NewCollectorID returns a new collector for the gitlab_ci_pipeline_id metric +// NewCollectorID returns a new collector for the gitlab_ci_pipeline_id metric. func NewCollectorID() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -242,7 +242,7 @@ func NewCollectorID() prometheus.Collector { ) } -// NewCollectorJobArtifactSizeBytes returns a new collector for the gitlab_ci_pipeline_job_artifact_size_bytes metric +// NewCollectorJobArtifactSizeBytes returns a new collector for the gitlab_ci_pipeline_job_artifact_size_bytes metric. func NewCollectorJobArtifactSizeBytes() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -253,7 +253,7 @@ func NewCollectorJobArtifactSizeBytes() prometheus.Collector { ) } -// NewCollectorJobDurationSeconds returns a new collector for the gitlab_ci_pipeline_job_duration_seconds metric +// NewCollectorJobDurationSeconds returns a new collector for the gitlab_ci_pipeline_job_duration_seconds metric. func NewCollectorJobDurationSeconds() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -264,7 +264,7 @@ func NewCollectorJobDurationSeconds() prometheus.Collector { ) } -// NewCollectorJobID returns a new collector for the gitlab_ci_pipeline_job_id metric +// NewCollectorJobID returns a new collector for the gitlab_ci_pipeline_job_id metric. func NewCollectorJobID() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -275,7 +275,7 @@ func NewCollectorJobID() prometheus.Collector { ) } -// NewCollectorJobQueuedDurationSeconds returns a new collector for the gitlab_ci_pipeline_job_queued_duration_seconds metric +// NewCollectorJobQueuedDurationSeconds returns a new collector for the gitlab_ci_pipeline_job_queued_duration_seconds metric. func NewCollectorJobQueuedDurationSeconds() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -286,7 +286,7 @@ func NewCollectorJobQueuedDurationSeconds() prometheus.Collector { ) } -// NewCollectorJobRunCount returns a new collector for the gitlab_ci_pipeline_job_run_count metric +// NewCollectorJobRunCount returns a new collector for the gitlab_ci_pipeline_job_run_count metric. func NewCollectorJobRunCount() prometheus.Collector { return prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -297,7 +297,7 @@ func NewCollectorJobRunCount() prometheus.Collector { ) } -// NewCollectorJobStatus returns a new collector for the gitlab_ci_pipeline_job_status metric +// NewCollectorJobStatus returns a new collector for the gitlab_ci_pipeline_job_status metric. func NewCollectorJobStatus() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -308,7 +308,7 @@ func NewCollectorJobStatus() prometheus.Collector { ) } -// NewCollectorJobTimestamp returns a new collector for the gitlab_ci_pipeline_job_timestamp metric +// NewCollectorJobTimestamp returns a new collector for the gitlab_ci_pipeline_job_timestamp metric. func NewCollectorJobTimestamp() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -319,7 +319,7 @@ func NewCollectorJobTimestamp() prometheus.Collector { ) } -// NewCollectorStatus returns a new collector for the gitlab_ci_pipeline_status metric +// NewCollectorStatus returns a new collector for the gitlab_ci_pipeline_status metric. func NewCollectorStatus() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -330,7 +330,7 @@ func NewCollectorStatus() prometheus.Collector { ) } -// NewCollectorTimestamp returns a new collector for the gitlab_ci_pipeline_timestamp metric +// NewCollectorTimestamp returns a new collector for the gitlab_ci_pipeline_timestamp metric. func NewCollectorTimestamp() prometheus.Collector { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ @@ -341,7 +341,7 @@ func NewCollectorTimestamp() prometheus.Collector { ) } -// NewCollectorRunCount returns a new collector for the gitlab_ci_pipeline_run_count metric +// NewCollectorRunCount returns a new collector for the gitlab_ci_pipeline_run_count metric. func NewCollectorRunCount() prometheus.Collector { return prometheus.NewCounterVec( prometheus.CounterOpts{ diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 892d5b4a..ba201692 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -15,7 +15,7 @@ import ( "github.com/vmihailenco/taskq/v3" ) -// Controller holds the necessary clients to run the app and handle requests +// Controller holds the necessary clients to run the app and handle requests. type Controller struct { Config config.Config Redis *redis.Client @@ -28,7 +28,7 @@ type Controller struct { UUID uuid.UUID } -// New creates a new controller +// New creates a new controller. func New(ctx context.Context, cfg config.Config, version string) (c Controller, err error) { c.Config = cfg c.UUID = uuid.New() @@ -76,8 +76,8 @@ func (c *Controller) registerTasks() { } } -func (c *Controller) unqueueTask(tt schemas.TaskType, uniqueID string) { - if err := c.Store.UnqueueTask(tt, uniqueID); err != nil { +func (c *Controller) unqueueTask(ctx context.Context, tt schemas.TaskType, uniqueID string) { + if err := c.Store.UnqueueTask(ctx, tt, uniqueID); err != nil { log.WithFields(log.Fields{ "task_type": tt, "task_unique_id": uniqueID, @@ -89,7 +89,7 @@ func (c *Controller) configureGitlab(cfg config.Gitlab, version string) (err err var rl ratelimit.Limiter if c.Redis != nil { - rl = ratelimit.NewRedisLimiter(context.Background(), c.Redis, cfg.MaximumRequestsPerSecond) + rl = ratelimit.NewRedisLimiter(c.Redis, cfg.MaximumRequestsPerSecond) } else { rl = ratelimit.NewLocalLimiter(cfg.MaximumRequestsPerSecond) } @@ -102,18 +102,21 @@ func (c *Controller) configureGitlab(cfg config.Gitlab, version string) (err err RateLimiter: rl, ReadinessURL: cfg.HealthURL, }) + return } func (c *Controller) configureRedis(url string) (err error) { if len(url) <= 0 { log.Debug("redis url is not configured, skipping configuration & using local driver") + return } log.Info("redis url configured, initializing connection..") var opt *redis.Options + if opt, err = redis.ParseURL(url); err != nil { return } diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index a3c0ade5..f78291d2 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -13,10 +13,12 @@ import ( func newMockedGitlabAPIServer() (mux *http.ServeMux, srv *httptest.Server) { mux = http.NewServeMux() srv = httptest.NewServer(mux) + return } -func newTestController(cfg config.Config) (c Controller, mux *http.ServeMux, srv *httptest.Server) { +func newTestController(cfg config.Config) (ctx context.Context, c Controller, mux *http.ServeMux, srv *httptest.Server) { + ctx = context.Background() mux, srv = newMockedGitlabAPIServer() cfg.Gitlab.URL = srv.URL @@ -25,6 +27,7 @@ func newTestController(cfg config.Config) (c Controller, mux *http.ServeMux, srv } c, _ = New(context.Background(), cfg, "0.0.0-ci") + return } diff --git a/pkg/controller/environments.go b/pkg/controller/environments.go index 823d72ed..97ade9e5 100644 --- a/pkg/controller/environments.go +++ b/pkg/controller/environments.go @@ -10,21 +10,23 @@ import ( // PullEnvironmentsFromProject .. func (c *Controller) PullEnvironmentsFromProject(ctx context.Context, p schemas.Project) (err error) { var envs schemas.Environments - envs, err = c.Gitlab.GetProjectEnvironments(p) + + envs, err = c.Gitlab.GetProjectEnvironments(ctx, p) if err != nil { return } for k := range envs { var exists bool - exists, err = c.Store.EnvironmentExists(k) + + exists, err = c.Store.EnvironmentExists(ctx, k) if err != nil { return } if !exists { env := envs[k] - if err = c.UpdateEnvironment(&env); err != nil { + if err = c.UpdateEnvironment(ctx, &env); err != nil { return } @@ -37,12 +39,13 @@ func (c *Controller) PullEnvironmentsFromProject(ctx context.Context, p schemas. c.ScheduleTask(ctx, schemas.TaskTypePullEnvironmentMetrics, string(env.Key()), env) } } + return } // UpdateEnvironment .. -func (c *Controller) UpdateEnvironment(env *schemas.Environment) error { - pulledEnv, err := c.Gitlab.GetEnvironment(env.ProjectName, env.ID) +func (c *Controller) UpdateEnvironment(ctx context.Context, env *schemas.Environment) error { + pulledEnv, err := c.Gitlab.GetEnvironment(ctx, env.ProjectName, env.ID) if err != nil { return err } @@ -51,31 +54,35 @@ func (c *Controller) UpdateEnvironment(env *schemas.Environment) error { env.ExternalURL = pulledEnv.ExternalURL env.LatestDeployment = pulledEnv.LatestDeployment - return c.Store.SetEnvironment(*env) + return c.Store.SetEnvironment(ctx, *env) } // PullEnvironmentMetrics .. -func (c *Controller) PullEnvironmentMetrics(env schemas.Environment) (err error) { +func (c *Controller) PullEnvironmentMetrics(ctx context.Context, env schemas.Environment) (err error) { // At scale, the scheduled environment may be behind the actual state being stored // to avoid issues, we refresh it from the store before manipulating it - if err := c.Store.GetEnvironment(&env); err != nil { + if err := c.Store.GetEnvironment(ctx, &env); err != nil { return err } // Save the existing deployment ID before we updated environment from the API deploymentJobID := env.LatestDeployment.JobID - if err = c.UpdateEnvironment(&env); err != nil { + + if err = c.UpdateEnvironment(ctx, &env); err != nil { return } - infoLabels := env.InformationLabelsValues() - var commitDate float64 + var ( + infoLabels = env.InformationLabelsValues() + commitDate float64 + ) + switch env.LatestDeployment.RefKind { case schemas.RefKindBranch: - infoLabels["latest_commit_short_id"], commitDate, err = c.Gitlab.GetBranchLatestCommit(env.ProjectName, env.LatestDeployment.RefName) + infoLabels["latest_commit_short_id"], commitDate, err = c.Gitlab.GetBranchLatestCommit(ctx, env.ProjectName, env.LatestDeployment.RefName) case schemas.RefKindTag: // TODO: Review how to manage this in a nicier fashion - infoLabels["latest_commit_short_id"], commitDate, err = c.Gitlab.GetProjectMostRecentTagCommit(env.ProjectName, ".*") + infoLabels["latest_commit_short_id"], commitDate, err = c.Gitlab.GetProjectMostRecentTagCommit(ctx, env.ProjectName, ".*") default: infoLabels["latest_commit_short_id"] = env.LatestDeployment.CommitShortID commitDate = env.LatestDeployment.Timestamp @@ -104,27 +111,29 @@ func (c *Controller) PullEnvironmentMetrics(env schemas.Environment) (err error) } var commitCount int - if err = c.Store.GetMetric(&infoMetric); err != nil { + + if err = c.Store.GetMetric(ctx, &infoMetric); err != nil { return err } if infoMetric.Labels["latest_commit_short_id"] != infoLabels["latest_commit_short_id"] || infoMetric.Labels["current_commit_short_id"] != infoLabels["current_commit_short_id"] { - commitCount, err = c.Gitlab.GetCommitCountBetweenRefs(env.ProjectName, infoLabels["current_commit_short_id"], infoLabels["latest_commit_short_id"]) + commitCount, err = c.Gitlab.GetCommitCountBetweenRefs(ctx, env.ProjectName, infoLabels["current_commit_short_id"], infoLabels["latest_commit_short_id"]) if err != nil { return err } + envBehindCommitCount = float64(commitCount) } else { // TODO: Find a more efficient way - if err = c.Store.GetMetric(&behindCommitsCountMetric); err != nil { + if err = c.Store.GetMetric(ctx, &behindCommitsCountMetric); err != nil { return err } envBehindCommitCount = behindCommitsCountMetric.Value } } - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindEnvironmentBehindCommitsCount, Labels: env.DefaultLabelsValues(), Value: envBehindCommitCount, @@ -139,31 +148,34 @@ func (c *Controller) PullEnvironmentMetrics(env schemas.Environment) (err error) Labels: env.DefaultLabelsValues(), } - storeGetMetric(c.Store, &envDeploymentCount) + storeGetMetric(ctx, c.Store, &envDeploymentCount) + if env.LatestDeployment.JobID > deploymentJobID { envDeploymentCount.Value++ } - storeSetMetric(c.Store, envDeploymentCount) - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, envDeploymentCount) + + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindEnvironmentBehindDurationSeconds, Labels: env.DefaultLabelsValues(), Value: envBehindDurationSeconds, }) - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindEnvironmentDeploymentDurationSeconds, Labels: env.DefaultLabelsValues(), Value: env.LatestDeployment.DurationSeconds, }) - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindEnvironmentDeploymentJobID, Labels: env.DefaultLabelsValues(), Value: float64(env.LatestDeployment.JobID), }) emitStatusMetric( + ctx, c.Store, schemas.MetricKindEnvironmentDeploymentStatus, env.DefaultLabelsValues(), @@ -172,13 +184,13 @@ func (c *Controller) PullEnvironmentMetrics(env schemas.Environment) (err error) env.OutputSparseStatusMetrics, ) - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindEnvironmentDeploymentTimestamp, Labels: env.DefaultLabelsValues(), Value: env.LatestDeployment.Timestamp, }) - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindEnvironmentInformation, Labels: infoLabels, Value: 1, diff --git a/pkg/controller/environments_test.go b/pkg/controller/environments_test.go index 6518f631..0eba6ce3 100644 --- a/pkg/controller/environments_test.go +++ b/pkg/controller/environments_test.go @@ -1,7 +1,6 @@ package controller import ( - "context" "fmt" "net/http" "testing" @@ -12,7 +11,7 @@ import ( ) func TestPullEnvironmentsFromProject(t *testing.T) { - c, mux, srv := newTestController(config.Config{}) + ctx, c, mux, srv := newTestController(config.Config{}) defer srv.Close() mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/environments"), @@ -49,9 +48,9 @@ func TestPullEnvironmentsFromProject(t *testing.T) { p := schemas.NewProject("foo") p.Pull.Environments.Regexp = "^prod" - assert.NoError(t, c.PullEnvironmentsFromProject(context.Background(), p)) + assert.NoError(t, c.PullEnvironmentsFromProject(ctx, p)) - storedEnvironments, _ := c.Store.Environments() + storedEnvironments, _ := c.Store.Environments(ctx) expectedEnvironments := schemas.Environments{ "54146361": schemas.Environment{ ProjectName: "foo", @@ -76,7 +75,7 @@ func TestPullEnvironmentsFromProject(t *testing.T) { } func TestPullEnvironmentMetricsSucceed(t *testing.T) { - c, mux, srv := newTestController(config.Config{}) + ctx, c, mux, srv := newTestController(config.Config{}) defer srv.Close() mux.HandleFunc("/api/v4/projects/foo/environments/1", @@ -124,10 +123,10 @@ func TestPullEnvironmentMetricsSucceed(t *testing.T) { } // Metrics pull shall succeed - assert.NoError(t, c.PullEnvironmentMetrics(env)) + assert.NoError(t, c.PullEnvironmentMetrics(ctx, env)) // Check if all the metrics exist - metrics, _ := c.Store.Metrics() + metrics, _ := c.Store.Metrics(ctx) labels := map[string]string{ "project": "foo", "environment": "prod", diff --git a/pkg/controller/garbage_collector.go b/pkg/controller/garbage_collector.go index 3f779946..116a97a8 100644 --- a/pkg/controller/garbage_collector.go +++ b/pkg/controller/garbage_collector.go @@ -12,11 +12,11 @@ import ( ) // GarbageCollectProjects .. -func (c *Controller) GarbageCollectProjects(_ context.Context) error { +func (c *Controller) GarbageCollectProjects(ctx context.Context) error { log.Info("starting 'projects' garbage collection") defer log.Info("ending 'projects' garbage collection") - storedProjects, err := c.Store.Projects() + storedProjects, err := c.Store.Projects(ctx) if err != nil { return err } @@ -29,7 +29,7 @@ func (c *Controller) GarbageCollectProjects(_ context.Context) error { // Loop through what can be found from the wildcards for _, w := range c.Config.Wildcards { - foundProjects, err := c.Gitlab.ListProjects(w) + foundProjects, err := c.Gitlab.ListProjects(ctx, w) if err != nil { return err } @@ -44,7 +44,7 @@ func (c *Controller) GarbageCollectProjects(_ context.Context) error { }).Debug("found projects to garbage collect") for k, p := range storedProjects { - if err = c.Store.DelProject(k); err != nil { + if err = c.Store.DelProject(ctx, k); err != nil { return err } @@ -57,41 +57,44 @@ func (c *Controller) GarbageCollectProjects(_ context.Context) error { } // GarbageCollectEnvironments .. -func (c *Controller) GarbageCollectEnvironments(_ context.Context) error { +func (c *Controller) GarbageCollectEnvironments(ctx context.Context) error { log.Info("starting 'environments' garbage collection") defer log.Info("ending 'environments' garbage collection") - storedEnvironments, err := c.Store.Environments() + storedEnvironments, err := c.Store.Environments(ctx) if err != nil { return err } envProjects := make(map[schemas.Project]bool) + for _, env := range storedEnvironments { p := schemas.NewProject(env.ProjectName) - projectExists, err := c.Store.ProjectExists(p.Key()) + projectExists, err := c.Store.ProjectExists(ctx, p.Key()) if err != nil { return err } // If the project does not exist anymore, delete the environment if !projectExists { - if err = deleteEnv(c.Store, env, "non-existent-project"); err != nil { + if err = deleteEnv(ctx, c.Store, env, "non-existent-project"); err != nil { return err } + continue } - if err = c.Store.GetProject(&p); err != nil { + if err = c.Store.GetProject(ctx, &p); err != nil { return err } // If the environment is not configured to be pulled anymore, delete it if !p.Pull.Environments.Enabled { - if err = deleteEnv(c.Store, env, "project-pull-environments-disabled"); err != nil { + if err = deleteEnv(ctx, c.Store, env, "project-pull-environments-disabled"); err != nil { return err } + continue } @@ -102,9 +105,10 @@ func (c *Controller) GarbageCollectEnvironments(_ context.Context) error { // If the environment is not configured to be pulled anymore, delete it re := regexp.MustCompile(p.Pull.Environments.Regexp) if !re.MatchString(env.Name) { - if err = deleteEnv(c.Store, env, "environment-not-in-regexp"); err != nil { + if err = deleteEnv(ctx, c.Store, env, "environment-not-in-regexp"); err != nil { return err } + continue } @@ -112,7 +116,7 @@ func (c *Controller) GarbageCollectEnvironments(_ context.Context) error { if env.OutputSparseStatusMetrics != p.OutputSparseStatusMetrics { env.OutputSparseStatusMetrics = p.OutputSparseStatusMetrics - if err = c.Store.SetEnvironment(env); err != nil { + if err = c.Store.SetEnvironment(ctx, env); err != nil { return err } @@ -125,8 +129,9 @@ func (c *Controller) GarbageCollectEnvironments(_ context.Context) error { // Refresh the environments from the API existingEnvs := make(schemas.Environments) + for p := range envProjects { - projectEnvs, err := c.Gitlab.GetProjectEnvironments(p) + projectEnvs, err := c.Gitlab.GetProjectEnvironments(ctx, p) if err != nil { return err } @@ -136,14 +141,14 @@ func (c *Controller) GarbageCollectEnvironments(_ context.Context) error { } } - storedEnvironments, err = c.Store.Environments() + storedEnvironments, err = c.Store.Environments(ctx) if err != nil { return err } for k, env := range storedEnvironments { if _, exists := existingEnvs[k]; !exists { - if err = deleteEnv(c.Store, env, "non-existent-environment"); err != nil { + if err = deleteEnv(ctx, c.Store, env, "non-existent-environment"); err != nil { return err } } @@ -153,54 +158,59 @@ func (c *Controller) GarbageCollectEnvironments(_ context.Context) error { } // GarbageCollectRefs .. -func (c *Controller) GarbageCollectRefs(_ context.Context) error { +func (c *Controller) GarbageCollectRefs(ctx context.Context) error { log.Info("starting 'refs' garbage collection") defer log.Info("ending 'refs' garbage collection") - storedRefs, err := c.Store.Refs() + storedRefs, err := c.Store.Refs(ctx) if err != nil { return err } for _, ref := range storedRefs { - projectExists, err := c.Store.ProjectExists(ref.Project.Key()) + projectExists, err := c.Store.ProjectExists(ctx, ref.Project.Key()) if err != nil { return err } // If the project does not exist anymore, delete the ref if !projectExists { - if err = deleteRef(c.Store, ref, "non-existent-project"); err != nil { + if err = deleteRef(ctx, c.Store, ref, "non-existent-project"); err != nil { return err } + continue } // If the ref is not configured to be pulled anymore, delete the ref var re *regexp.Regexp + if re, err = schemas.GetRefRegexp(ref.Project.Pull.Refs, ref.Kind); err != nil { - if err = deleteRef(c.Store, ref, "invalid-ref-kind"); err != nil { + if err = deleteRef(ctx, c.Store, ref, "invalid-ref-kind"); err != nil { return err } } if !re.MatchString(ref.Name) { - if err = deleteRef(c.Store, ref, "ref-not-matching-regexp"); err != nil { + if err = deleteRef(ctx, c.Store, ref, "ref-not-matching-regexp"); err != nil { return err } } // Check if the latest configuration of the project in store matches the ref one p := ref.Project - if err = c.Store.GetProject(&p); err != nil { + + if err = c.Store.GetProject(ctx, &p); err != nil { return err } if !reflect.DeepEqual(ref.Project, p) { ref.Project = p - if err = c.Store.SetRef(ref); err != nil { + + if err = c.Store.SetRef(ctx, ref); err != nil { return err } + log.WithFields(log.Fields{ "project-name": ref.Project.Name, "ref": ref.Name, @@ -209,14 +219,15 @@ func (c *Controller) GarbageCollectRefs(_ context.Context) error { } // Refresh the refs from the API - projects, err := c.Store.Projects() + projects, err := c.Store.Projects(ctx) if err != nil { return err } expectedRefs := make(map[schemas.RefKey]bool) + for _, p := range projects { - refs, err := c.GetRefs(p) + refs, err := c.GetRefs(ctx, p) if err != nil { return err } @@ -227,14 +238,14 @@ func (c *Controller) GarbageCollectRefs(_ context.Context) error { } // Refresh the stored refs as we may have already removed some - storedRefs, err = c.Store.Refs() + storedRefs, err = c.Store.Refs(ctx) if err != nil { return err } for k, ref := range storedRefs { if _, expected := expectedRefs[k]; !expected { - if err = deleteRef(c.Store, ref, "not-expected"); err != nil { + if err = deleteRef(ctx, c.Store, ref, "not-expected"); err != nil { return err } } @@ -244,21 +255,21 @@ func (c *Controller) GarbageCollectRefs(_ context.Context) error { } // GarbageCollectMetrics .. -func (c *Controller) GarbageCollectMetrics(_ context.Context) error { +func (c *Controller) GarbageCollectMetrics(ctx context.Context) error { log.Info("starting 'metrics' garbage collection") defer log.Info("ending 'metrics' garbage collection") - storedEnvironments, err := c.Store.Environments() + storedEnvironments, err := c.Store.Environments(ctx) if err != nil { return err } - storedRefs, err := c.Store.Refs() + storedRefs, err := c.Store.Refs(ctx) if err != nil { return err } - storedMetrics, err := c.Store.Metrics() + storedMetrics, err := c.Store.Metrics(ctx) if err != nil { return err } @@ -271,7 +282,7 @@ func (c *Controller) GarbageCollectMetrics(_ context.Context) error { metricLabelEnvironment, metricLabelEnvironmentExists := m.Labels["environment"] if !metricLabelProjectExists || (!metricLabelRefExists && !metricLabelEnvironmentExists) { - if err = c.Store.DelMetric(k); err != nil { + if err = c.Store.DelMetric(ctx, k); err != nil { return err } @@ -293,7 +304,7 @@ func (c *Controller) GarbageCollectMetrics(_ context.Context) error { // If the ref does not exist anymore, delete the metric if !refExists { - if err = c.Store.DelMetric(k); err != nil { + if err = c.Store.DelMetric(ctx, k); err != nil { return err } @@ -302,6 +313,7 @@ func (c *Controller) GarbageCollectMetrics(_ context.Context) error { "metric-labels": m.Labels, "reason": "non-existent-ref", }).Info("deleted metric from the store") + continue } @@ -313,9 +325,8 @@ func (c *Controller) GarbageCollectMetrics(_ context.Context) error { schemas.MetricKindJobRunCount, schemas.MetricKindJobStatus, schemas.MetricKindJobTimestamp: - if !ref.Project.Pull.Pipeline.Jobs.Enabled { - if err = c.Store.DelMetric(k); err != nil { + if err = c.Store.DelMetric(ctx, k); err != nil { return err } @@ -324,6 +335,7 @@ func (c *Controller) GarbageCollectMetrics(_ context.Context) error { "metric-labels": m.Labels, "reason": "jobs-metrics-disabled-on-ref", }).Info("deleted metric from the store") + continue } @@ -334,9 +346,8 @@ func (c *Controller) GarbageCollectMetrics(_ context.Context) error { switch m.Kind { case schemas.MetricKindJobStatus, schemas.MetricKindStatus: - if ref.Project.OutputSparseStatusMetrics && m.Value != 1 { - if err = c.Store.DelMetric(k); err != nil { + if err = c.Store.DelMetric(ctx, k); err != nil { return err } @@ -345,9 +356,9 @@ func (c *Controller) GarbageCollectMetrics(_ context.Context) error { "metric-labels": m.Labels, "reason": "output-sparse-metrics-enabled-on-ref", }).Info("deleted metric from the store") + continue } - default: } } @@ -362,7 +373,7 @@ func (c *Controller) GarbageCollectMetrics(_ context.Context) error { // If the ref does not exist anymore, delete the metric if !envExists { - if err = c.Store.DelMetric(k); err != nil { + if err = c.Store.DelMetric(ctx, k); err != nil { return err } @@ -371,6 +382,7 @@ func (c *Controller) GarbageCollectMetrics(_ context.Context) error { "metric-labels": m.Labels, "reason": "non-existent-environment", }).Info("deleted metric from the store") + continue } @@ -378,7 +390,7 @@ func (c *Controller) GarbageCollectMetrics(_ context.Context) error { switch m.Kind { case schemas.MetricKindEnvironmentDeploymentStatus: if env.OutputSparseStatusMetrics && m.Value != 1 { - if err = c.Store.DelMetric(k); err != nil { + if err = c.Store.DelMetric(ctx, k); err != nil { return err } @@ -387,6 +399,7 @@ func (c *Controller) GarbageCollectMetrics(_ context.Context) error { "metric-labels": m.Labels, "reason": "output-sparse-metrics-enabled-on-environment", }).Info("deleted metric from the store") + continue } } @@ -396,8 +409,8 @@ func (c *Controller) GarbageCollectMetrics(_ context.Context) error { return nil } -func deleteEnv(s store.Store, env schemas.Environment, reason string) (err error) { - if err = s.DelEnvironment(env.Key()); err != nil { +func deleteEnv(ctx context.Context, s store.Store, env schemas.Environment, reason string) (err error) { + if err = s.DelEnvironment(ctx, env.Key()); err != nil { return } @@ -410,8 +423,8 @@ func deleteEnv(s store.Store, env schemas.Environment, reason string) (err error return } -func deleteRef(s store.Store, ref schemas.Ref, reason string) (err error) { - if err = s.DelRef(ref.Key()); err != nil { +func deleteRef(ctx context.Context, s store.Store, ref schemas.Ref, reason string) (err error) { + if err = s.DelRef(ctx, ref.Key()); err != nil { return } diff --git a/pkg/controller/garbage_collector_test.go b/pkg/controller/garbage_collector_test.go index 11808f9c..e44e47c7 100644 --- a/pkg/controller/garbage_collector_test.go +++ b/pkg/controller/garbage_collector_test.go @@ -18,7 +18,7 @@ func TestGarbageCollectProjects(t *testing.T) { p3 := schemas.NewProject("wc/p3") p4 := schemas.NewProject("wc/p4") - c, mux, srv := newTestController(config.Config{ + ctx, c, mux, srv := newTestController(config.Config{ Projects: []config.Project{p1.Project}, Wildcards: config.Wildcards{ config.Wildcard{ @@ -36,13 +36,13 @@ func TestGarbageCollectProjects(t *testing.T) { fmt.Fprint(w, `[{"id":1, "path_with_namespace": "wc/p3", "jobs_enabled": true}]`) }) - c.Store.SetProject(p1) - c.Store.SetProject(p2) - c.Store.SetProject(p3) - c.Store.SetProject(p4) + c.Store.SetProject(ctx, p1) + c.Store.SetProject(ctx, p2) + c.Store.SetProject(ctx, p3) + c.Store.SetProject(ctx, p4) assert.NoError(t, c.GarbageCollectProjects(context.Background())) - storedProjects, err := c.Store.Projects() + storedProjects, err := c.Store.Projects(ctx) assert.NoError(t, err) expectedProjects := schemas.Projects{ @@ -53,7 +53,7 @@ func TestGarbageCollectProjects(t *testing.T) { } func TestGarbageCollectEnvironments(t *testing.T) { - c, mux, srv := newTestController(config.Config{}) + ctx, c, mux, srv := newTestController(config.Config{}) defer srv.Close() mux.HandleFunc("/api/v4/projects/p2/environments", @@ -69,13 +69,13 @@ func TestGarbageCollectEnvironments(t *testing.T) { envp2dev := schemas.Environment{ProjectName: "p2", Name: "dev"} envp2main := schemas.Environment{ProjectName: "p2", Name: "main"} - c.Store.SetProject(p2) - c.Store.SetEnvironment(envp1main) - c.Store.SetEnvironment(envp2dev) - c.Store.SetEnvironment(envp2main) + c.Store.SetProject(ctx, p2) + c.Store.SetEnvironment(ctx, envp1main) + c.Store.SetEnvironment(ctx, envp2dev) + c.Store.SetEnvironment(ctx, envp2main) assert.NoError(t, c.GarbageCollectEnvironments(context.Background())) - storedEnvironments, err := c.Store.Environments() + storedEnvironments, err := c.Store.Environments(ctx) assert.NoError(t, err) expectedEnvironments := schemas.Environments{ @@ -89,7 +89,7 @@ func TestGarbageCollectEnvironments(t *testing.T) { } func TestGarbageCollectRefs(t *testing.T) { - c, mux, srv := newTestController(config.Config{}) + ctx, c, mux, srv := newTestController(config.Config{}) defer srv.Close() mux.HandleFunc("/api/v4/projects/p2/repository/branches", @@ -111,14 +111,14 @@ func TestGarbageCollectRefs(t *testing.T) { pr2dev := schemas.NewRef(p2, schemas.RefKindBranch, "dev") pr2main := schemas.NewRef(p2, schemas.RefKindBranch, "main") - c.Store.SetProject(p2) - c.Store.SetRef(pr1dev) - c.Store.SetRef(pr1main) - c.Store.SetRef(pr2dev) - c.Store.SetRef(pr2main) + c.Store.SetProject(ctx, p2) + c.Store.SetRef(ctx, pr1dev) + c.Store.SetRef(ctx, pr1main) + c.Store.SetRef(ctx, pr2dev) + c.Store.SetRef(ctx, pr2main) assert.NoError(t, c.GarbageCollectRefs(context.Background())) - storedRefs, err := c.Store.Refs() + storedRefs, err := c.Store.Refs(ctx) assert.NoError(t, err) newPR2main := schemas.NewRef(p2, schemas.RefKindBranch, "main") @@ -129,7 +129,7 @@ func TestGarbageCollectRefs(t *testing.T) { } func TestGarbageCollectMetrics(t *testing.T) { - c, _, srv := newTestController(config.Config{}) + ctx, c, _, srv := newTestController(config.Config{}) srv.Close() p1 := schemas.NewProject("p1") @@ -145,16 +145,16 @@ func TestGarbageCollectMetrics(t *testing.T) { ref3m1 := schemas.Metric{Kind: schemas.MetricKindCoverage, Labels: prometheus.Labels{"project": "foo", "kind": "branch"}} ref4m1 := schemas.Metric{Kind: schemas.MetricKindCoverage, Labels: prometheus.Labels{"ref": "bar", "kind": "branch"}} - c.Store.SetRef(ref1) - c.Store.SetMetric(ref1m1) - c.Store.SetMetric(ref1m2) - c.Store.SetMetric(ref1m3) - c.Store.SetMetric(ref2m1) - c.Store.SetMetric(ref3m1) - c.Store.SetMetric(ref4m1) + c.Store.SetRef(ctx, ref1) + c.Store.SetMetric(ctx, ref1m1) + c.Store.SetMetric(ctx, ref1m2) + c.Store.SetMetric(ctx, ref1m3) + c.Store.SetMetric(ctx, ref2m1) + c.Store.SetMetric(ctx, ref3m1) + c.Store.SetMetric(ctx, ref4m1) assert.NoError(t, c.GarbageCollectMetrics(context.Background())) - storedMetrics, err := c.Store.Metrics() + storedMetrics, err := c.Store.Metrics(ctx) assert.NoError(t, err) expectedMetrics := schemas.Metrics{ diff --git a/pkg/controller/handlers.go b/pkg/controller/handlers.go index 948c84a6..e08e7228 100644 --- a/pkg/controller/handlers.go +++ b/pkg/controller/handlers.go @@ -1,6 +1,7 @@ package controller import ( + "context" "fmt" "io/ioutil" "net/http" @@ -13,25 +14,29 @@ import ( ) // HealthCheckHandler .. -func (c *Controller) HealthCheckHandler() (h healthcheck.Handler) { +func (c *Controller) HealthCheckHandler(ctx context.Context) (h healthcheck.Handler) { h = healthcheck.NewHandler() if c.Config.Gitlab.EnableHealthCheck { - h.AddReadinessCheck("gitlab-reachable", c.Gitlab.ReadinessCheck()) + h.AddReadinessCheck("gitlab-reachable", c.Gitlab.ReadinessCheck(ctx)) } else { log.Warn("GitLab health check has been disabled. Readiness checks won't be operated.") } + return } // MetricsHandler .. func (c *Controller) MetricsHandler(w http.ResponseWriter, r *http.Request) { + ctx := context.TODO() registry := NewRegistry() - metrics, err := c.Store.Metrics() + + metrics, err := c.Store.Metrics(ctx) if err != nil { log.Error(err.Error()) } if err := registry.ExportInternalMetrics( + ctx, c.Gitlab, c.Store, ); err != nil { @@ -48,22 +53,26 @@ func (c *Controller) MetricsHandler(w http.ResponseWriter, r *http.Request) { // WebhookHandler .. func (c *Controller) WebhookHandler(w http.ResponseWriter, r *http.Request) { + ctx := context.TODO() logFields := log.Fields{ "ip-address": r.RemoteAddr, "user-agent": r.UserAgent(), } + log.WithFields(logFields).Debug("webhook request") if r.Header.Get("X-Gitlab-Token") != c.Config.Server.Webhook.SecretToken { log.WithFields(logFields).Debug("invalid token provided for a webhook request") w.WriteHeader(http.StatusForbidden) fmt.Fprint(w, "{\"error\": \"invalid token\"}") + return } if r.Body == http.NoBody { log.WithFields(logFields).WithField("error", "nil body").Warn("unable to read body of a received webhook") w.WriteHeader(http.StatusBadRequest) + return } @@ -71,6 +80,7 @@ func (c *Controller) WebhookHandler(w http.ResponseWriter, r *http.Request) { if err != nil { log.WithFields(logFields).WithField("error", err.Error()).Warn("unable to read body of a received webhook") w.WriteHeader(http.StatusBadRequest) + return } @@ -78,14 +88,15 @@ func (c *Controller) WebhookHandler(w http.ResponseWriter, r *http.Request) { if err != nil { log.WithFields(logFields).WithFields(logFields).WithField("error", err.Error()).Warn("unable to parse body of a received webhook") w.WriteHeader(http.StatusBadRequest) + return } switch event := event.(type) { case *gitlab.PipelineEvent: - go c.processPipelineEvent(*event) + go c.processPipelineEvent(ctx, *event) case *gitlab.DeploymentEvent: - go c.processDeploymentEvent(*event) + go c.processDeploymentEvent(ctx, *event) default: log.WithFields(logFields).WithField("event-type", reflect.TypeOf(event).String()).Warn("received a non supported event type as a webhook") w.WriteHeader(http.StatusUnprocessableEntity) diff --git a/pkg/controller/handlers_test.go b/pkg/controller/handlers_test.go index 36fab279..3bf4eecc 100644 --- a/pkg/controller/handlers_test.go +++ b/pkg/controller/handlers_test.go @@ -12,7 +12,7 @@ import ( ) func TestWebhookHandler(t *testing.T) { - c, _, srv := newTestController(config.Config{ + _, c, _, srv := newTestController(config.Config{ Server: config.Server{ Webhook: config.ServerWebhook{ Enabled: true, diff --git a/pkg/controller/jobs.go b/pkg/controller/jobs.go index 12b07559..979183e8 100644 --- a/pkg/controller/jobs.go +++ b/pkg/controller/jobs.go @@ -1,6 +1,7 @@ package controller import ( + "context" "reflect" "regexp" @@ -9,39 +10,39 @@ import ( ) // PullRefPipelineJobsMetrics .. -func (c *Controller) PullRefPipelineJobsMetrics(ref schemas.Ref) error { - jobs, err := c.Gitlab.ListRefPipelineJobs(ref) +func (c *Controller) PullRefPipelineJobsMetrics(ctx context.Context, ref schemas.Ref) error { + jobs, err := c.Gitlab.ListRefPipelineJobs(ctx, ref) if err != nil { return err } for _, job := range jobs { - c.ProcessJobMetrics(ref, job) + c.ProcessJobMetrics(ctx, ref, job) } return nil } // PullRefMostRecentJobsMetrics .. -func (c *Controller) PullRefMostRecentJobsMetrics(ref schemas.Ref) error { +func (c *Controller) PullRefMostRecentJobsMetrics(ctx context.Context, ref schemas.Ref) error { if !ref.Project.Pull.Pipeline.Jobs.Enabled { return nil } - jobs, err := c.Gitlab.ListRefMostRecentJobs(ref) + jobs, err := c.Gitlab.ListRefMostRecentJobs(ctx, ref) if err != nil { return err } for _, job := range jobs { - c.ProcessJobMetrics(ref, job) + c.ProcessJobMetrics(ctx, ref, job) } return nil } // ProcessJobMetrics .. -func (c *Controller) ProcessJobMetrics(ref schemas.Ref, job schemas.Job) { +func (c *Controller) ProcessJobMetrics(ctx context.Context, ref schemas.Ref, job schemas.Job) { projectRefLogFields := log.Fields{ "project-name": ref.Project.Name, "job-name": job.Name, @@ -69,8 +70,9 @@ func (c *Controller) ProcessJobMetrics(ref schemas.Ref, job schemas.Job) { } // Refresh ref state from the store - if err := c.Store.GetRef(&ref); err != nil { + if err := c.Store.GetRef(ctx, &ref); err != nil { log.WithFields(projectRefLogFields).WithField("error", err.Error()).Error("getting ref from the store") + return } @@ -85,35 +87,38 @@ func (c *Controller) ProcessJobMetrics(ref schemas.Ref, job schemas.Job) { if ref.LatestJobs == nil { ref.LatestJobs = make(schemas.Jobs) } + ref.LatestJobs[job.Name] = job - if err := c.Store.SetRef(ref); err != nil { + + if err := c.Store.SetRef(ctx, ref); err != nil { log.WithFields( projectRefLogFields, ).WithField("error", err.Error()).Error("writing ref in the store") + return } log.WithFields(projectRefLogFields).Trace("processing job metrics") - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindJobID, Labels: labels, Value: float64(job.ID), }) - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindJobTimestamp, Labels: labels, Value: job.Timestamp, }) - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindJobDurationSeconds, Labels: labels, Value: job.DurationSeconds, }) - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindJobQueuedDurationSeconds, Labels: labels, Value: job.QueuedDurationSeconds, @@ -127,11 +132,12 @@ func (c *Controller) ProcessJobMetrics(ref schemas.Ref, job schemas.Job) { // If the metric does not exist yet, start with 0 instead of 1 // this could cause some false positives in prometheus // when restarting the exporter otherwise - jobRunCountExists, err := c.Store.MetricExists(jobRunCount.Key()) + jobRunCountExists, err := c.Store.MetricExists(ctx, jobRunCount.Key()) if err != nil { log.WithFields( projectRefLogFields, ).WithField("error", err.Error()).Error("checking if metric exists in the store") + return } @@ -141,20 +147,22 @@ func (c *Controller) ProcessJobMetrics(ref schemas.Ref, job schemas.Job) { jobTriggeredRegexp := regexp.MustCompile("^(skipped|manual|scheduled)$") lastJobTriggered := !jobTriggeredRegexp.MatchString(lastJob.Status) jobTriggered := !jobTriggeredRegexp.MatchString(job.Status) + if jobRunCountExists && ((lastJob.ID != job.ID && jobTriggered) || (lastJob.ID == job.ID && jobTriggered && !lastJobTriggered)) { - storeGetMetric(c.Store, &jobRunCount) + storeGetMetric(ctx, c.Store, &jobRunCount) jobRunCount.Value++ } - storeSetMetric(c.Store, jobRunCount) + storeSetMetric(ctx, c.Store, jobRunCount) - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindJobArtifactSizeBytes, Labels: labels, Value: job.ArtifactSize, }) emitStatusMetric( + ctx, c.Store, schemas.MetricKindJobStatus, labels, diff --git a/pkg/controller/jobs_test.go b/pkg/controller/jobs_test.go index a80f91d1..c5da34f9 100644 --- a/pkg/controller/jobs_test.go +++ b/pkg/controller/jobs_test.go @@ -11,7 +11,7 @@ import ( ) func TestPullRefPipelineJobsMetrics(t *testing.T) { - c, mux, srv := newTestController(config.Config{}) + ctx, c, mux, srv := newTestController(config.Config{}) defer srv.Close() mux.HandleFunc("/api/v4/projects/foo/pipelines/1/jobs", @@ -25,15 +25,14 @@ func TestPullRefPipelineJobsMetrics(t *testing.T) { ref := schemas.NewRef(p, schemas.RefKindBranch, "bar") ref.LatestPipeline.ID = 1 - assert.NoError(t, c.PullRefPipelineJobsMetrics(ref)) - srv.Close() - assert.Error(t, c.PullRefPipelineJobsMetrics(ref)) - // TODO: assert the results? + assert.NoError(t, c.PullRefPipelineJobsMetrics(ctx, ref)) + srv.Close() + assert.Error(t, c.PullRefPipelineJobsMetrics(ctx, ref)) } func TestPullRefMostRecentJobsMetrics(t *testing.T) { - c, mux, srv := newTestController(config.Config{}) + ctx, c, mux, srv := newTestController(config.Config{}) defer srv.Close() mux.HandleFunc("/api/v4/projects/foo/jobs", @@ -52,17 +51,17 @@ func TestPullRefMostRecentJobsMetrics(t *testing.T) { } // Test with FetchPipelineJobMetrics disabled - assert.NoError(t, c.PullRefMostRecentJobsMetrics(ref)) + assert.NoError(t, c.PullRefMostRecentJobsMetrics(ctx, ref)) // Enable FetchPipelineJobMetrics ref.Project.Pull.Pipeline.Jobs.Enabled = true - assert.NoError(t, c.PullRefMostRecentJobsMetrics(ref)) + assert.NoError(t, c.PullRefMostRecentJobsMetrics(ctx, ref)) srv.Close() - assert.Error(t, c.PullRefMostRecentJobsMetrics(ref)) + assert.Error(t, c.PullRefMostRecentJobsMetrics(ctx, ref)) } func TestProcessJobMetrics(t *testing.T) { - c, _, srv := newTestController(config.Config{}) + ctx, c, _, srv := newTestController(config.Config{}) srv.Close() oldJob := schemas.Job{ @@ -95,24 +94,24 @@ func TestProcessJobMetrics(t *testing.T) { "foo": oldJob, } - c.Store.SetRef(ref) + c.Store.SetRef(ctx, ref) // If we run it against the same job, nothing should change in the store - c.ProcessJobMetrics(ref, oldJob) - refs, _ := c.Store.Refs() + c.ProcessJobMetrics(ctx, ref, oldJob) + refs, _ := c.Store.Refs(ctx) assert.Equal(t, schemas.Jobs{ "foo": oldJob, }, refs[ref.Key()].LatestJobs) // Update the ref - c.ProcessJobMetrics(ref, newJob) - refs, _ = c.Store.Refs() + c.ProcessJobMetrics(ctx, ref, newJob) + refs, _ = c.Store.Refs(ctx) assert.Equal(t, schemas.Jobs{ "foo": newJob, }, refs[ref.Key()].LatestJobs) // Check if all the metrics exist - metrics, _ := c.Store.Metrics() + metrics, _ := c.Store.Metrics(ctx) labels := map[string]string{ "project": ref.Project.Name, "topics": ref.Project.Topics, diff --git a/pkg/controller/metrics.go b/pkg/controller/metrics.go index 4a9fd2e1..9596d8c7 100644 --- a/pkg/controller/metrics.go +++ b/pkg/controller/metrics.go @@ -1,6 +1,7 @@ package controller import ( + "context" "fmt" "reflect" @@ -11,7 +12,7 @@ import ( log "github.com/sirupsen/logrus" ) -// Registry wraps a pointer of prometheus.Registry +// Registry wraps a pointer of prometheus.Registry. type Registry struct { *prometheus.Registry @@ -33,7 +34,7 @@ type Registry struct { // RegistryCollectors .. type RegistryCollectors map[schemas.MetricKind]prometheus.Collector -// NewRegistry initialize a new registry +// NewRegistry initialize a new registry. func NewRegistry() *Registry { r := &Registry{ Registry: prometheus.NewRegistry(), @@ -72,7 +73,7 @@ func NewRegistry() *Registry { return r } -// RegisterInternalCollectors declare our internal collectors to the registry +// RegisterInternalCollectors declare our internal collectors to the registry. func (r *Registry) RegisterInternalCollectors() { r.InternalCollectors.CurrentlyQueuedTasksCount = NewInternalCollectorCurrentlyQueuedTasksCount() r.InternalCollectors.EnvironmentsCount = NewInternalCollectorEnvironmentsCount() @@ -97,6 +98,7 @@ func (r *Registry) RegisterInternalCollectors() { // ExportInternalMetrics .. func (r *Registry) ExportInternalMetrics( + ctx context.Context, g *gitlab.Client, s store.Store, ) (err error) { @@ -109,32 +111,32 @@ func (r *Registry) ExportInternalMetrics( refsCount int64 ) - currentlyQueuedTasks, err = s.CurrentlyQueuedTasksCount() + currentlyQueuedTasks, err = s.CurrentlyQueuedTasksCount(ctx) if err != nil { return } - executedTasksCount, err = s.ExecutedTasksCount() + executedTasksCount, err = s.ExecutedTasksCount(ctx) if err != nil { return } - projectsCount, err = s.ProjectsCount() + projectsCount, err = s.ProjectsCount(ctx) if err != nil { return } - environmentsCount, err = s.EnvironmentsCount() + environmentsCount, err = s.EnvironmentsCount(ctx) if err != nil { return } - refsCount, err = s.RefsCount() + refsCount, err = s.RefsCount(ctx) if err != nil { return } - metricsCount, err = s.MetricsCount() + metricsCount, err = s.MetricsCount(ctx) if err != nil { return } @@ -148,16 +150,18 @@ func (r *Registry) ExportInternalMetrics( r.InternalCollectors.MetricsCount.(*prometheus.GaugeVec).With(prometheus.Labels{}).Set(float64(metricsCount)) r.InternalCollectors.ProjectsCount.(*prometheus.GaugeVec).With(prometheus.Labels{}).Set(float64(projectsCount)) r.InternalCollectors.RefsCount.(*prometheus.GaugeVec).With(prometheus.Labels{}).Set(float64(refsCount)) + return } -// RegisterCollectors add all our metrics to the registry +// RegisterCollectors add all our metrics to the registry. func (r *Registry) RegisterCollectors() error { for _, c := range r.Collectors { if err := r.Register(c); err != nil { return fmt.Errorf("could not add provided collector '%v' to the Prometheus registry: %v", c, err) } } + return nil } @@ -180,16 +184,20 @@ func (r *Registry) ExportMetrics(metrics schemas.Metrics) { } } -func emitStatusMetric(s store.Store, metricKind schemas.MetricKind, labelValues map[string]string, statuses []string, status string, sparseMetrics bool) { +func emitStatusMetric(ctx context.Context, s store.Store, metricKind schemas.MetricKind, labelValues map[string]string, statuses []string, status string, sparseMetrics bool) { // Moved into separate function to reduce cyclomatic complexity // List of available statuses from the API spec // ref: https://docs.gitlab.com/ee/api/jobs.html#list-pipeline-jobs for _, currentStatus := range statuses { - var value float64 - statusLabels := make(map[string]string) + var ( + value float64 + statusLabels = make(map[string]string) + ) + for k, v := range labelValues { statusLabels[k] = v } + statusLabels["status"] = currentStatus statusMetric := schemas.Metric{ @@ -202,12 +210,13 @@ func emitStatusMetric(s store.Store, metricKind schemas.MetricKind, labelValues statusMetric.Value = 1 } else { if sparseMetrics { - storeDelMetric(s, statusMetric) + storeDelMetric(ctx, s, statusMetric) + continue } statusMetric.Value = 0 } - storeSetMetric(s, statusMetric) + storeSetMetric(ctx, s, statusMetric) } } diff --git a/pkg/controller/metrics_test.go b/pkg/controller/metrics_test.go index 13ab76a3..754d9c61 100644 --- a/pkg/controller/metrics_test.go +++ b/pkg/controller/metrics_test.go @@ -17,16 +17,17 @@ func TestNewRegistry(t *testing.T) { assert.NotNil(t, r.Collectors) } -// introduce a test to check the /metrics endpoint body +// introduce a test to check the /metrics endpoint body. func TestMetricsHandler(t *testing.T) { - c, _, srv := newTestController(config.Config{}) + _, c, _, srv := newTestController(config.Config{}) srv.Close() w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/", nil) c.MetricsHandler(w, r) - assert.Equal(t, http.StatusOK, w.Result().StatusCode) + // TODO: Find a way to see if expected metrics are present + assert.Equal(t, http.StatusOK, w.Result().StatusCode) } func TestRegistryGetCollector(t *testing.T) { diff --git a/pkg/controller/pipelines.go b/pkg/controller/pipelines.go index 4b7be151..750b0ee5 100644 --- a/pkg/controller/pipelines.go +++ b/pkg/controller/pipelines.go @@ -1,6 +1,7 @@ package controller import ( + "context" "fmt" "reflect" @@ -10,10 +11,10 @@ import ( ) // PullRefMetrics .. -func (c *Controller) PullRefMetrics(ref schemas.Ref) error { +func (c *Controller) PullRefMetrics(ctx context.Context, ref schemas.Ref) error { // At scale, the scheduled ref may be behind the actual state being stored // to avoid issues, we refresh it from the store before manipulating it - if err := c.Store.GetRef(&ref); err != nil { + if err := c.Store.GetRef(ctx, &ref); err != nil { return err } @@ -31,7 +32,7 @@ func (c *Controller) PullRefMetrics(ref schemas.Ref) error { refName = ref.Name } - pipelines, _, err := c.Gitlab.GetProjectPipelines(ref.Project.Name, &goGitlab.ListProjectPipelinesOptions{ + pipelines, _, err := c.Gitlab.GetProjectPipelines(ctx, ref.Project.Name, &goGitlab.ListProjectPipelinesOptions{ // We only need the most recent pipeline ListOptions: goGitlab.ListOptions{ PerPage: 1, @@ -45,10 +46,11 @@ func (c *Controller) PullRefMetrics(ref schemas.Ref) error { if len(pipelines) == 0 { log.WithFields(logFields).Debug("could not find any pipeline for the ref") + return nil } - pipeline, err := c.Gitlab.GetRefPipeline(ref, pipelines[0].ID) + pipeline, err := c.Gitlab.GetRefPipeline(ctx, ref, pipelines[0].ID) if err != nil { return err } @@ -59,14 +61,14 @@ func (c *Controller) PullRefMetrics(ref schemas.Ref) error { // fetch pipeline variables if ref.Project.Pull.Pipeline.Variables.Enabled { - ref.LatestPipeline.Variables, err = c.Gitlab.GetRefPipelineVariablesAsConcatenatedString(ref) + ref.LatestPipeline.Variables, err = c.Gitlab.GetRefPipelineVariablesAsConcatenatedString(ctx, ref) if err != nil { return err } } // Update the ref in the store - if err = c.Store.SetRef(ref); err != nil { + if err = c.Store.SetRef(ctx, ref); err != nil { return err } @@ -77,25 +79,29 @@ func (c *Controller) PullRefMetrics(ref schemas.Ref) error { Kind: schemas.MetricKindRunCount, Labels: ref.DefaultLabelsValues(), } - storeGetMetric(c.Store, &runCount) + + storeGetMetric(ctx, c.Store, &runCount) + if formerPipeline.ID != 0 && formerPipeline.ID != ref.LatestPipeline.ID { runCount.Value++ } - storeSetMetric(c.Store, runCount) - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, runCount) + + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindCoverage, Labels: ref.DefaultLabelsValues(), Value: pipeline.Coverage, }) - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindID, Labels: ref.DefaultLabelsValues(), Value: float64(pipeline.ID), }) emitStatusMetric( + ctx, c.Store, schemas.MetricKindStatus, ref.DefaultLabelsValues(), @@ -104,34 +110,35 @@ func (c *Controller) PullRefMetrics(ref schemas.Ref) error { ref.Project.OutputSparseStatusMetrics, ) - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindDurationSeconds, Labels: ref.DefaultLabelsValues(), Value: pipeline.DurationSeconds, }) - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindQueuedDurationSeconds, Labels: ref.DefaultLabelsValues(), Value: pipeline.QueuedDurationSeconds, }) - storeSetMetric(c.Store, schemas.Metric{ + storeSetMetric(ctx, c.Store, schemas.Metric{ Kind: schemas.MetricKindTimestamp, Labels: ref.DefaultLabelsValues(), Value: pipeline.Timestamp, }) if ref.Project.Pull.Pipeline.Jobs.Enabled { - if err := c.PullRefPipelineJobsMetrics(ref); err != nil { + if err := c.PullRefPipelineJobsMetrics(ctx, ref); err != nil { return err } } + return nil } if ref.Project.Pull.Pipeline.Jobs.Enabled { - if err := c.PullRefMostRecentJobsMetrics(ref); err != nil { + if err := c.PullRefMostRecentJobsMetrics(ctx, ref); err != nil { return err } } diff --git a/pkg/controller/pipelines_test.go b/pkg/controller/pipelines_test.go index 64b0e8e4..a6012c62 100644 --- a/pkg/controller/pipelines_test.go +++ b/pkg/controller/pipelines_test.go @@ -11,7 +11,7 @@ import ( ) func TestPullRefMetricsSucceed(t *testing.T) { - c, mux, srv := newTestController(config.Config{}) + ctx, c, mux, srv := newTestController(config.Config{}) defer srv.Close() mux.HandleFunc("/api/v4/projects/foo/pipelines", @@ -36,14 +36,16 @@ func TestPullRefMetricsSucceed(t *testing.T) { p := schemas.NewProject("foo") p.Pull.Pipeline.Variables.Enabled = true - assert.NoError(t, c.PullRefMetrics(schemas.NewRef( - p, - schemas.RefKindBranch, - "bar", - ))) + assert.NoError(t, c.PullRefMetrics( + ctx, + schemas.NewRef( + p, + schemas.RefKindBranch, + "bar", + ))) // Check if all the metrics exist - metrics, _ := c.Store.Metrics() + metrics, _ := c.Store.Metrics(ctx) labels := map[string]string{ "kind": string(schemas.RefKindBranch), "project": "foo", @@ -90,7 +92,7 @@ func TestPullRefMetricsSucceed(t *testing.T) { } func TestPullRefMetricsMergeRequestPipeline(t *testing.T) { - c, mux, srv := newTestController(config.Config{}) + ctx, c, mux, srv := newTestController(config.Config{}) defer srv.Close() mux.HandleFunc("/api/v4/projects/foo/pipelines", @@ -114,9 +116,11 @@ func TestPullRefMetricsMergeRequestPipeline(t *testing.T) { p := schemas.NewProject("foo") p.Pull.Pipeline.Variables.Enabled = true - assert.NoError(t, c.PullRefMetrics(schemas.NewRef( - p, - schemas.RefKindMergeRequest, - "1234", - ))) + assert.NoError(t, c.PullRefMetrics( + ctx, + schemas.NewRef( + p, + schemas.RefKindMergeRequest, + "1234", + ))) } diff --git a/pkg/controller/projects.go b/pkg/controller/projects.go index c35cb116..8203a96a 100644 --- a/pkg/controller/projects.go +++ b/pkg/controller/projects.go @@ -10,13 +10,13 @@ import ( // PullProjectsFromWildcard .. func (c *Controller) PullProjectsFromWildcard(ctx context.Context, w config.Wildcard) error { - foundProjects, err := c.Gitlab.ListProjects(w) + foundProjects, err := c.Gitlab.ListProjects(ctx, w) if err != nil { return err } for _, p := range foundProjects { - projectExists, err := c.Store.ProjectExists(p.Key()) + projectExists, err := c.Store.ProjectExists(ctx, p.Key()) if err != nil { return err } @@ -31,7 +31,7 @@ func (c *Controller) PullProjectsFromWildcard(ctx context.Context, w config.Wild "project-name": p.Name, }).Info("discovered new project") - if err := c.Store.SetProject(p); err != nil { + if err := c.Store.SetProject(ctx, p); err != nil { log.Errorf(err.Error()) } diff --git a/pkg/controller/projects_test.go b/pkg/controller/projects_test.go index 34918a5a..003e4fba 100644 --- a/pkg/controller/projects_test.go +++ b/pkg/controller/projects_test.go @@ -1,7 +1,6 @@ package controller import ( - "context" "fmt" "net/http" "testing" @@ -12,7 +11,7 @@ import ( ) func TestPullProjectsFromWildcard(t *testing.T) { - c, mux, srv := newTestController(config.Config{}) + ctx, c, mux, srv := newTestController(config.Config{}) defer srv.Close() mux.HandleFunc("/api/v4/projects", @@ -21,9 +20,9 @@ func TestPullProjectsFromWildcard(t *testing.T) { }) w := config.NewWildcard() - assert.NoError(t, c.PullProjectsFromWildcard(context.Background(), w)) + assert.NoError(t, c.PullProjectsFromWildcard(ctx, w)) - projects, _ := c.Store.Projects() + projects, _ := c.Store.Projects(ctx) p1 := schemas.NewProject("bar") expectedProjects := schemas.Projects{ diff --git a/pkg/controller/refs.go b/pkg/controller/refs.go index 993467ad..54652d80 100644 --- a/pkg/controller/refs.go +++ b/pkg/controller/refs.go @@ -9,25 +9,25 @@ import ( ) // GetRefs .. -func (c *Controller) GetRefs(p schemas.Project) ( +func (c *Controller) GetRefs(ctx context.Context, p schemas.Project) ( refs schemas.Refs, err error, ) { - refs = make(schemas.Refs) var pulledRefs schemas.Refs + refs = make(schemas.Refs) + if p.Pull.Refs.Branches.Enabled { // If one of these parameter is set, we will need to fetch the branches from the // pipelines API instead of the branches one if !p.Pull.Refs.Branches.ExcludeDeleted || p.Pull.Refs.Branches.MostRecent > 0 || p.Pull.Refs.Branches.MaxAgeSeconds > 0 { - - if pulledRefs, err = c.Gitlab.GetRefsFromPipelines(p, schemas.RefKindBranch); err != nil { + if pulledRefs, err = c.Gitlab.GetRefsFromPipelines(ctx, p, schemas.RefKindBranch); err != nil { return } } else { - if pulledRefs, err = c.Gitlab.GetProjectBranches(p); err != nil { + if pulledRefs, err = c.Gitlab.GetProjectBranches(ctx, p); err != nil { return } } @@ -43,12 +43,11 @@ func (c *Controller) GetRefs(p schemas.Project) ( if !p.Pull.Refs.Tags.ExcludeDeleted || p.Pull.Refs.Tags.MostRecent > 0 || p.Pull.Refs.Tags.MaxAgeSeconds > 0 { - - if pulledRefs, err = c.Gitlab.GetRefsFromPipelines(p, schemas.RefKindTag); err != nil { + if pulledRefs, err = c.Gitlab.GetRefsFromPipelines(ctx, p, schemas.RefKindTag); err != nil { return } } else { - if pulledRefs, err = c.Gitlab.GetProjectTags(p); err != nil { + if pulledRefs, err = c.Gitlab.GetProjectTags(ctx, p); err != nil { return } } @@ -60,6 +59,7 @@ func (c *Controller) GetRefs(p schemas.Project) ( if p.Pull.Refs.MergeRequests.Enabled { if pulledRefs, err = c.Gitlab.GetRefsFromPipelines( + ctx, p, schemas.RefKindMergeRequest, ); err != nil { @@ -76,13 +76,13 @@ func (c *Controller) GetRefs(p schemas.Project) ( // PullRefsFromProject .. func (c *Controller) PullRefsFromProject(ctx context.Context, p schemas.Project) error { - refs, err := c.GetRefs(p) + refs, err := c.GetRefs(ctx, p) if err != nil { return err } for _, ref := range refs { - refExists, err := c.Store.RefExists(ref.Key()) + refExists, err := c.Store.RefExists(ctx, ref.Key()) if err != nil { return err } @@ -94,12 +94,13 @@ func (c *Controller) PullRefsFromProject(ctx context.Context, p schemas.Project) "ref-kind": ref.Kind, }).Info("discovered new ref") - if err = c.Store.SetRef(ref); err != nil { + if err = c.Store.SetRef(ctx, ref); err != nil { return err } c.ScheduleTask(ctx, schemas.TaskTypePullRefMetrics, string(ref.Key()), ref) } } + return nil } diff --git a/pkg/controller/refs_test.go b/pkg/controller/refs_test.go index a16d9ec9..067a000e 100644 --- a/pkg/controller/refs_test.go +++ b/pkg/controller/refs_test.go @@ -1,7 +1,6 @@ package controller import ( - "context" "fmt" "net/http" "testing" @@ -12,7 +11,7 @@ import ( ) func TestGetRefs(t *testing.T) { - c, mux, srv := newTestController(config.Config{}) + ctx, c, mux, srv := newTestController(config.Config{}) defer srv.Close() mux.HandleFunc("/api/v4/projects/foo/repository/branches", @@ -35,7 +34,7 @@ func TestGetRefs(t *testing.T) { p.Pull.Refs.Tags.Regexp = `^v` p.Pull.Refs.MergeRequests.Enabled = true - foundRefs, err := c.GetRefs(p) + foundRefs, err := c.GetRefs(ctx, p) assert.NoError(t, err) ref1 := schemas.NewRef(p, schemas.RefKindBranch, "main") @@ -50,7 +49,7 @@ func TestGetRefs(t *testing.T) { } func TestPullRefsFromProject(t *testing.T) { - c, mux, srv := newTestController(config.Config{}) + ctx, c, mux, srv := newTestController(config.Config{}) defer srv.Close() mux.HandleFunc("/api/v4/projects/foo", @@ -69,13 +68,13 @@ func TestPullRefsFromProject(t *testing.T) { }) p1 := schemas.NewProject("foo") - assert.NoError(t, c.PullRefsFromProject(context.Background(), p1)) + assert.NoError(t, c.PullRefsFromProject(ctx, p1)) ref1 := schemas.NewRef(p1, schemas.RefKindBranch, "main") expectedRefs := schemas.Refs{ ref1.Key(): ref1, } - projectsRefs, _ := c.Store.Refs() + projectsRefs, _ := c.Store.Refs(ctx) assert.Equal(t, expectedRefs, projectsRefs) } diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index ea7a559d..36524ea0 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -18,7 +18,7 @@ import ( const bufferSize = 1000 -// TaskController holds task related clients +// TaskController holds task related clients. type TaskController struct { Factory taskq.Factory Queue taskq.Queue @@ -26,7 +26,7 @@ type TaskController struct { TaskSchedulingMonitoring map[schemas.TaskType]*monitor.TaskSchedulingStatus } -// NewTaskController initializes and returns a new TaskController object +// NewTaskController initializes and returns a new TaskController object. func NewTaskController(r *redis.Client) (t TaskController) { t.TaskMap = &taskq.TaskMap{} @@ -72,14 +72,14 @@ func NewTaskController(r *redis.Client) (t TaskController) { // TaskHandlerPullProjectsFromWildcard .. func (c *Controller) TaskHandlerPullProjectsFromWildcard(ctx context.Context, id string, w config.Wildcard) error { - defer c.unqueueTask(schemas.TaskTypePullProjectsFromWildcard, id) + defer c.unqueueTask(ctx, schemas.TaskTypePullProjectsFromWildcard, id) return c.PullProjectsFromWildcard(ctx, w) } // TaskHandlerPullEnvironmentsFromProject .. func (c *Controller) TaskHandlerPullEnvironmentsFromProject(ctx context.Context, p schemas.Project) { - defer c.unqueueTask(schemas.TaskTypePullEnvironmentsFromProject, string(p.Key())) + defer c.unqueueTask(ctx, schemas.TaskTypePullEnvironmentsFromProject, string(p.Key())) // On errors, we do not want to retry these tasks if p.Pull.Environments.Enabled { @@ -93,11 +93,11 @@ func (c *Controller) TaskHandlerPullEnvironmentsFromProject(ctx context.Context, } // TaskHandlerPullEnvironmentMetrics .. -func (c *Controller) TaskHandlerPullEnvironmentMetrics(env schemas.Environment) { - defer c.unqueueTask(schemas.TaskTypePullEnvironmentMetrics, string(env.Key())) +func (c *Controller) TaskHandlerPullEnvironmentMetrics(ctx context.Context, env schemas.Environment) { + defer c.unqueueTask(ctx, schemas.TaskTypePullEnvironmentMetrics, string(env.Key())) // On errors, we do not want to retry these tasks - if err := c.PullEnvironmentMetrics(env); err != nil { + if err := c.PullEnvironmentMetrics(ctx, env); err != nil { log.WithFields(log.Fields{ "project-name": env.ProjectName, "environment-name": env.Name, @@ -109,7 +109,7 @@ func (c *Controller) TaskHandlerPullEnvironmentMetrics(env schemas.Environment) // TaskHandlerPullRefsFromProject .. func (c *Controller) TaskHandlerPullRefsFromProject(ctx context.Context, p schemas.Project) { - defer c.unqueueTask(schemas.TaskTypePullRefsFromProject, string(p.Key())) + defer c.unqueueTask(ctx, schemas.TaskTypePullRefsFromProject, string(p.Key())) // On errors, we do not want to retry these tasks if err := c.PullRefsFromProject(ctx, p); err != nil { @@ -121,11 +121,11 @@ func (c *Controller) TaskHandlerPullRefsFromProject(ctx context.Context, p schem } // TaskHandlerPullRefMetrics .. -func (c *Controller) TaskHandlerPullRefMetrics(ref schemas.Ref) { - defer c.unqueueTask(schemas.TaskTypePullRefMetrics, string(ref.Key())) +func (c *Controller) TaskHandlerPullRefMetrics(ctx context.Context, ref schemas.Ref) { + defer c.unqueueTask(ctx, schemas.TaskTypePullRefMetrics, string(ref.Key())) // On errors, we do not want to retry these tasks - if err := c.PullRefMetrics(ref); err != nil { + if err := c.PullRefMetrics(ctx, ref); err != nil { log.WithFields(log.Fields{ "project-name": ref.Project.Name, "ref": ref.Name, @@ -136,7 +136,7 @@ func (c *Controller) TaskHandlerPullRefMetrics(ref schemas.Ref) { // TaskHandlerPullProjectsFromWildcards .. func (c *Controller) TaskHandlerPullProjectsFromWildcards(ctx context.Context) { - defer c.unqueueTask(schemas.TaskTypePullProjectsFromWildcards, "_") + defer c.unqueueTask(ctx, schemas.TaskTypePullProjectsFromWildcards, "_") defer c.TaskController.monitorLastTaskScheduling(schemas.TaskTypePullProjectsFromWildcards) log.WithFields( @@ -152,10 +152,10 @@ func (c *Controller) TaskHandlerPullProjectsFromWildcards(ctx context.Context) { // TaskHandlerPullEnvironmentsFromProjects .. func (c *Controller) TaskHandlerPullEnvironmentsFromProjects(ctx context.Context) { - defer c.unqueueTask(schemas.TaskTypePullEnvironmentsFromProjects, "_") + defer c.unqueueTask(ctx, schemas.TaskTypePullEnvironmentsFromProjects, "_") defer c.TaskController.monitorLastTaskScheduling(schemas.TaskTypePullEnvironmentsFromProjects) - projectsCount, err := c.Store.ProjectsCount() + projectsCount, err := c.Store.ProjectsCount(ctx) if err != nil { log.Error(err.Error()) } @@ -166,7 +166,7 @@ func (c *Controller) TaskHandlerPullEnvironmentsFromProjects(ctx context.Context }, ).Info("scheduling environments from projects pull") - projects, err := c.Store.Projects() + projects, err := c.Store.Projects(ctx) if err != nil { log.Error(err) } @@ -178,10 +178,10 @@ func (c *Controller) TaskHandlerPullEnvironmentsFromProjects(ctx context.Context // TaskHandlerPullRefsFromProjects .. func (c *Controller) TaskHandlerPullRefsFromProjects(ctx context.Context) { - defer c.unqueueTask(schemas.TaskTypePullRefsFromProjects, "_") + defer c.unqueueTask(ctx, schemas.TaskTypePullRefsFromProjects, "_") defer c.TaskController.monitorLastTaskScheduling(schemas.TaskTypePullRefsFromProjects) - projectsCount, err := c.Store.ProjectsCount() + projectsCount, err := c.Store.ProjectsCount(ctx) if err != nil { log.Error(err.Error()) } @@ -192,7 +192,7 @@ func (c *Controller) TaskHandlerPullRefsFromProjects(ctx context.Context) { }, ).Info("scheduling refs from projects pull") - projects, err := c.Store.Projects() + projects, err := c.Store.Projects(ctx) if err != nil { log.Error(err) } @@ -204,15 +204,15 @@ func (c *Controller) TaskHandlerPullRefsFromProjects(ctx context.Context) { // TaskHandlerPullMetrics .. func (c *Controller) TaskHandlerPullMetrics(ctx context.Context) { - defer c.unqueueTask(schemas.TaskTypePullMetrics, "_") + defer c.unqueueTask(ctx, schemas.TaskTypePullMetrics, "_") defer c.TaskController.monitorLastTaskScheduling(schemas.TaskTypePullMetrics) - refsCount, err := c.Store.RefsCount() + refsCount, err := c.Store.RefsCount(ctx) if err != nil { log.Error(err) } - envsCount, err := c.Store.EnvironmentsCount() + envsCount, err := c.Store.EnvironmentsCount(ctx) if err != nil { log.Error(err) } @@ -225,7 +225,7 @@ func (c *Controller) TaskHandlerPullMetrics(ctx context.Context) { ).Info("scheduling metrics pull") // ENVIRONMENTS - envs, err := c.Store.Environments() + envs, err := c.Store.Environments(ctx) if err != nil { log.Error(err) } @@ -235,7 +235,7 @@ func (c *Controller) TaskHandlerPullMetrics(ctx context.Context) { } // REFS - refs, err := c.Store.Refs() + refs, err := c.Store.Refs(ctx) if err != nil { log.Error(err) } @@ -247,7 +247,7 @@ func (c *Controller) TaskHandlerPullMetrics(ctx context.Context) { // TaskHandlerGarbageCollectProjects .. func (c *Controller) TaskHandlerGarbageCollectProjects(ctx context.Context) error { - defer c.unqueueTask(schemas.TaskTypeGarbageCollectProjects, "_") + defer c.unqueueTask(ctx, schemas.TaskTypeGarbageCollectProjects, "_") defer c.TaskController.monitorLastTaskScheduling(schemas.TaskTypeGarbageCollectProjects) return c.GarbageCollectProjects(ctx) @@ -255,7 +255,7 @@ func (c *Controller) TaskHandlerGarbageCollectProjects(ctx context.Context) erro // TaskHandlerGarbageCollectEnvironments .. func (c *Controller) TaskHandlerGarbageCollectEnvironments(ctx context.Context) error { - defer c.unqueueTask(schemas.TaskTypeGarbageCollectEnvironments, "_") + defer c.unqueueTask(ctx, schemas.TaskTypeGarbageCollectEnvironments, "_") defer c.TaskController.monitorLastTaskScheduling(schemas.TaskTypeGarbageCollectEnvironments) return c.GarbageCollectEnvironments(ctx) @@ -263,7 +263,7 @@ func (c *Controller) TaskHandlerGarbageCollectEnvironments(ctx context.Context) // TaskHandlerGarbageCollectRefs .. func (c *Controller) TaskHandlerGarbageCollectRefs(ctx context.Context) error { - defer c.unqueueTask(schemas.TaskTypeGarbageCollectRefs, "_") + defer c.unqueueTask(ctx, schemas.TaskTypeGarbageCollectRefs, "_") defer c.TaskController.monitorLastTaskScheduling(schemas.TaskTypeGarbageCollectRefs) return c.GarbageCollectRefs(ctx) @@ -271,7 +271,7 @@ func (c *Controller) TaskHandlerGarbageCollectRefs(ctx context.Context) error { // TaskHandlerGarbageCollectMetrics .. func (c *Controller) TaskHandlerGarbageCollectMetrics(ctx context.Context) error { - defer c.unqueueTask(schemas.TaskTypeGarbageCollectMetrics, "_") + defer c.unqueueTask(ctx, schemas.TaskTypeGarbageCollectMetrics, "_") defer c.TaskController.monitorLastTaskScheduling(schemas.TaskTypeGarbageCollectMetrics) return c.GarbageCollectMetrics(ctx) @@ -309,14 +309,15 @@ func (c *Controller) Schedule(ctx context.Context, pull config.Pull, gc config.G func (c *Controller) ScheduleRedisSetKeepalive(ctx context.Context) { go func(ctx context.Context) { ticker := time.NewTicker(time.Duration(5) * time.Second) + for { select { case <-ctx.Done(): log.Info("stopped redis keepalive") + return case <-ticker.C: - _, err := c.Store.(*store.Redis).SetKeepalive(c.UUID.String(), time.Duration(10)*time.Second) - if err != nil { + if _, err := c.Store.(*store.Redis).SetKeepalive(ctx, c.UUID.String(), time.Duration(10)*time.Second); err != nil { log.WithError(err).Fatal("setting keepalive") } } @@ -336,22 +337,26 @@ func (c *Controller) ScheduleTask(ctx context.Context, tt schemas.TaskType, uniq qlen, err := c.TaskController.Queue.Len() if err != nil { log.WithFields(logFields).Warn("unable to read task queue length, skipping scheduling of task..") + return } if qlen >= c.TaskController.Queue.Options().BufferSize { log.WithFields(logFields).Warn("queue buffer size exhausted, skipping scheduling of task..") + return } - queued, err := c.Store.QueueTask(tt, uniqueID, c.UUID.String()) + queued, err := c.Store.QueueTask(ctx, tt, uniqueID, c.UUID.String()) if err != nil { log.WithFields(logFields).Warn("unable to declare the queueing, skipping scheduling of task..") + return } if !queued { log.WithFields(logFields).Debug("task already queued, skipping scheduling of task..") + return } @@ -366,6 +371,7 @@ func (c *Controller) ScheduleTask(ctx context.Context, tt schemas.TaskType, uniq func (c *Controller) ScheduleTaskWithTicker(ctx context.Context, tt schemas.TaskType, intervalSeconds int) { if intervalSeconds <= 0 { log.WithField("task", tt).Warn("task scheduling misconfigured, currently disabled") + return } @@ -378,10 +384,12 @@ func (c *Controller) ScheduleTaskWithTicker(ctx context.Context, tt schemas.Task go func(ctx context.Context) { ticker := time.NewTicker(time.Duration(intervalSeconds) * time.Second) + for { select { case <-ctx.Done(): log.WithField("task", tt).Info("scheduling of task stopped") + return case <-ticker.C: switch tt { @@ -398,6 +406,7 @@ func (tc *TaskController) monitorNextTaskScheduling(tt schemas.TaskType, duratio if _, ok := tc.TaskSchedulingMonitoring[tt]; !ok { tc.TaskSchedulingMonitoring[tt] = &monitor.TaskSchedulingStatus{} } + tc.TaskSchedulingMonitoring[tt].Next = time.Now().Add(time.Duration(duration) * time.Second) } @@ -405,5 +414,6 @@ func (tc *TaskController) monitorLastTaskScheduling(tt schemas.TaskType) { if _, ok := tc.TaskSchedulingMonitoring[tt]; !ok { tc.TaskSchedulingMonitoring[tt] = &monitor.TaskSchedulingStatus{} } + tc.TaskSchedulingMonitoring[tt].Last = time.Now() } diff --git a/pkg/controller/store.go b/pkg/controller/store.go index b0025392..eae61878 100644 --- a/pkg/controller/store.go +++ b/pkg/controller/store.go @@ -1,6 +1,8 @@ package controller import ( + "context" + "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/store" log "github.com/sirupsen/logrus" @@ -13,8 +15,8 @@ func metricLogFields(m schemas.Metric) log.Fields { } } -func storeGetMetric(s store.Store, m *schemas.Metric) { - if err := s.GetMetric(m); err != nil { +func storeGetMetric(ctx context.Context, s store.Store, m *schemas.Metric) { + if err := s.GetMetric(ctx, m); err != nil { log.WithFields( metricLogFields(*m), ).WithField( @@ -23,8 +25,8 @@ func storeGetMetric(s store.Store, m *schemas.Metric) { } } -func storeSetMetric(s store.Store, m schemas.Metric) { - if err := s.SetMetric(m); err != nil { +func storeSetMetric(ctx context.Context, s store.Store, m schemas.Metric) { + if err := s.SetMetric(ctx, m); err != nil { log.WithFields( metricLogFields(m), ).WithField( @@ -33,8 +35,8 @@ func storeSetMetric(s store.Store, m schemas.Metric) { } } -func storeDelMetric(s store.Store, m schemas.Metric) { - if err := s.DelMetric(m.Key()); err != nil { +func storeDelMetric(ctx context.Context, s store.Store, m schemas.Metric) { + if err := s.DelMetric(ctx, m.Key()); err != nil { log.WithFields( metricLogFields(m), ).WithField( diff --git a/pkg/controller/webhooks.go b/pkg/controller/webhooks.go index 8ccb1f3e..f494b563 100644 --- a/pkg/controller/webhooks.go +++ b/pkg/controller/webhooks.go @@ -13,9 +13,11 @@ import ( goGitlab "github.com/xanzy/go-gitlab" ) -func (c *Controller) processPipelineEvent(e goGitlab.PipelineEvent) { - var refKind schemas.RefKind - refName := e.ObjectAttributes.Ref +func (c *Controller) processPipelineEvent(ctx context.Context, e goGitlab.PipelineEvent) { + var ( + refKind schemas.RefKind + refName = e.ObjectAttributes.Ref + ) // TODO: Perhaps it would be nice to match upon the regexp to validate // that it is actually a merge request ref @@ -28,23 +30,24 @@ func (c *Controller) processPipelineEvent(e goGitlab.PipelineEvent) { refKind = schemas.RefKindBranch } - c.triggerRefMetricsPull(schemas.NewRef( + c.triggerRefMetricsPull(ctx, schemas.NewRef( schemas.NewProject(e.Project.PathWithNamespace), refKind, refName, )) } -func (c *Controller) triggerRefMetricsPull(ref schemas.Ref) { +func (c *Controller) triggerRefMetricsPull(ctx context.Context, ref schemas.Ref) { logFields := log.Fields{ "project-name": ref.Project.Name, "ref": ref.Name, "ref-kind": ref.Kind, } - refExists, err := c.Store.RefExists(ref.Key()) + refExists, err := c.Store.RefExists(ctx, ref.Key()) if err != nil { log.WithFields(logFields).WithError(err).Error("reading ref from the store") + return } @@ -52,9 +55,10 @@ func (c *Controller) triggerRefMetricsPull(ref schemas.Ref) { if !refExists { p := schemas.NewProject(ref.Project.Name) - projectExists, err := c.Store.ProjectExists(p.Key()) + projectExists, err := c.Store.ProjectExists(ctx, p.Key()) if err != nil { log.WithFields(logFields).WithError(err).Error("reading project from the store") + return } @@ -66,6 +70,7 @@ func (c *Controller) triggerRefMetricsPull(ref schemas.Ref) { matches, err := isRefMatchingWilcard(w, ref) if err != nil { log.WithError(err).Warn("checking if the ref matches the wildcard config") + continue } @@ -76,34 +81,42 @@ func (c *Controller) triggerRefMetricsPull(ref schemas.Ref) { log.WithFields(logFields).Debug("project ref not matching wildcard, skipping..") } } + log.WithFields(logFields).Info("done looking up for wildcards matching the project ref") + return } if projectExists { // If the project exists, we check that the ref matches it's configuration - if err := c.Store.GetProject(&p); err != nil { + if err := c.Store.GetProject(ctx, &p); err != nil { log.WithFields(logFields).WithError(err).Error("reading project from the store") + return } matches, err := isRefMatchingProjectPullRefs(p.Pull.Refs, ref) if err != nil { log.WithError(err).Error("checking if the ref matches the project config") + return } if matches { ref.Project = p - if err = c.Store.SetRef(ref); err != nil { + + if err = c.Store.SetRef(ctx, ref); err != nil { log.WithFields(logFields).WithError(err).Error("writing ref in the store") + return } + goto schedulePull } } log.WithFields(logFields).Info("ref not configured in the exporter, ignoring pipeline webhook") + return } @@ -114,31 +127,36 @@ schedulePull: c.ScheduleTask(context.TODO(), schemas.TaskTypePullRefMetrics, string(ref.Key()), ref) } -func (c *Controller) processDeploymentEvent(e goGitlab.DeploymentEvent) { - c.triggerEnvironmentMetricsPull(schemas.Environment{ - ProjectName: e.Project.PathWithNamespace, - Name: e.Environment, - }) +func (c *Controller) processDeploymentEvent(ctx context.Context, e goGitlab.DeploymentEvent) { + c.triggerEnvironmentMetricsPull( + ctx, + schemas.Environment{ + ProjectName: e.Project.PathWithNamespace, + Name: e.Environment, + }, + ) } -func (c *Controller) triggerEnvironmentMetricsPull(env schemas.Environment) { +func (c *Controller) triggerEnvironmentMetricsPull(ctx context.Context, env schemas.Environment) { logFields := log.Fields{ "project-name": env.ProjectName, "environment-name": env.Name, } - envExists, err := c.Store.EnvironmentExists(env.Key()) + envExists, err := c.Store.EnvironmentExists(ctx, env.Key()) if err != nil { log.WithFields(logFields).WithError(err).Error("reading environment from the store") + return } if !envExists { p := schemas.NewProject(env.ProjectName) - projectExists, err := c.Store.ProjectExists(p.Key()) + projectExists, err := c.Store.ProjectExists(ctx, p.Key()) if err != nil { log.WithFields(logFields).WithError(err).Error("reading project from the store") + return } @@ -150,55 +168,62 @@ func (c *Controller) triggerEnvironmentMetricsPull(env schemas.Environment) { matches, err := isEnvMatchingWilcard(w, env) if err != nil { log.WithError(err).Warn("checking if the env matches the wildcard config") + continue } if matches { - c.ScheduleTask(context.TODO(), schemas.TaskTypePullProjectsFromWildcard, strconv.Itoa(id), strconv.Itoa(id), w) + c.ScheduleTask(ctx, schemas.TaskTypePullProjectsFromWildcard, strconv.Itoa(id), strconv.Itoa(id), w) log.WithFields(logFields).Info("project environment not currently exported but its configuration matches a wildcard, triggering a pull of the projects from this wildcard") } else { log.WithFields(logFields).Debug("project ref not matching wildcard, skipping..") } } + log.WithFields(logFields).Info("done looking up for wildcards matching the project ref") + return } if projectExists { - if err := c.Store.GetProject(&p); err != nil { + if err := c.Store.GetProject(ctx, &p); err != nil { log.WithFields(logFields).WithError(err).Error("reading project from the store") } matches, err := isEnvMatchingProjectPullEnvironments(p.Pull.Environments, env) if err != nil { log.WithError(err).Error("checking if the env matches the project config") + return } if matches { // As we do not get the environment ID within the deployment event, we need to query it back.. - if err = c.UpdateEnvironment(&env); err != nil { + if err = c.UpdateEnvironment(ctx, &env); err != nil { log.WithFields(logFields).WithError(err).Error("updating event from GitLab API") + return } + goto schedulePull } } log.WithFields(logFields).Info("environment not configured in the exporter, ignoring deployment webhook") + return } // Need to refresh the env from the store in order to get at least it's ID if env.ID == 0 { - if err = c.Store.GetEnvironment(&env); err != nil { + if err = c.Store.GetEnvironment(ctx, &env); err != nil { log.WithFields(logFields).WithError(err).Error("reading environment from the store") } } schedulePull: log.WithFields(logFields).Info("received a deployment webhook from GitLab for an environment, triggering metrics pull") - c.ScheduleTask(context.TODO(), schemas.TaskTypePullEnvironmentMetrics, string(env.Key()), env) + c.ScheduleTask(ctx, schemas.TaskTypePullEnvironmentMetrics, string(env.Key()), env) } func isRefMatchingProjectPullRefs(pprs config.ProjectPullRefs, ref schemas.Ref) (matches bool, err error) { @@ -222,9 +247,11 @@ func isRefMatchingProjectPullRefs(pprs config.ProjectPullRefs, ref schemas.Ref) // Then we check if it matches the regexp var re *regexp.Regexp + if re, err = schemas.GetRefRegexp(pprs, ref.Kind); err != nil { return } + return re.MatchString(ref.Name), nil } @@ -236,9 +263,11 @@ func isEnvMatchingProjectPullEnvironments(ppe config.ProjectPullEnvironments, en // Then we check if it matches the regexp var re *regexp.Regexp + if re, err = regexp.Compile(ppe.Regexp); err != nil { return } + return re.MatchString(env.Name), nil } diff --git a/pkg/controller/webhooks_test.go b/pkg/controller/webhooks_test.go index f5617db5..61ebf218 100644 --- a/pkg/controller/webhooks_test.go +++ b/pkg/controller/webhooks_test.go @@ -8,7 +8,7 @@ import ( ) func TestTriggerRefMetricsPull(_ *testing.T) { - c, _, srv := newTestController(config.Config{}) + ctx, c, _, srv := newTestController(config.Config{}) srv.Close() ref1 := schemas.Ref{ @@ -22,16 +22,16 @@ func TestTriggerRefMetricsPull(_ *testing.T) { Name: "main", } - c.Store.SetRef(ref1) - c.Store.SetProject(p2) + c.Store.SetRef(ctx, ref1) + c.Store.SetProject(ctx, p2) // TODO: Assert results somehow - c.triggerRefMetricsPull(ref1) - c.triggerRefMetricsPull(ref2) + c.triggerRefMetricsPull(ctx, ref1) + c.triggerRefMetricsPull(ctx, ref2) } func TestTriggerEnvironmentMetricsPull(_ *testing.T) { - c, _, srv := newTestController(config.Config{}) + ctx, c, _, srv := newTestController(config.Config{}) srv.Close() p1 := schemas.NewProject("foo/bar") @@ -45,11 +45,11 @@ func TestTriggerEnvironmentMetricsPull(_ *testing.T) { Name: "prod", } - c.Store.SetProject(p1) - c.Store.SetEnvironment(env1) - c.Store.SetEnvironment(env2) + c.Store.SetProject(ctx, p1) + c.Store.SetEnvironment(ctx, env1) + c.Store.SetEnvironment(ctx, env2) // TODO: Assert results somehow - c.triggerEnvironmentMetricsPull(env1) - c.triggerEnvironmentMetricsPull(env2) + c.triggerEnvironmentMetricsPull(ctx, env1) + c.triggerEnvironmentMetricsPull(ctx, env2) } diff --git a/pkg/gitlab/branches.go b/pkg/gitlab/branches.go index c5c50fd4..d554bf0a 100644 --- a/pkg/gitlab/branches.go +++ b/pkg/gitlab/branches.go @@ -1,6 +1,7 @@ package gitlab import ( + "context" "regexp" "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" @@ -9,7 +10,7 @@ import ( ) // GetProjectBranches .. -func (c *Client) GetProjectBranches(p schemas.Project) ( +func (c *Client) GetProjectBranches(ctx context.Context, p schemas.Project) ( refs schemas.Refs, err error, ) { @@ -23,18 +24,24 @@ func (c *Client) GetProjectBranches(p schemas.Project) ( } var re *regexp.Regexp + if re, err = regexp.Compile(p.Pull.Refs.Branches.Regexp); err != nil { return } for { - c.rateLimit() - var branches []*goGitlab.Branch - var resp *goGitlab.Response - branches, resp, err = c.Branches.ListBranches(p.Name, options) + c.rateLimit(ctx) + + var ( + branches []*goGitlab.Branch + resp *goGitlab.Response + ) + + branches, resp, err = c.Branches.ListBranches(p.Name, options, goGitlab.WithContext(ctx)) if err != nil { return } + c.requestsRemaining(resp) for _, branch := range branches { @@ -47,6 +54,7 @@ func (c *Client) GetProjectBranches(p schemas.Project) ( if resp.CurrentPage >= resp.NextPage { break } + options.Page = resp.NextPage } @@ -54,17 +62,19 @@ func (c *Client) GetProjectBranches(p schemas.Project) ( } // GetBranchLatestCommit .. -func (c *Client) GetBranchLatestCommit(project, branch string) (string, float64, error) { +func (c *Client) GetBranchLatestCommit(ctx context.Context, project, branch string) (string, float64, error) { log.WithFields(log.Fields{ "project-name": project, "branch": branch, }).Debug("reading project branch") - c.rateLimit() - b, resp, err := c.Branches.GetBranch(project, branch, nil) + c.rateLimit(ctx) + + b, resp, err := c.Branches.GetBranch(project, branch, goGitlab.WithContext(ctx)) if err != nil { return "", 0, err } + c.requestsRemaining(resp) return b.Commit.ShortID, float64(b.Commit.CommittedDate.Unix()), nil diff --git a/pkg/gitlab/branches_test.go b/pkg/gitlab/branches_test.go index 00311755..afb095a4 100644 --- a/pkg/gitlab/branches_test.go +++ b/pkg/gitlab/branches_test.go @@ -11,7 +11,7 @@ import ( ) func TestGetProjectBranches(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/repository/branches"), @@ -30,6 +30,7 @@ func TestGetProjectBranches(t *testing.T) { if currentPage == 1 { fmt.Fprint(w, `[{"name":"main"},{"name":"dev"}]`) + return } @@ -43,7 +44,7 @@ func TestGetProjectBranches(t *testing.T) { p := schemas.NewProject("foo") expectedRef := schemas.NewRef(p, schemas.RefKindBranch, "main") - refs, err := c.GetProjectBranches(p) + refs, err := c.GetProjectBranches(ctx, p) assert.NoError(t, err) assert.Len(t, refs, 1) assert.Equal(t, schemas.Refs{ @@ -52,18 +53,18 @@ func TestGetProjectBranches(t *testing.T) { // Test invalid project name p.Name = "invalid" - _, err = c.GetProjectBranches(p) + _, err = c.GetProjectBranches(ctx, p) assert.Error(t, err) // Test invalid regexp p.Name = "foo" p.Pull.Refs.Branches.Regexp = `[` - _, err = c.GetProjectBranches(p) + _, err = c.GetProjectBranches(ctx, p) assert.Error(t, err) } func TestGetBranchLatestCommit(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() mux.HandleFunc("/api/v4/projects/1/repository/branches/main", @@ -78,7 +79,7 @@ func TestGetBranchLatestCommit(t *testing.T) { }`) }) - commitShortID, commitCreatedAt, err := c.GetBranchLatestCommit("1", "main") + commitShortID, commitCreatedAt, err := c.GetBranchLatestCommit(ctx, "1", "main") assert.NoError(t, err) assert.Equal(t, "7b5c3cc", commitShortID) assert.Equal(t, float64(1553540113), commitCreatedAt) diff --git a/pkg/gitlab/client.go b/pkg/gitlab/client.go index e1e0b4e4..e3c823b3 100644 --- a/pkg/gitlab/client.go +++ b/pkg/gitlab/client.go @@ -1,6 +1,7 @@ package gitlab import ( + "context" "crypto/tls" "fmt" "net/http" @@ -89,13 +90,23 @@ func NewClient(cfg ClientConfig) (*Client, error) { } // ReadinessCheck .. -func (c *Client) ReadinessCheck() healthcheck.Check { +func (c *Client) ReadinessCheck(ctx context.Context) healthcheck.Check { return func() error { if c.Readiness.HTTPClient == nil { return fmt.Errorf("readiness http client not configured") } - resp, err := c.Readiness.HTTPClient.Get(c.Readiness.URL) + req, err := http.NewRequestWithContext( + ctx, + http.MethodGet, + c.Readiness.URL, + nil, + ) + if err != nil { + return err + } + + resp, err := c.Readiness.HTTPClient.Do(req) if err != nil { return err } @@ -112,20 +123,19 @@ func (c *Client) ReadinessCheck() healthcheck.Check { } } -func (c *Client) rateLimit() { - ratelimit.Take(c.RateLimiter) +func (c *Client) rateLimit(ctx context.Context) { + ratelimit.Take(ctx, c.RateLimiter) // Used for monitoring purposes c.RateCounter.Incr(1) c.RequestsCounter++ } func (c *Client) requestsRemaining(response *goGitlab.Response) { - remaining := response.Header.Get("ratelimit-remaining") - if remaining != "" { + if remaining := response.Header.Get("ratelimit-remaining"); remaining != "" { c.RequestsRemaining, _ = strconv.Atoi(remaining) } - limit := response.Header.Get("ratelimit-limit") - if limit != "" { + + if limit := response.Header.Get("ratelimit-limit"); limit != "" { c.RequestsLimit, _ = strconv.Atoi(limit) } } diff --git a/pkg/gitlab/client_test.go b/pkg/gitlab/client_test.go index ef318067..f5f4a20c 100644 --- a/pkg/gitlab/client_test.go +++ b/pkg/gitlab/client_test.go @@ -1,6 +1,7 @@ package gitlab import ( + "context" "fmt" "net/http" "net/http/httptest" @@ -13,8 +14,8 @@ import ( goGitlab "github.com/xanzy/go-gitlab" ) -// Mocking helpers -func getMockedClient() (*http.ServeMux, *httptest.Server, *Client) { +// Mocking helpers. +func getMockedClient() (context.Context, *http.ServeMux, *httptest.Server, *Client) { mux := http.NewServeMux() server := httptest.NewServer(mux) @@ -31,7 +32,7 @@ func getMockedClient() (*http.ServeMux, *httptest.Server, *Client) { RateCounter: ratecounter.NewRateCounter(time.Second), } - return mux, server, c + return context.Background(), mux, server, c } func TestNewHTTPClient(t *testing.T) { @@ -60,24 +61,30 @@ func TestNewClient(t *testing.T) { } func TestReadinessCheck(t *testing.T) { - mux, server, c := getMockedClient() - mux.HandleFunc(fmt.Sprintf("/200"), + ctx, mux, server, c := getMockedClient() + mux.HandleFunc( + "/200", func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "GET", r.Method) w.WriteHeader(http.StatusOK) - }) - mux.HandleFunc(fmt.Sprintf("/500"), + }, + ) + mux.HandleFunc( + "/500", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) - }) + }, + ) - readinessCheck := c.ReadinessCheck() + readinessCheck := c.ReadinessCheck(ctx) assert.Error(t, readinessCheck()) c.Readiness.HTTPClient = NewHTTPClient(false) c.Readiness.URL = fmt.Sprintf("%s/200", server.URL) + assert.NoError(t, readinessCheck()) c.Readiness.URL = fmt.Sprintf("%s/500", server.URL) + assert.Error(t, readinessCheck()) } diff --git a/pkg/gitlab/environments.go b/pkg/gitlab/environments.go index cf71c5e3..ca5e235e 100644 --- a/pkg/gitlab/environments.go +++ b/pkg/gitlab/environments.go @@ -1,6 +1,7 @@ package gitlab import ( + "context" "regexp" "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" @@ -9,7 +10,7 @@ import ( ) // GetProjectEnvironments .. -func (c *Client) GetProjectEnvironments(p schemas.Project) ( +func (c *Client) GetProjectEnvironments(ctx context.Context, p schemas.Project) ( envs schemas.Environments, err error, ) { @@ -32,13 +33,18 @@ func (c *Client) GetProjectEnvironments(p schemas.Project) ( } for { - c.rateLimit() - var glenvs []*goGitlab.Environment - var resp *goGitlab.Response - glenvs, resp, err = c.Environments.ListEnvironments(p.Name, options) + c.rateLimit(ctx) + + var ( + glenvs []*goGitlab.Environment + resp *goGitlab.Response + ) + + glenvs, resp, err = c.Environments.ListEnvironments(p.Name, options, goGitlab.WithContext(ctx)) if err != nil { return } + c.requestsRemaining(resp) for _, glenv := range glenvs { @@ -61,6 +67,7 @@ func (c *Client) GetProjectEnvironments(p schemas.Project) ( if resp.CurrentPage >= resp.NextPage { break } + options.Page = resp.NextPage } @@ -68,17 +75,31 @@ func (c *Client) GetProjectEnvironments(p schemas.Project) ( } // GetEnvironment .. -func (c *Client) GetEnvironment(project string, environmentID int) (schemas.Environment, error) { - environment := schemas.Environment{ +func (c *Client) GetEnvironment( + ctx context.Context, + project string, + environmentID int, +) ( + environment schemas.Environment, + err error, +) { + environment = schemas.Environment{ ProjectName: project, ID: environmentID, } - c.rateLimit() - e, resp, err := c.Environments.GetEnvironment(project, environmentID, nil) + c.rateLimit(ctx) + + var ( + e *goGitlab.Environment + resp *goGitlab.Response + ) + + e, resp, err = c.Environments.GetEnvironment(project, environmentID, goGitlab.WithContext(ctx)) if err != nil || e == nil { - return environment, err + return } + c.requestsRemaining(resp) environment.Name = e.Name @@ -88,35 +109,37 @@ func (c *Client) GetEnvironment(project string, environmentID int) (schemas.Envi environment.Available = true } - if e.LastDeployment != nil { - if e.LastDeployment.Deployable.Tag { - environment.LatestDeployment.RefKind = schemas.RefKindTag - } else { - environment.LatestDeployment.RefKind = schemas.RefKindBranch - } - - environment.LatestDeployment.RefName = e.LastDeployment.Ref - environment.LatestDeployment.JobID = e.LastDeployment.Deployable.ID - environment.LatestDeployment.DurationSeconds = e.LastDeployment.Deployable.Duration - environment.LatestDeployment.Status = e.LastDeployment.Deployable.Status - - if e.LastDeployment.Deployable.User != nil { - environment.LatestDeployment.Username = e.LastDeployment.Deployable.User.Username - } - - if e.LastDeployment.Deployable.Commit != nil { - environment.LatestDeployment.CommitShortID = e.LastDeployment.Deployable.Commit.ShortID - } - - if e.LastDeployment.CreatedAt != nil { - environment.LatestDeployment.Timestamp = float64(e.LastDeployment.CreatedAt.Unix()) - } - } else { + if e.LastDeployment == nil { log.WithFields(log.Fields{ "project-name": project, "environment-name": e.Name, }).Warn("no deployments found for the environment") + + return } - return environment, nil + if e.LastDeployment.Deployable.Tag { + environment.LatestDeployment.RefKind = schemas.RefKindTag + } else { + environment.LatestDeployment.RefKind = schemas.RefKindBranch + } + + environment.LatestDeployment.RefName = e.LastDeployment.Ref + environment.LatestDeployment.JobID = e.LastDeployment.Deployable.ID + environment.LatestDeployment.DurationSeconds = e.LastDeployment.Deployable.Duration + environment.LatestDeployment.Status = e.LastDeployment.Deployable.Status + + if e.LastDeployment.Deployable.User != nil { + environment.LatestDeployment.Username = e.LastDeployment.Deployable.User.Username + } + + if e.LastDeployment.Deployable.Commit != nil { + environment.LatestDeployment.CommitShortID = e.LastDeployment.Deployable.Commit.ShortID + } + + if e.LastDeployment.CreatedAt != nil { + environment.LatestDeployment.Timestamp = float64(e.LastDeployment.CreatedAt.Unix()) + } + + return } diff --git a/pkg/gitlab/environments_test.go b/pkg/gitlab/environments_test.go index 09adc028..850422ef 100644 --- a/pkg/gitlab/environments_test.go +++ b/pkg/gitlab/environments_test.go @@ -11,10 +11,11 @@ import ( ) func TestGetProjectEnvironments(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() - mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/environments"), + mux.HandleFunc( + "/api/v4/projects/foo/environments", func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "GET", r.Method) assert.Equal(t, []string{"100"}, r.URL.Query()["per_page"]) @@ -30,21 +31,26 @@ func TestGetProjectEnvironments(t *testing.T) { if scope, ok := r.URL.Query()["states"]; ok && len(scope) == 1 && scope[0] == "available" { fmt.Fprint(w, `[{"id":1338,"name":"main"}]`) + return } if currentPage == 1 { fmt.Fprint(w, `[{"id":1338,"name":"main"},{"id":1337,"name":"dev"}]`) + return } fmt.Fprint(w, `[]`) - }) + }, + ) - mux.HandleFunc(fmt.Sprintf("/api/v4/projects/0/environments"), + mux.HandleFunc( + "/api/v4/projects/0/environments", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) - }) + }, + ) p := schemas.NewProject("foo") p.Pull.Environments.Regexp = "^dev" @@ -61,19 +67,19 @@ func TestGetProjectEnvironments(t *testing.T) { xenv.Key(): xenv, } - envs, err := c.GetProjectEnvironments(p) + envs, err := c.GetProjectEnvironments(ctx, p) assert.NoError(t, err) assert.Equal(t, xenvs, envs) // Test invalid project p.Name = "" - _, err = c.GetProjectEnvironments(p) + _, err = c.GetProjectEnvironments(ctx, p) assert.Error(t, err) // Test invalid regexp p.Name = "foo" p.Pull.Environments.Regexp = "[" - _, err = c.GetProjectEnvironments(p) + _, err = c.GetProjectEnvironments(ctx, p) assert.Error(t, err) // Test exclude stopped @@ -90,13 +96,13 @@ func TestGetProjectEnvironments(t *testing.T) { p.Pull.Environments.Regexp = ".*" p.Pull.Environments.ExcludeStopped = true - envs, err = c.GetProjectEnvironments(p) + envs, err = c.GetProjectEnvironments(ctx, p) assert.NoError(t, err) assert.Equal(t, xenvs, envs) } func TestGetEnvironment(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() mux.HandleFunc("/api/v4/projects/foo/environments/1", @@ -127,7 +133,7 @@ func TestGetEnvironment(t *testing.T) { }`) }) - e, err := c.GetEnvironment("foo", 1) + e, err := c.GetEnvironment(ctx, "foo", 1) assert.NoError(t, err) assert.NotNil(t, e) diff --git a/pkg/gitlab/jobs.go b/pkg/gitlab/jobs.go index 61920b44..23dfe5d9 100644 --- a/pkg/gitlab/jobs.go +++ b/pkg/gitlab/jobs.go @@ -1,6 +1,7 @@ package gitlab import ( + "context" "strings" "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" @@ -9,7 +10,7 @@ import ( ) // ListRefPipelineJobs .. -func (c *Client) ListRefPipelineJobs(ref schemas.Ref) (jobs []schemas.Job, err error) { +func (c *Client) ListRefPipelineJobs(ctx context.Context, ref schemas.Ref) (jobs []schemas.Job, err error) { if ref.LatestPipeline == (schemas.Pipeline{}) { log.WithFields( log.Fields{ @@ -17,17 +18,19 @@ func (c *Client) ListRefPipelineJobs(ref schemas.Ref) (jobs []schemas.Job, err e "ref": ref.Name, }, ).Debug("most recent pipeline not defined, exiting..") + return } - jobs, err = c.ListPipelineJobs(ref.Project.Name, ref.LatestPipeline.ID) + jobs, err = c.ListPipelineJobs(ctx, ref.Project.Name, ref.LatestPipeline.ID) if err != nil { return } if ref.Project.Pull.Pipeline.Jobs.FromChildPipelines.Enabled { var childJobs []schemas.Job - childJobs, err = c.ListPipelineChildJobs(ref.Project.Name, ref.LatestPipeline.ID) + + childJobs, err = c.ListPipelineChildJobs(ctx, ref.Project.Name, ref.LatestPipeline.ID) if err != nil { return } @@ -39,9 +42,11 @@ func (c *Client) ListRefPipelineJobs(ref schemas.Ref) (jobs []schemas.Job, err e } // ListPipelineJobs .. -func (c *Client) ListPipelineJobs(projectName string, pipelineID int) (jobs []schemas.Job, err error) { - var foundJobs []*goGitlab.Job - var resp *goGitlab.Response +func (c *Client) ListPipelineJobs(ctx context.Context, projectName string, pipelineID int) (jobs []schemas.Job, err error) { + var ( + foundJobs []*goGitlab.Job + resp *goGitlab.Response + ) options := &goGitlab.ListJobsOptions{ ListOptions: goGitlab.ListOptions{ @@ -51,11 +56,13 @@ func (c *Client) ListPipelineJobs(projectName string, pipelineID int) (jobs []sc } for { - c.rateLimit() - foundJobs, resp, err = c.Jobs.ListPipelineJobs(projectName, pipelineID, options) + c.rateLimit(ctx) + + foundJobs, resp, err = c.Jobs.ListPipelineJobs(projectName, pipelineID, options, goGitlab.WithContext(ctx)) if err != nil { return } + c.requestsRemaining(resp) for _, job := range foundJobs { @@ -70,18 +77,22 @@ func (c *Client) ListPipelineJobs(projectName string, pipelineID int) (jobs []sc "jobs-count": resp.TotalItems, }, ).Debug("found pipeline jobs") + break } options.Page = resp.NextPage } + return } // ListPipelineBridges .. -func (c *Client) ListPipelineBridges(projectName string, pipelineID int) (bridges []*goGitlab.Bridge, err error) { - var foundBridges []*goGitlab.Bridge - var resp *goGitlab.Response +func (c *Client) ListPipelineBridges(ctx context.Context, projectName string, pipelineID int) (bridges []*goGitlab.Bridge, err error) { + var ( + foundBridges []*goGitlab.Bridge + resp *goGitlab.Response + ) options := &goGitlab.ListJobsOptions{ ListOptions: goGitlab.ListOptions{ @@ -91,11 +102,13 @@ func (c *Client) ListPipelineBridges(projectName string, pipelineID int) (bridge } for { - c.rateLimit() - foundBridges, resp, err = c.Jobs.ListPipelineBridges(projectName, pipelineID, options) + c.rateLimit(ctx) + + foundBridges, resp, err = c.Jobs.ListPipelineBridges(projectName, pipelineID, options, goGitlab.WithContext(ctx)) if err != nil { return } + c.requestsRemaining(resp) bridges = append(bridges, foundBridges...) @@ -108,16 +121,18 @@ func (c *Client) ListPipelineBridges(projectName string, pipelineID int) (bridge "bridges-count": resp.TotalItems, }, ).Debug("found pipeline bridges") + break } options.Page = resp.NextPage } + return } // ListPipelineChildJobs .. -func (c *Client) ListPipelineChildJobs(projectName string, parentPipelineID int) (jobs []schemas.Job, err error) { +func (c *Client) ListPipelineChildJobs(ctx context.Context, projectName string, parentPipelineID int) (jobs []schemas.Job, err error) { pipelineIDs := []int{parentPipelineID} for { @@ -129,7 +144,8 @@ func (c *Client) ListPipelineChildJobs(projectName string, parentPipelineID int) pipelineIDs = pipelineIDs[:len(pipelineIDs)-1] var foundBridges []*goGitlab.Bridge - foundBridges, err = c.ListPipelineBridges(projectName, pipelineID) + + foundBridges, err = c.ListPipelineBridges(ctx, projectName, pipelineID) if err != nil { return } @@ -143,8 +159,10 @@ func (c *Client) ListPipelineChildJobs(projectName string, parentPipelineID int) } pipelineIDs = append(pipelineIDs, foundBridge.DownstreamPipeline.ID) + var foundJobs []schemas.Job - foundJobs, err = c.ListPipelineJobs(projectName, foundBridge.DownstreamPipeline.ID) + + foundJobs, err = c.ListPipelineJobs(ctx, projectName, foundBridge.DownstreamPipeline.ID) if err != nil { return } @@ -155,7 +173,7 @@ func (c *Client) ListPipelineChildJobs(projectName string, parentPipelineID int) } // ListRefMostRecentJobs .. -func (c *Client) ListRefMostRecentJobs(ref schemas.Ref) (jobs []schemas.Job, err error) { +func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jobs []schemas.Job, err error) { if len(ref.LatestJobs) == 0 { log.WithFields( log.Fields{ @@ -163,6 +181,7 @@ func (c *Client) ListRefMostRecentJobs(ref schemas.Ref) (jobs []schemas.Job, err "ref": ref.Name, }, ).Debug("no jobs are currently held in memory, exiting..") + return } @@ -172,8 +191,10 @@ func (c *Client) ListRefMostRecentJobs(ref schemas.Ref) (jobs []schemas.Job, err jobsToRefresh[k] = v } - var foundJobs []*goGitlab.Job - var resp *goGitlab.Response + var ( + foundJobs []*goGitlab.Job + resp *goGitlab.Response + ) options := &goGitlab.ListJobsOptions{ ListOptions: goGitlab.ListOptions{ @@ -183,11 +204,13 @@ func (c *Client) ListRefMostRecentJobs(ref schemas.Ref) (jobs []schemas.Job, err } for { - c.rateLimit() - foundJobs, resp, err = c.Jobs.ListProjectJobs(ref.Project.Name, options) + c.rateLimit(ctx) + + foundJobs, resp, err = c.Jobs.ListProjectJobs(ref.Project.Name, options, goGitlab.WithContext(ctx)) if err != nil { return } + c.requestsRemaining(resp) for _, job := range foundJobs { @@ -207,12 +230,14 @@ func (c *Client) ListRefMostRecentJobs(ref schemas.Ref) (jobs []schemas.Job, err "jobs-count": len(ref.LatestJobs), }, ).Debug("found all jobs to refresh") + return } } if resp.CurrentPage >= resp.NextPage { var notFoundJobs []string + for k := range jobsToRefresh { notFoundJobs = append(notFoundJobs, k) } @@ -225,10 +250,12 @@ func (c *Client) ListRefMostRecentJobs(ref schemas.Ref) (jobs []schemas.Job, err "not-found-jobs": strings.Join(notFoundJobs, ","), }, ).Warn("found some ref jobs but did not manage to refresh all jobs which were in memory") + break } options.Page = resp.NextPage } + return } diff --git a/pkg/gitlab/jobs_test.go b/pkg/gitlab/jobs_test.go index 15c2f562..122d55ee 100644 --- a/pkg/gitlab/jobs_test.go +++ b/pkg/gitlab/jobs_test.go @@ -11,7 +11,7 @@ import ( ) func TestListRefPipelineJobs(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() ref := schemas.Ref{ @@ -20,7 +20,7 @@ func TestListRefPipelineJobs(t *testing.T) { } // Test with no most recent pipeline defined - jobs, err := c.ListRefPipelineJobs(ref) + jobs, err := c.ListRefPipelineJobs(ctx, ref) assert.NoError(t, err) assert.Len(t, jobs, 0) @@ -58,7 +58,7 @@ func TestListRefPipelineJobs(t *testing.T) { ID: 1, } - jobs, err = c.ListRefPipelineJobs(ref) + jobs, err = c.ListRefPipelineJobs(ctx, ref) assert.NoError(t, err) assert.Equal(t, []schemas.Job{ {ID: 10}, @@ -68,12 +68,12 @@ func TestListRefPipelineJobs(t *testing.T) { // Test invalid project id ref.Project.Name = "bar" - _, err = c.ListRefPipelineJobs(ref) + _, err = c.ListRefPipelineJobs(ctx, ref) assert.Error(t, err) } func TestListPipelineJobs(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() mux.HandleFunc("/api/v4/projects/foo/pipelines/1/jobs", @@ -92,17 +92,17 @@ func TestListPipelineJobs(t *testing.T) { w.WriteHeader(http.StatusNotFound) }) - jobs, err := c.ListPipelineJobs("foo", 1) + jobs, err := c.ListPipelineJobs(ctx, "foo", 1) assert.NoError(t, err) assert.Len(t, jobs, 2) // Test invalid project id - _, err = c.ListPipelineJobs("bar", 1) + _, err = c.ListPipelineJobs(ctx, "bar", 1) assert.Error(t, err) } func TestListPipelineBridges(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() mux.HandleFunc("/api/v4/projects/foo/pipelines/1/bridges", @@ -121,17 +121,17 @@ func TestListPipelineBridges(t *testing.T) { w.WriteHeader(http.StatusNotFound) }) - bridges, err := c.ListPipelineBridges("foo", 1) + bridges, err := c.ListPipelineBridges(ctx, "foo", 1) assert.NoError(t, err) assert.Len(t, bridges, 1) // Test invalid project id - _, err = c.ListPipelineBridges("bar", 1) + _, err = c.ListPipelineBridges(ctx, "bar", 1) assert.Error(t, err) } func TestListRefMostRecentJobs(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() ref := schemas.Ref{ @@ -139,7 +139,7 @@ func TestListRefMostRecentJobs(t *testing.T) { Name: "yay", } - jobs, err := c.ListRefMostRecentJobs(ref) + jobs, err := c.ListRefMostRecentJobs(ctx, ref) assert.NoError(t, err) assert.Len(t, jobs, 0) @@ -170,7 +170,7 @@ func TestListRefMostRecentJobs(t *testing.T) { }, } - jobs, err = c.ListRefMostRecentJobs(ref) + jobs, err = c.ListRefMostRecentJobs(ctx, ref) assert.NoError(t, err) assert.Len(t, jobs, 2) assert.Equal(t, 3, jobs[0].ID) @@ -181,7 +181,7 @@ func TestListRefMostRecentJobs(t *testing.T) { Name: "baz", } - jobs, err = c.ListRefMostRecentJobs(ref) + jobs, err = c.ListRefMostRecentJobs(ctx, ref) assert.NoError(t, err) assert.Len(t, jobs, 2) assert.Equal(t, 3, jobs[0].ID) @@ -189,6 +189,6 @@ func TestListRefMostRecentJobs(t *testing.T) { // Test invalid project id ref.Project.Name = "bar" - _, err = c.ListRefMostRecentJobs(ref) + _, err = c.ListRefMostRecentJobs(ctx, ref) assert.Error(t, err) } diff --git a/pkg/gitlab/pipelines.go b/pkg/gitlab/pipelines.go index 02ee466a..b25694a2 100644 --- a/pkg/gitlab/pipelines.go +++ b/pkg/gitlab/pipelines.go @@ -1,6 +1,7 @@ package gitlab import ( + "context" "fmt" "regexp" "strings" @@ -12,19 +13,29 @@ import ( ) // GetRefPipeline .. -func (c *Client) GetRefPipeline(ref schemas.Ref, pipelineID int) (p schemas.Pipeline, err error) { - c.rateLimit() - gp, resp, err := c.Pipelines.GetPipeline(ref.Project.Name, pipelineID) +func (c *Client) GetRefPipeline(ctx context.Context, ref schemas.Ref, pipelineID int) (p schemas.Pipeline, err error) { + c.rateLimit(ctx) + + gp, resp, err := c.Pipelines.GetPipeline(ref.Project.Name, pipelineID, goGitlab.WithContext(ctx)) if err != nil || gp == nil { return schemas.Pipeline{}, fmt.Errorf("could not read content of pipeline %s - %s | %s", ref.Project.Name, ref.Name, err.Error()) } + c.requestsRemaining(resp) return schemas.NewPipeline(*gp), nil } // GetProjectPipelines .. -func (c *Client) GetProjectPipelines(projectName string, options *goGitlab.ListProjectPipelinesOptions) ([]*goGitlab.PipelineInfo, *goGitlab.Response, error) { +func (c *Client) GetProjectPipelines( + ctx context.Context, + projectName string, + options *goGitlab.ListProjectPipelinesOptions, +) ( + []*goGitlab.PipelineInfo, + *goGitlab.Response, + error, +) { fields := log.Fields{ "project-name": projectName, } @@ -48,18 +59,20 @@ func (c *Client) GetProjectPipelines(projectName string, options *goGitlab.ListP fields["page"] = options.Page log.WithFields(fields).Trace("listing project pipelines") - c.rateLimit() - pipelines, resp, err := c.Pipelines.ListProjectPipelines(projectName, options) + c.rateLimit(ctx) + + pipelines, resp, err := c.Pipelines.ListProjectPipelines(projectName, options, goGitlab.WithContext(ctx)) if err != nil { return nil, resp, fmt.Errorf("error listing project pipelines for project %s: %s", projectName, err.Error()) } + c.requestsRemaining(resp) return pipelines, resp, nil } // GetRefPipelineVariablesAsConcatenatedString .. -func (c *Client) GetRefPipelineVariablesAsConcatenatedString(ref schemas.Ref) (string, error) { +func (c *Client) GetRefPipelineVariablesAsConcatenatedString(ctx context.Context, ref schemas.Ref) (string, error) { if ref.LatestPipeline == (schemas.Pipeline{}) { log.WithFields( log.Fields{ @@ -67,6 +80,7 @@ func (c *Client) GetRefPipelineVariablesAsConcatenatedString(ref schemas.Ref) (s "ref": ref.Name, }, ).Debug("most recent pipeline not defined, exiting..") + return "", nil } @@ -80,17 +94,24 @@ func (c *Client) GetRefPipelineVariablesAsConcatenatedString(ref schemas.Ref) (s variablesFilter, err := regexp.Compile(ref.Project.Pull.Pipeline.Variables.Regexp) if err != nil { - return "", fmt.Errorf("the provided filter regex for pipeline variables is invalid '(%s)': %v", ref.Project.Pull.Pipeline.Variables.Regexp, err) + return "", fmt.Errorf( + "the provided filter regex for pipeline variables is invalid '(%s)': %v", + ref.Project.Pull.Pipeline.Variables.Regexp, + err, + ) } - c.rateLimit() - variables, resp, err := c.Pipelines.GetPipelineVariables(ref.Project.Name, ref.LatestPipeline.ID) + c.rateLimit(ctx) + + variables, resp, err := c.Pipelines.GetPipelineVariables(ref.Project.Name, ref.LatestPipeline.ID, goGitlab.WithContext(ctx)) if err != nil { return "", fmt.Errorf("could not fetch pipeline variables for %d: %s", ref.LatestPipeline.ID, err.Error()) } + c.requestsRemaining(resp) var keptVariables []string + if len(variables) > 0 { for _, v := range variables { if variablesFilter.MatchString(v.Key) { @@ -103,7 +124,7 @@ func (c *Client) GetRefPipelineVariablesAsConcatenatedString(ref schemas.Ref) (s } // GetRefsFromPipelines .. -func (c *Client) GetRefsFromPipelines(p schemas.Project, refKind schemas.RefKind) (refs schemas.Refs, err error) { +func (c *Client) GetRefsFromPipelines(ctx context.Context, p schemas.Project, refKind schemas.RefKind) (refs schemas.Refs, err error) { refs = make(schemas.Refs) options := &goGitlab.ListProjectPipelinesOptions{ @@ -115,13 +136,17 @@ func (c *Client) GetRefsFromPipelines(p schemas.Project, refKind schemas.RefKind } var re *regexp.Regexp + if re, err = schemas.GetRefRegexp(p.Pull.Refs, refKind); err != nil { return } - var mostRecent, maxAgeSeconds uint - var limitToMostRecent, excludeDeleted bool - var existingRefs schemas.Refs + var ( + mostRecent, maxAgeSeconds uint + limitToMostRecent, excludeDeleted bool + existingRefs schemas.Refs + ) + switch refKind { case schemas.RefKindMergeRequest: maxAgeSeconds = p.Pull.Refs.MergeRequests.MaxAgeSeconds @@ -130,9 +155,11 @@ func (c *Client) GetRefsFromPipelines(p schemas.Project, refKind schemas.RefKind options.Scope = goGitlab.String("branches") maxAgeSeconds = p.Pull.Refs.Branches.MaxAgeSeconds mostRecent = p.Pull.Refs.Branches.MostRecent + if p.Pull.Refs.Branches.ExcludeDeleted { excludeDeleted = true - if existingRefs, err = c.GetProjectBranches(p); err != nil { + + if existingRefs, err = c.GetProjectBranches(ctx, p); err != nil { return } } @@ -140,9 +167,11 @@ func (c *Client) GetRefsFromPipelines(p schemas.Project, refKind schemas.RefKind options.Scope = goGitlab.String("tags") maxAgeSeconds = p.Pull.Refs.Tags.MaxAgeSeconds mostRecent = p.Pull.Refs.Tags.MostRecent + if p.Pull.Refs.Tags.ExcludeDeleted { excludeDeleted = true - if existingRefs, err = c.GetProjectTags(p); err != nil { + + if existingRefs, err = c.GetProjectTags(ctx, p); err != nil { return } } @@ -160,30 +189,35 @@ func (c *Client) GetRefsFromPipelines(p schemas.Project, refKind schemas.RefKind } for { - var pipelines []*goGitlab.PipelineInfo - var resp *goGitlab.Response - pipelines, resp, err = c.GetProjectPipelines(p.Name, options) + var ( + pipelines []*goGitlab.PipelineInfo + resp *goGitlab.Response + ) + + pipelines, resp, err = c.GetProjectPipelines(ctx, p.Name, options) if err != nil { return } for _, pipeline := range pipelines { refName := pipeline.Ref - if re.MatchString(refName) { - if refKind == schemas.RefKindMergeRequest { - if refName, err = schemas.GetMergeRequestIIDFromRefName(refName); err != nil { - log.WithField("ref", refName).WithError(err).Warn() - continue - } - } - } else { + if !re.MatchString(refName) { // It is quite verbose otherwise.. if refKind != schemas.RefKindMergeRequest { log.WithField("ref", refName).Debug("discovered pipeline ref not matching regexp") } + continue } + if refKind == schemas.RefKindMergeRequest { + if refName, err = schemas.GetMergeRequestIIDFromRefName(refName); err != nil { + log.WithField("ref", refName).WithError(err).Warn() + + continue + } + } + ref := schemas.NewRef( p, refKind, @@ -197,6 +231,7 @@ func (c *Client) GetRefsFromPipelines(p schemas.Project, refKind schemas.RefKind "ref": ref.Name, "ref-kind": ref.Kind, }).Debug("found deleted ref, ignoring..") + continue } } @@ -207,6 +242,7 @@ func (c *Client) GetRefsFromPipelines(p schemas.Project, refKind schemas.RefKind "ref": ref.Name, "ref-kind": ref.Kind, }).Trace("found ref") + refs[ref.Key()] = ref if limitToMostRecent { @@ -221,6 +257,7 @@ func (c *Client) GetRefsFromPipelines(p schemas.Project, refKind schemas.RefKind if resp.CurrentPage >= resp.NextPage { break } + options.Page = resp.NextPage } diff --git a/pkg/gitlab/pipelines_test.go b/pkg/gitlab/pipelines_test.go index a5e81eb6..25b389a2 100644 --- a/pkg/gitlab/pipelines_test.go +++ b/pkg/gitlab/pipelines_test.go @@ -14,7 +14,7 @@ import ( ) func TestGetRefPipeline(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() mux.HandleFunc("/api/v4/projects/foo/pipelines/1", @@ -28,14 +28,14 @@ func TestGetRefPipeline(t *testing.T) { Name: "yay", } - pipeline, err := c.GetRefPipeline(ref, 1) + pipeline, err := c.GetRefPipeline(ctx, ref, 1) assert.NoError(t, err) assert.NotNil(t, pipeline) assert.Equal(t, 1, pipeline.ID) } func TestGetProjectPipelines(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/pipelines"), @@ -51,7 +51,7 @@ func TestGetProjectPipelines(t *testing.T) { fmt.Fprint(w, `[{"id":1},{"id":2}]`) }) - pipelines, _, err := c.GetProjectPipelines("foo", &gitlab.ListProjectPipelinesOptions{ + pipelines, _, err := c.GetProjectPipelines(ctx, "foo", &gitlab.ListProjectPipelinesOptions{ Ref: pointy.String("foo"), Scope: pointy.String("bar"), }) @@ -61,7 +61,7 @@ func TestGetProjectPipelines(t *testing.T) { } func TestGetRefPipelineVariablesAsConcatenatedString(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() mux.HandleFunc("/api/v4/projects/foo/pipelines/1/variables", @@ -79,7 +79,7 @@ func TestGetRefPipelineVariablesAsConcatenatedString(t *testing.T) { } // Should return right away as MostRecentPipeline is not defined - variables, err := c.GetRefPipelineVariablesAsConcatenatedString(ref) + variables, err := c.GetRefPipelineVariablesAsConcatenatedString(ctx, ref) assert.NoError(t, err) assert.Equal(t, "", variables) @@ -88,26 +88,27 @@ func TestGetRefPipelineVariablesAsConcatenatedString(t *testing.T) { } // Should fail as we have an invalid regexp pattern - variables, err = c.GetRefPipelineVariablesAsConcatenatedString(ref) + variables, err = c.GetRefPipelineVariablesAsConcatenatedString(ctx, ref) assert.Error(t, err) assert.Contains(t, err.Error(), "the provided filter regex for pipeline variables is invalid") assert.Equal(t, "", variables) // Should work ref.Project.Pull.Pipeline.Variables.Regexp = `.*` - variables, err = c.GetRefPipelineVariablesAsConcatenatedString(ref) + variables, err = c.GetRefPipelineVariablesAsConcatenatedString(ctx, ref) assert.NoError(t, err) assert.Equal(t, "foo:bar,bar:baz", variables) } func TestGetRefsFromPipelines(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() log.SetLevel(log.TraceLevel) mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/repository/branches"), func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `[{"name":"keep_main"}]`) + return }) @@ -120,11 +121,13 @@ func TestGetRefsFromPipelines(t *testing.T) { if scope, ok := urlValues["scope"]; ok && len(scope) == 1 && scope[0] == "branches" { fmt.Fprint(w, `[{"id":1,"ref":"keep_dev"},{"id":2,"ref":"keep_main"}]`) + return } if scope, ok := urlValues["scope"]; ok && len(scope) == 1 && scope[0] == "tags" { fmt.Fprint(w, `[{"id":3,"ref":"donotkeep_0.0.1"},{"id":4,"ref":"keep_0.0.2"}]`) + return } @@ -135,13 +138,13 @@ func TestGetRefsFromPipelines(t *testing.T) { // Branches p.Pull.Refs.Branches.Regexp = `[` // invalid regexp pattern - refs, err := c.GetRefsFromPipelines(p, schemas.RefKindBranch) + refs, err := c.GetRefsFromPipelines(ctx, p, schemas.RefKindBranch) assert.Error(t, err) assert.Contains(t, err.Error(), "error parsing regexp") assert.Len(t, refs, 0) p.Pull.Refs.Branches.Regexp = "^keep.*" - refs, err = c.GetRefsFromPipelines(p, schemas.RefKindBranch) + refs, err = c.GetRefsFromPipelines(ctx, p, schemas.RefKindBranch) assert.NoError(t, err) assert.Equal(t, schemas.Refs{ @@ -150,14 +153,14 @@ func TestGetRefsFromPipelines(t *testing.T) { // Tags p.Pull.Refs.Tags.Regexp = `[` // invalid regexp pattern - refs, err = c.GetRefsFromPipelines(p, schemas.RefKindTag) + refs, err = c.GetRefsFromPipelines(ctx, p, schemas.RefKindTag) assert.Error(t, err) assert.Contains(t, err.Error(), "error parsing regexp") assert.Len(t, refs, 0) p.Pull.Refs.Tags.Regexp = `^keep` p.Pull.Refs.Tags.ExcludeDeleted = false - refs, err = c.GetRefsFromPipelines(p, schemas.RefKindTag) + refs, err = c.GetRefsFromPipelines(ctx, p, schemas.RefKindTag) assert.NoError(t, err) assert.Equal(t, schemas.Refs{ @@ -165,7 +168,7 @@ func TestGetRefsFromPipelines(t *testing.T) { }, refs) // Merge requests - refs, err = c.GetRefsFromPipelines(p, schemas.RefKindMergeRequest) + refs, err = c.GetRefsFromPipelines(ctx, p, schemas.RefKindMergeRequest) assert.NoError(t, err) assert.Equal(t, schemas.Refs{ "622996356": schemas.NewRef(p, schemas.RefKindMergeRequest, "1234"), diff --git a/pkg/gitlab/projects.go b/pkg/gitlab/projects.go index 7bc2ca62..644d32cd 100644 --- a/pkg/gitlab/projects.go +++ b/pkg/gitlab/projects.go @@ -1,6 +1,7 @@ package gitlab import ( + "context" "fmt" "regexp" @@ -8,25 +9,24 @@ import ( "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" "github.com/openlyinc/pointy" log "github.com/sirupsen/logrus" - "github.com/xanzy/go-gitlab" goGitlab "github.com/xanzy/go-gitlab" ) // GetProject .. -func (c *Client) GetProject(name string) (*goGitlab.Project, error) { +func (c *Client) GetProject(ctx context.Context, name string) (*goGitlab.Project, error) { log.WithFields(log.Fields{ "project-name": name, }).Debug("reading project") - c.rateLimit() - p, resp, err := c.Projects.GetProject(name, &goGitlab.GetProjectOptions{}) + c.rateLimit(ctx) + p, resp, err := c.Projects.GetProject(name, &goGitlab.GetProjectOptions{}, goGitlab.WithContext(ctx)) c.requestsRemaining(resp) return p, err } // ListProjects .. -func (c *Client) ListProjects(w config.Wildcard) ([]schemas.Project, error) { +func (c *Client) ListProjects(ctx context.Context, w config.Wildcard) ([]schemas.Project, error) { logFields := log.Fields{ "wildcard-search": w.Search, "wildcard-owner-kind": w.Owner.Kind, @@ -37,7 +37,8 @@ func (c *Client) ListProjects(w config.Wildcard) ([]schemas.Project, error) { log.WithFields(logFields).Debug("listing all projects from wildcard") var projects []schemas.Project - listOptions := gitlab.ListOptions{ + + listOptions := goGitlab.ListOptions{ Page: 1, PerPage: 100, } @@ -47,6 +48,7 @@ func (c *Client) ListProjects(w config.Wildcard) ([]schemas.Project, error) { // scoped wildcard. Therefore, if the wildcard owner name is set, we want to filter // out to project actually *belonging* to the owner. var ownerRegexp *regexp.Regexp + if len(w.Owner.Name) > 0 { ownerRegexp = regexp.MustCompile(fmt.Sprintf(`^%s\/`, w.Owner.Name)) } else { @@ -54,46 +56,53 @@ func (c *Client) ListProjects(w config.Wildcard) ([]schemas.Project, error) { } for { - var gps []*gitlab.Project - var resp *gitlab.Response - var err error + var ( + gps []*goGitlab.Project + resp *goGitlab.Response + err error + ) + + c.rateLimit(ctx) - c.rateLimit() switch w.Owner.Kind { case "user": gps, resp, err = c.Projects.ListUserProjects( w.Owner.Name, - &gitlab.ListProjectsOptions{ + &goGitlab.ListProjectsOptions{ Archived: &w.Archived, ListOptions: listOptions, Search: &w.Search, }, + goGitlab.WithContext(ctx), ) case "group": gps, resp, err = c.Groups.ListGroupProjects( w.Owner.Name, - &gitlab.ListGroupProjectsOptions{ + &goGitlab.ListGroupProjectsOptions{ Archived: &w.Archived, WithShared: pointy.Bool(false), IncludeSubGroups: &w.Owner.IncludeSubgroups, ListOptions: listOptions, Search: &w.Search, }, + goGitlab.WithContext(ctx), ) default: // List all visible projects gps, resp, err = c.Projects.ListProjects( - &gitlab.ListProjectsOptions{ + &goGitlab.ListProjectsOptions{ ListOptions: listOptions, Archived: &w.Archived, Search: &w.Search, }, + goGitlab.WithContext(ctx), ) } if err != nil { return projects, fmt.Errorf("unable to list projects with search pattern '%s' from the GitLab API : %v", w.Search, err.Error()) } + c.requestsRemaining(resp) // Copy relevant settings from wildcard into created project @@ -103,6 +112,7 @@ func (c *Client) ListProjects(w config.Wildcard) ([]schemas.Project, error) { "project-id": gp.ID, "project-name": gp.PathWithNamespace, }).Debug("project path not matching owner's name, skipping") + continue } @@ -111,6 +121,7 @@ func (c *Client) ListProjects(w config.Wildcard) ([]schemas.Project, error) { "project-id": gp.ID, "project-name": gp.PathWithNamespace, }).Debug("jobs/pipelines not enabled on project, skipping") + continue } diff --git a/pkg/gitlab/projects_test.go b/pkg/gitlab/projects_test.go index 92577491..14b56d1d 100644 --- a/pkg/gitlab/projects_test.go +++ b/pkg/gitlab/projects_test.go @@ -10,7 +10,7 @@ import ( ) func TestGetProject(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() project := "foo/bar" @@ -20,14 +20,14 @@ func TestGetProject(t *testing.T) { fmt.Fprint(w, `{"id":1}`) }) - p, err := c.GetProject(project) + p, err := c.GetProject(ctx, project) assert.NoError(t, err) assert.NotNil(t, p) assert.Equal(t, 1, p.ID) } func TestListUserProjects(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() w := config.Wildcard{ @@ -46,14 +46,14 @@ func TestListUserProjects(t *testing.T) { fmt.Fprint(w, `[{"id":1,"path_with_namespace":"foo/bar","jobs_enabled":true},{"id":2,"path_with_namespace":"bar/baz","jobs_enabled":true}]`) }) - projects, err := c.ListProjects(w) + projects, err := c.ListProjects(ctx, w) assert.NoError(t, err) assert.Len(t, projects, 1) assert.Equal(t, "foo/bar", projects[0].Name) } func TestListGroupProjects(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() w := config.Wildcard{ @@ -72,14 +72,14 @@ func TestListGroupProjects(t *testing.T) { fmt.Fprint(w, `[{"id":1,"path_with_namespace":"foo/bar","jobs_enabled":true},{"id":2,"path_with_namespace":"bar/baz","jobs_enabled":true}]`) }) - projects, err := c.ListProjects(w) + projects, err := c.ListProjects(ctx, w) assert.NoError(t, err) assert.Len(t, projects, 1) assert.Equal(t, "foo/bar", projects[0].Name) } func TestListProjects(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() w := config.Wildcard{ @@ -98,14 +98,14 @@ func TestListProjects(t *testing.T) { fmt.Fprint(w, `[{"id":1,"path_with_namespace":"foo","jobs_enabled":false},{"id":2,"path_with_namespace":"bar","jobs_enabled":true}]`) }) - projects, err := c.ListProjects(w) + projects, err := c.ListProjects(ctx, w) assert.NoError(t, err) assert.Len(t, projects, 1) assert.Equal(t, "bar", projects[0].Name) } func TestListProjectsAPIError(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() w := config.Wildcard{ @@ -120,10 +120,10 @@ func TestListProjectsAPIError(t *testing.T) { mux.HandleFunc(fmt.Sprintf("/api/v4/users/%s/projects", w.Owner.Name), func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("500 - Something bad happened!")) + _, _ = w.Write([]byte("500 - Something bad happened!")) }) - _, err := c.ListProjects(w) + _, err := c.ListProjects(ctx, w) assert.Error(t, err) assert.Contains(t, err.Error(), "unable to list projects with search pattern") } diff --git a/pkg/gitlab/repositories.go b/pkg/gitlab/repositories.go index 79d873e2..6b0ebf51 100644 --- a/pkg/gitlab/repositories.go +++ b/pkg/gitlab/repositories.go @@ -1,6 +1,7 @@ package gitlab import ( + "context" "fmt" "github.com/openlyinc/pointy" @@ -9,22 +10,24 @@ import ( ) // GetCommitCountBetweenRefs .. -func (c *Client) GetCommitCountBetweenRefs(project, from, to string) (int, error) { +func (c *Client) GetCommitCountBetweenRefs(ctx context.Context, project, from, to string) (int, error) { log.WithFields(log.Fields{ "project-name": project, "from-ref": from, "to-ref": to, }).Debug("comparing refs") - c.rateLimit() + c.rateLimit(ctx) + cmp, resp, err := c.Repositories.Compare(project, &goGitlab.CompareOptions{ From: &from, To: &to, Straight: pointy.Bool(true), - }, nil) + }, goGitlab.WithContext(ctx)) if err != nil { return 0, err } + c.requestsRemaining(resp) if cmp == nil { diff --git a/pkg/gitlab/repositories_test.go b/pkg/gitlab/repositories_test.go index 296cef60..dbed1085 100644 --- a/pkg/gitlab/repositories_test.go +++ b/pkg/gitlab/repositories_test.go @@ -9,7 +9,7 @@ import ( ) func TestGetCommitCountBetweenRefs(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() mux.HandleFunc("/api/v4/projects/foo/repository/compare", @@ -23,11 +23,11 @@ func TestGetCommitCountBetweenRefs(t *testing.T) { fmt.Fprint(w, `{`) }) - commitCount, err := c.GetCommitCountBetweenRefs("foo", "bar", "baz") + commitCount, err := c.GetCommitCountBetweenRefs(ctx, "foo", "bar", "baz") assert.NoError(t, err) assert.Equal(t, 3, commitCount) - commitCount, err = c.GetCommitCountBetweenRefs("bar", "", "") + commitCount, err = c.GetCommitCountBetweenRefs(ctx, "bar", "", "") assert.Error(t, err) assert.Equal(t, 0, commitCount) } diff --git a/pkg/gitlab/tags.go b/pkg/gitlab/tags.go index ef443c05..2130cdd1 100644 --- a/pkg/gitlab/tags.go +++ b/pkg/gitlab/tags.go @@ -1,6 +1,7 @@ package gitlab import ( + "context" "regexp" "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" @@ -8,7 +9,7 @@ import ( ) // GetProjectTags .. -func (c *Client) GetProjectTags(p schemas.Project) ( +func (c *Client) GetProjectTags(ctx context.Context, p schemas.Project) ( refs schemas.Refs, err error, ) { @@ -22,18 +23,24 @@ func (c *Client) GetProjectTags(p schemas.Project) ( } var re *regexp.Regexp + if re, err = regexp.Compile(p.Pull.Refs.Tags.Regexp); err != nil { return } for { - c.rateLimit() - var tags []*goGitlab.Tag - var resp *goGitlab.Response - tags, resp, err = c.Tags.ListTags(p.Name, options) + c.rateLimit(ctx) + + var ( + tags []*goGitlab.Tag + resp *goGitlab.Response + ) + + tags, resp, err = c.Tags.ListTags(p.Name, options, goGitlab.WithContext(ctx)) if err != nil { return } + c.requestsRemaining(resp) for _, tag := range tags { @@ -46,6 +53,7 @@ func (c *Client) GetProjectTags(p schemas.Project) ( if resp.CurrentPage >= resp.NextPage { break } + options.Page = resp.NextPage } @@ -53,7 +61,7 @@ func (c *Client) GetProjectTags(p schemas.Project) ( } // GetProjectMostRecentTagCommit .. -func (c *Client) GetProjectMostRecentTagCommit(projectName, filterRegexp string) (string, float64, error) { +func (c *Client) GetProjectMostRecentTagCommit(ctx context.Context, projectName, filterRegexp string) (string, float64, error) { options := &goGitlab.ListTagsOptions{ ListOptions: goGitlab.ListOptions{ Page: 1, @@ -67,11 +75,13 @@ func (c *Client) GetProjectMostRecentTagCommit(projectName, filterRegexp string) } for { - c.rateLimit() - tags, resp, err := c.Tags.ListTags(projectName, options) + c.rateLimit(ctx) + + tags, resp, err := c.Tags.ListTags(projectName, options, goGitlab.WithContext(ctx)) if err != nil { return "", 0, err } + c.requestsRemaining(resp) for _, tag := range tags { @@ -83,6 +93,7 @@ func (c *Client) GetProjectMostRecentTagCommit(projectName, filterRegexp string) if resp.CurrentPage >= resp.NextPage { break } + options.Page = resp.NextPage } diff --git a/pkg/gitlab/tags_test.go b/pkg/gitlab/tags_test.go index 78bd0c7b..e894b279 100644 --- a/pkg/gitlab/tags_test.go +++ b/pkg/gitlab/tags_test.go @@ -11,7 +11,7 @@ import ( ) func TestGetProjectTags(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() mux.HandleFunc("/api/v4/projects/foo/repository/tags", @@ -29,7 +29,7 @@ func TestGetProjectTags(t *testing.T) { p.Pull.Refs.Tags.Regexp = `^f` expectedRef := schemas.NewRef(p, schemas.RefKindTag, "foo") - refs, err := c.GetProjectTags(p) + refs, err := c.GetProjectTags(ctx, p) assert.NoError(t, err) assert.Len(t, refs, 1) assert.Equal(t, schemas.Refs{ @@ -38,18 +38,18 @@ func TestGetProjectTags(t *testing.T) { // Test invalid project name p.Name = "invalid" - _, err = c.GetProjectTags(p) + _, err = c.GetProjectTags(ctx, p) assert.Error(t, err) // Test invalid regexp p.Name = "foo" p.Pull.Refs.Tags.Regexp = `[` - _, err = c.GetProjectTags(p) + _, err = c.GetProjectTags(ctx, p) assert.Error(t, err) } func TestGetProjectMostRecentTagCommit(t *testing.T) { - mux, server, c := getMockedClient() + ctx, mux, server, c := getMockedClient() defer server.Close() mux.HandleFunc(fmt.Sprintf("/api/v4/projects/foo/repository/tags"), @@ -75,11 +75,11 @@ func TestGetProjectMostRecentTagCommit(t *testing.T) { ]`) }) - _, _, err := c.GetProjectMostRecentTagCommit("foo", "[") + _, _, err := c.GetProjectMostRecentTagCommit(ctx, "foo", "[") assert.Error(t, err) assert.Contains(t, err.Error(), "error parsing regexp") - commitShortID, commitCreatedAt, err := c.GetProjectMostRecentTagCommit("foo", "^f") + commitShortID, commitCreatedAt, err := c.GetProjectMostRecentTagCommit(ctx, "foo", "^f") assert.NoError(t, err) assert.Equal(t, "7b5c3cc", commitShortID) assert.Equal(t, float64(1553540113), commitCreatedAt) diff --git a/pkg/monitor/rpc/client.go b/pkg/monitor/rpc/client.go index 1d44652a..fbdc27b3 100644 --- a/pkg/monitor/rpc/client.go +++ b/pkg/monitor/rpc/client.go @@ -19,28 +19,31 @@ func NewClient(serverAddress *url.URL) (c *Client) { c = &Client{ serverAddress: serverAddress, } + var err error + c.Client, err = rpc.Dial(c.serverAddress.Scheme, c.serverAddress.Host) if err != nil { log.Fatal("dialing:", err) } + return } // Status .. func (c *Client) Status() (s monitor.Status) { - err := c.Call("Server.Status", "", &s) - if err != nil { + if err := c.Call("Server.Status", "", &s); err != nil { log.WithError(err).Fatal() } + return } // Config .. func (c *Client) Config() (s string) { - err := c.Call("Server.Config", "", &s) - if err != nil { + if err := c.Call("Server.Config", "", &s); err != nil { log.WithError(err).Fatal() } + return } diff --git a/pkg/monitor/rpc/server.go b/pkg/monitor/rpc/server.go index 6005a82a..13b6f031 100644 --- a/pkg/monitor/rpc/server.go +++ b/pkg/monitor/rpc/server.go @@ -1,6 +1,7 @@ package rpc import ( + "context" "net" "net/rpc" "net/url" @@ -36,6 +37,7 @@ func NewServer( store: st, taskSchedulingMonitoring: tsm, } + return } @@ -45,6 +47,7 @@ func ServeUNIX(r *Server) { r.cfg.Global.InternalMonitoringListenerAddress.Scheme == "" || r.cfg.Global.InternalMonitoringListenerAddress.Host == "" { log.Info("internal monitoring listener address not set") + return } @@ -83,6 +86,7 @@ func ServeUNIX(r *Server) { if err != nil { log.WithError(err).Fatal() } + go s.ServeConn(conn) } } @@ -90,11 +94,13 @@ func ServeUNIX(r *Server) { // Config .. func (r *Server) Config(_ string, reply *string) error { *reply = r.cfg.ToYAML() + return nil } // Status .. func (r *Server) Status(_ string, reply *monitor.Status) (err error) { + ctx := context.Background() s := monitor.Status{} s.GitLabAPIUsage = float64(r.gitlabClient.RateCounter.Rate()) / float64(r.cfg.Gitlab.MaximumRequestsPerSecond) @@ -112,33 +118,35 @@ func (r *Server) Status(_ string, reply *monitor.Status) (err error) { s.GitLabAPILimitRemaining = r.gitlabClient.RequestsRemaining var queuedTasks uint64 - queuedTasks, err = r.store.CurrentlyQueuedTasksCount() + + queuedTasks, err = r.store.CurrentlyQueuedTasksCount(ctx) if err != nil { return } s.TasksBufferUsage = float64(queuedTasks) / 1000 - s.TasksExecutedCount, err = r.store.ExecutedTasksCount() + + s.TasksExecutedCount, err = r.store.ExecutedTasksCount(ctx) if err != nil { return } - s.Projects.Count, err = r.store.ProjectsCount() + s.Projects.Count, err = r.store.ProjectsCount(ctx) if err != nil { return } - s.Envs.Count, err = r.store.EnvironmentsCount() + s.Envs.Count, err = r.store.EnvironmentsCount(ctx) if err != nil { return } - s.Refs.Count, err = r.store.RefsCount() + s.Refs.Count, err = r.store.RefsCount(ctx) if err != nil { return } - s.Metrics.Count, err = r.store.MetricsCount() + s.Metrics.Count, err = r.store.MetricsCount(ctx) if err != nil { return } @@ -184,5 +192,6 @@ func (r *Server) Status(_ string, reply *monitor.Status) (err error) { } *reply = s + return nil } diff --git a/pkg/monitor/ui/ui.go b/pkg/monitor/ui/ui.go index 52b230ca..883c3b30 100644 --- a/pkg/monitor/ui/ui.go +++ b/pkg/monitor/ui/ui.go @@ -15,19 +15,12 @@ import ( "github.com/charmbracelet/bubbles/viewport" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" - "github.com/muesli/termenv" "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/monitor" "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/monitor/rpc" log "github.com/sirupsen/logrus" "github.com/xeonx/timeago" ) -var ( - color = termenv.ColorProfile().Color - keyword = termenv.Style{}.Foreground(color("204")).Background(color("235")).Styled - help = termenv.Style{}.Foreground(color("241")).Styled -) - type tab string const ( @@ -43,12 +36,6 @@ var tabs = [...]tab{ var ( subtle = lipgloss.AdaptiveColor{Light: "#D9DCCF", Dark: "#383838"} highlight = lipgloss.AdaptiveColor{Light: "#874BFD", Dark: "#7D56F4"} - special = lipgloss.AdaptiveColor{Light: "#43BF6D", Dark: "#73F59F"} - divider = lipgloss.NewStyle(). - SetString("•"). - Padding(0, 1). - Foreground(subtle). - String() dataStyle = lipgloss.NewStyle(). MarginLeft(1). @@ -58,7 +45,7 @@ var ( Foreground(lipgloss.Color("#000000")). Background(lipgloss.Color("#a9a9a9")) - // Tabs + // Tabs. activeTabBorder = lipgloss.Border{ Top: "─", @@ -94,13 +81,13 @@ var ( BorderLeft(false). BorderRight(false) - // List + // List. entityStyle = lipgloss.NewStyle(). Border(lipgloss.NormalBorder(), true, false, false, false). BorderForeground(subtle) - // Status Bar + // Status Bar. statusStyle = lipgloss.NewStyle(). Inherit(statusBarStyle). @@ -122,7 +109,7 @@ var ( versionStyle = statusNugget.Copy(). Background(lipgloss.Color("#0062cc")) - // Page + // Page. docStyle = lipgloss.NewStyle() ) @@ -130,6 +117,7 @@ func max(a, b int) int { if a > b { return a } + return b } @@ -226,6 +214,7 @@ func prettyTimeago(t time.Time) string { if t.IsZero() { return "N/A" } + return timeago.English.Format(t) } @@ -241,6 +230,7 @@ func newModel(version string, listenerAddress *url.URL) (m *model) { progress: &p, rpcClient: rpcClient, } + return } @@ -258,6 +248,7 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.vp.Height = msg.Height - 4 m.progress.Width = msg.Width - 27 m.setPaneContent() + return m, nil case tea.KeyMsg: switch msg.Type { @@ -268,16 +259,19 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.tabID-- m.setPaneContent() } + return m, nil case tea.KeyRight: if m.tabID < len(tabs)-1 { m.tabID++ m.setPaneContent() } + return m, nil case tea.KeyUp, tea.KeyDown, tea.KeyPgDown, tea.KeyPgUp: vp, cmd := m.vp.Update(msg) m.vp = vp + return m, cmd } case monitor.Status: @@ -285,6 +279,7 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { if m.tabID == 0 { m.vp.SetContent(m.renderLastStatus()) } + return m, waitForActivity(m.sub) } @@ -300,6 +295,7 @@ func (m model) View() string { for tabID, t := range tabs { if m.tabID == tabID { renderedTabs = append(renderedTabs, activeTab.Render(string(t))) + continue } renderedTabs = append(renderedTabs, inactiveTab.Render(string(t))) @@ -311,12 +307,12 @@ func (m model) View() string { doc.WriteString(row + "\n") } - // PANE + // Pane. { doc.WriteString(m.vp.View() + "\n") } - // Status bar + // Status bar. { bar := lipgloss.JoinHorizontal(lipgloss.Top, statusStyle.Render("github.com/mvisonneau/gitlab-ci-pipelines-exporter"), @@ -364,7 +360,9 @@ func (m *model) setPaneContent() { m.vp.SetContent(m.renderLastStatus()) case tabConfig: var b bytes.Buffer + foo := bufio.NewWriter(&b) + if err := chromaQuick.Highlight(foo, m.rpcClient.Config(), "yaml", "terminal16m", "monokai"); err != nil { log.WithError(err).Fatal() } diff --git a/pkg/ratelimit/local.go b/pkg/ratelimit/local.go index 45376394..c2df062c 100644 --- a/pkg/ratelimit/local.go +++ b/pkg/ratelimit/local.go @@ -1,6 +1,9 @@ package ratelimit import ( + "context" + "time" + localRatelimit "go.uber.org/ratelimit" ) @@ -15,3 +18,8 @@ func NewLocalLimiter(maxRPS int) Limiter { localRatelimit.New(maxRPS), } } + +// Take .. +func (l Local) Take(_ context.Context) time.Time { + return l.Limiter.Take() +} diff --git a/pkg/ratelimit/ratelimit.go b/pkg/ratelimit/ratelimit.go index 6c354455..fcdf16c8 100644 --- a/pkg/ratelimit/ratelimit.go +++ b/pkg/ratelimit/ratelimit.go @@ -1,6 +1,7 @@ package ratelimit import ( + "context" "time" log "github.com/sirupsen/logrus" @@ -8,13 +9,14 @@ import ( // Limiter .. type Limiter interface { - Take() time.Time + Take(context.Context) time.Time } // Take .. -func Take(l Limiter) { +func Take(ctx context.Context, l Limiter) { now := time.Now() - throttled := l.Take() + throttled := l.Take(ctx) + if throttled.Sub(now).Milliseconds() > 10 { log.WithFields( log.Fields{ diff --git a/pkg/ratelimit/ratelimit_test.go b/pkg/ratelimit/ratelimit_test.go index b907525f..663a34d8 100644 --- a/pkg/ratelimit/ratelimit_test.go +++ b/pkg/ratelimit/ratelimit_test.go @@ -14,12 +14,15 @@ import ( func MeasureTakeDuration(l Limiter) int64 { start := time.Now() - Take(l) + + Take(context.TODO(), l) + return int64(time.Since(start)) } func TestLocalTake(t *testing.T) { l := NewLocalLimiter(1) + assert.LessOrEqual(t, MeasureTakeDuration(l), int64(100*time.Millisecond)) assert.GreaterOrEqual(t, MeasureTakeDuration(l), int64(time.Second)) } @@ -29,10 +32,10 @@ func TestRedisTake(t *testing.T) { if err != nil { panic(err) } + defer s.Close() l := NewRedisLimiter( - context.Background(), redis.NewClient(&redis.Options{Addr: s.Addr()}), 1, ) @@ -44,19 +47,22 @@ func TestRedisTake(t *testing.T) { func TestRedisTakeError(t *testing.T) { if os.Getenv("SHOULD_ERROR") == "1" { l := NewRedisLimiter( - context.Background(), redis.NewClient(&redis.Options{Addr: "doesnotexist"}), 1, ) - Take(l) + + Take(context.TODO(), l) + return } cmd := exec.Command(os.Args[0], "-test.run=TestRedisTakeError") cmd.Env = append(os.Environ(), "SHOULD_ERROR=1") + err := cmd.Run() if e, ok := err.(*exec.ExitError); ok && !e.Success() { return } + t.Fatal("process ran successfully, wanted exit status 1") } diff --git a/pkg/ratelimit/redis.go b/pkg/ratelimit/redis.go index 9592a403..b92ee728 100644 --- a/pkg/ratelimit/redis.go +++ b/pkg/ratelimit/redis.go @@ -14,25 +14,25 @@ const redisKey string = `gcpe:gitlab:api` // Redis .. type Redis struct { *redis_rate.Limiter - Context context.Context - MaxRPS int + MaxRPS int } // NewRedisLimiter .. -func NewRedisLimiter(ctx context.Context, redisClient *redis.Client, maxRPS int) Limiter { +func NewRedisLimiter(redisClient *redis.Client, maxRPS int) Limiter { return Redis{ Limiter: redis_rate.NewLimiter(redisClient), - Context: ctx, MaxRPS: maxRPS, } } // Take .. -func (r Redis) Take() time.Time { - res, err := r.Allow(r.Context, redisKey, redis_rate.PerSecond(r.MaxRPS)) +func (r Redis) Take(ctx context.Context) time.Time { + res, err := r.Allow(ctx, redisKey, redis_rate.PerSecond(r.MaxRPS)) if err != nil { log.Fatalf(err.Error()) } + time.Sleep(res.RetryAfter) + return time.Now() } diff --git a/pkg/ratelimit/redis_test.go b/pkg/ratelimit/redis_test.go index 65141c41..f998b6b4 100644 --- a/pkg/ratelimit/redis_test.go +++ b/pkg/ratelimit/redis_test.go @@ -1,7 +1,6 @@ package ratelimit import ( - "context" "testing" "github.com/go-redis/redis/v8" @@ -12,14 +11,12 @@ import ( func TestNewRedisLimiter(t *testing.T) { redisClient := redis.NewClient(&redis.Options{}) l := NewRedisLimiter( - context.Background(), redisClient, 10, ) expectedValue := Redis{ Limiter: redis_rate.NewLimiter(redisClient), - Context: context.Background(), MaxRPS: 10, } diff --git a/pkg/schemas/environments.go b/pkg/schemas/environments.go index 9c5adbeb..7bc4d196 100644 --- a/pkg/schemas/environments.go +++ b/pkg/schemas/environments.go @@ -25,11 +25,10 @@ func (e Environment) Key() EnvironmentKey { return EnvironmentKey(strconv.Itoa(int(crc32.ChecksumIEEE([]byte(e.ProjectName + e.Name))))) } -// Environments allows us to keep track of all the Environment -// objects we have discovered +// Environments allows us to keep track of all the Environment objects we have discovered. type Environments map[EnvironmentKey]Environment -// Count returns the amount of environments in the map +// Count returns the amount of environments in the map. func (envs Environments) Count() int { return len(envs) } diff --git a/pkg/schemas/jobs.go b/pkg/schemas/jobs.go index a398e2ce..3f970374 100644 --- a/pkg/schemas/jobs.go +++ b/pkg/schemas/jobs.go @@ -29,17 +29,20 @@ type Jobs map[string]Job // NewJob .. func NewJob(gj goGitlab.Job) Job { - var artifactSize float64 + var ( + artifactSize float64 + timestamp float64 + queued time.Duration + ) + for _, artifact := range gj.Artifacts { artifactSize += float64(artifact.Size) } - var timestamp float64 if gj.CreatedAt != nil { timestamp = float64(gj.CreatedAt.Unix()) } - var queued time.Duration if gj.StartedAt != nil && gj.CreatedAt != nil { if gj.CreatedAt.Before(*gj.StartedAt) { queued = gj.StartedAt.Sub(*gj.CreatedAt) diff --git a/pkg/schemas/metric.go b/pkg/schemas/metric.go index c5429378..cc90482d 100644 --- a/pkg/schemas/metric.go +++ b/pkg/schemas/metric.go @@ -9,7 +9,7 @@ import ( ) const ( - // MetricKindCoverage refers to the coerage of a job/pipeline + // MetricKindCoverage refers to the coerage of a job/pipeline. MetricKindCoverage MetricKind = iota // MetricKindDurationSeconds .. diff --git a/pkg/schemas/pipelines.go b/pkg/schemas/pipelines.go index 9bcd1d49..b4f67c37 100644 --- a/pkg/schemas/pipelines.go +++ b/pkg/schemas/pipelines.go @@ -21,8 +21,13 @@ type Pipeline struct { // NewPipeline .. func NewPipeline(gp goGitlab.Pipeline) Pipeline { - var coverage float64 - var err error + var ( + coverage float64 + err error + timestamp float64 + queued time.Duration + ) + if gp.Coverage != "" { coverage, err = strconv.ParseFloat(gp.Coverage, 64) if err != nil { @@ -30,12 +35,10 @@ func NewPipeline(gp goGitlab.Pipeline) Pipeline { } } - var timestamp float64 if gp.UpdatedAt != nil { timestamp = float64(gp.UpdatedAt.Unix()) } - var queued time.Duration if gp.StartedAt != nil && gp.CreatedAt != nil { if gp.CreatedAt.Before(*gp.StartedAt) { queued = gp.StartedAt.Sub(*gp.CreatedAt) diff --git a/pkg/schemas/ref.go b/pkg/schemas/ref.go index 3f5cd4f5..7ac8d8d8 100644 --- a/pkg/schemas/ref.go +++ b/pkg/schemas/ref.go @@ -12,21 +12,21 @@ import ( const ( mergeRequestRegexp string = `^((\d+)|refs/merge-requests/(\d+)/head)$` - // RefKindBranch refers to a branch + // RefKindBranch refers to a branch. RefKindBranch RefKind = "branch" - // RefKindTag refers to a tag + // RefKindTag refers to a tag. RefKindTag RefKind = "tag" - // RefKindMergeRequest refers to a tag + // RefKindMergeRequest refers to a tag. RefKindMergeRequest RefKind = "merge-request" ) -// RefKind is used to determine the kind of the ref +// RefKind is used to determine the kind of the ref. type RefKind string // Ref is what we will use a metrics entity on which we will -// perform regular pulling operations +// perform regular pulling operations. type Ref struct { Kind RefKind Name string @@ -44,10 +44,10 @@ func (ref Ref) Key() RefKey { } // Refs allows us to keep track of all the Ref -// we have configured/discovered +// we have configured/discovered. type Refs map[RefKey]Ref -// Count returns the amount of projects refs in the map +// Count returns the amount of projects refs in the map. func (refs Refs) Count() int { return len(refs) } @@ -63,7 +63,7 @@ func (ref Ref) DefaultLabelsValues() map[string]string { } } -// NewRef is an helper which returns a new Ref +// NewRef is an helper which returns a new Ref. func NewRef( project Project, kind RefKind, @@ -77,7 +77,7 @@ func NewRef( } } -// GetRefRegexp returns the expected regexp given a ProjectPullRefs config and a RefKind +// GetRefRegexp returns the expected regexp given a ProjectPullRefs config and a RefKind. func GetRefRegexp(ppr config.ProjectPullRefs, rk RefKind) (re *regexp.Regexp, err error) { switch rk { case RefKindBranch: @@ -87,10 +87,11 @@ func GetRefRegexp(ppr config.ProjectPullRefs, rk RefKind) (re *regexp.Regexp, er case RefKindMergeRequest: return regexp.Compile(mergeRequestRegexp) } + return nil, fmt.Errorf("invalid ref kind (%v)", rk) } -// GetMergeRequestIIDFromRefName parse a refName to extract a merge request IID +// GetMergeRequestIIDFromRefName parse a refName to extract a merge request IID. func GetMergeRequestIIDFromRefName(refName string) (string, error) { re := regexp.MustCompile(mergeRequestRegexp) if matches := re.FindStringSubmatch(refName); len(matches) == 4 { @@ -102,5 +103,6 @@ func GetMergeRequestIIDFromRefName(refName string) (string, error) { return matches[3], nil } } + return refName, fmt.Errorf("unable to extract the merge-request ID from the ref (%s)", refName) } diff --git a/pkg/schemas/tasks.go b/pkg/schemas/tasks.go index bf5f8b83..932b0786 100644 --- a/pkg/schemas/tasks.go +++ b/pkg/schemas/tasks.go @@ -1,6 +1,6 @@ package schemas -// TaskType represents the type of a task +// TaskType represents the type of a task. type TaskType string const ( @@ -44,5 +44,5 @@ const ( TaskTypeGarbageCollectMetrics TaskType = "GarbageCollectMetrics" ) -// Tasks can be used to keep track of tasks +// Tasks can be used to keep track of tasks. type Tasks map[TaskType]map[string]interface{} diff --git a/pkg/store/local.go b/pkg/store/local.go index c37c3f16..afa34be5 100644 --- a/pkg/store/local.go +++ b/pkg/store/local.go @@ -1,6 +1,7 @@ package store import ( + "context" "sync" "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas" @@ -26,26 +27,28 @@ type Local struct { } // SetProject .. -func (l *Local) SetProject(p schemas.Project) error { +func (l *Local) SetProject(_ context.Context, p schemas.Project) error { l.projectsMutex.Lock() defer l.projectsMutex.Unlock() l.projects[p.Key()] = p + return nil } // DelProject .. -func (l *Local) DelProject(k schemas.ProjectKey) error { +func (l *Local) DelProject(_ context.Context, k schemas.ProjectKey) error { l.projectsMutex.Lock() defer l.projectsMutex.Unlock() delete(l.projects, k) + return nil } // GetProject .. -func (l *Local) GetProject(p *schemas.Project) error { - exists, _ := l.ProjectExists(p.Key()) +func (l *Local) GetProject(ctx context.Context, p *schemas.Project) error { + exists, _ := l.ProjectExists(ctx, p.Key()) if exists { l.projectsMutex.RLock() @@ -57,28 +60,31 @@ func (l *Local) GetProject(p *schemas.Project) error { } // ProjectExists .. -func (l *Local) ProjectExists(k schemas.ProjectKey) (bool, error) { +func (l *Local) ProjectExists(_ context.Context, k schemas.ProjectKey) (bool, error) { l.projectsMutex.RLock() defer l.projectsMutex.RUnlock() _, ok := l.projects[k] + return ok, nil } // Projects .. -func (l *Local) Projects() (projects schemas.Projects, err error) { +func (l *Local) Projects(_ context.Context) (projects schemas.Projects, err error) { projects = make(schemas.Projects) + l.projectsMutex.RLock() defer l.projectsMutex.RUnlock() for k, v := range l.projects { projects[k] = v } + return } // ProjectsCount .. -func (l *Local) ProjectsCount() (int64, error) { +func (l *Local) ProjectsCount(_ context.Context) (int64, error) { l.projectsMutex.RLock() defer l.projectsMutex.RUnlock() @@ -86,26 +92,28 @@ func (l *Local) ProjectsCount() (int64, error) { } // SetEnvironment .. -func (l *Local) SetEnvironment(environment schemas.Environment) error { +func (l *Local) SetEnvironment(_ context.Context, environment schemas.Environment) error { l.environmentsMutex.Lock() defer l.environmentsMutex.Unlock() l.environments[environment.Key()] = environment + return nil } // DelEnvironment .. -func (l *Local) DelEnvironment(k schemas.EnvironmentKey) error { +func (l *Local) DelEnvironment(_ context.Context, k schemas.EnvironmentKey) error { l.environmentsMutex.Lock() defer l.environmentsMutex.Unlock() delete(l.environments, k) + return nil } // GetEnvironment .. -func (l *Local) GetEnvironment(environment *schemas.Environment) error { - exists, _ := l.EnvironmentExists(environment.Key()) +func (l *Local) GetEnvironment(ctx context.Context, environment *schemas.Environment) error { + exists, _ := l.EnvironmentExists(ctx, environment.Key()) if exists { l.environmentsMutex.RLock() @@ -117,28 +125,31 @@ func (l *Local) GetEnvironment(environment *schemas.Environment) error { } // EnvironmentExists .. -func (l *Local) EnvironmentExists(k schemas.EnvironmentKey) (bool, error) { +func (l *Local) EnvironmentExists(_ context.Context, k schemas.EnvironmentKey) (bool, error) { l.environmentsMutex.RLock() defer l.environmentsMutex.RUnlock() _, ok := l.environments[k] + return ok, nil } // Environments .. -func (l *Local) Environments() (environments schemas.Environments, err error) { +func (l *Local) Environments(_ context.Context) (environments schemas.Environments, err error) { environments = make(schemas.Environments) + l.environmentsMutex.RLock() defer l.environmentsMutex.RUnlock() for k, v := range l.environments { environments[k] = v } + return } // EnvironmentsCount .. -func (l *Local) EnvironmentsCount() (int64, error) { +func (l *Local) EnvironmentsCount(_ context.Context) (int64, error) { l.environmentsMutex.RLock() defer l.environmentsMutex.RUnlock() @@ -146,26 +157,28 @@ func (l *Local) EnvironmentsCount() (int64, error) { } // SetRef .. -func (l *Local) SetRef(ref schemas.Ref) error { +func (l *Local) SetRef(_ context.Context, ref schemas.Ref) error { l.refsMutex.Lock() defer l.refsMutex.Unlock() l.refs[ref.Key()] = ref + return nil } // DelRef .. -func (l *Local) DelRef(k schemas.RefKey) error { +func (l *Local) DelRef(_ context.Context, k schemas.RefKey) error { l.refsMutex.Lock() defer l.refsMutex.Unlock() delete(l.refs, k) + return nil } // GetRef .. -func (l *Local) GetRef(ref *schemas.Ref) error { - exists, _ := l.RefExists(ref.Key()) +func (l *Local) GetRef(ctx context.Context, ref *schemas.Ref) error { + exists, _ := l.RefExists(ctx, ref.Key()) if exists { l.refsMutex.RLock() @@ -177,28 +190,31 @@ func (l *Local) GetRef(ref *schemas.Ref) error { } // RefExists .. -func (l *Local) RefExists(k schemas.RefKey) (bool, error) { +func (l *Local) RefExists(_ context.Context, k schemas.RefKey) (bool, error) { l.refsMutex.RLock() defer l.refsMutex.RUnlock() _, ok := l.refs[k] + return ok, nil } // Refs .. -func (l *Local) Refs() (refs schemas.Refs, err error) { +func (l *Local) Refs(_ context.Context) (refs schemas.Refs, err error) { refs = make(schemas.Refs) + l.refsMutex.RLock() defer l.refsMutex.RUnlock() for k, v := range l.refs { refs[k] = v } + return } // RefsCount .. -func (l *Local) RefsCount() (int64, error) { +func (l *Local) RefsCount(_ context.Context) (int64, error) { l.refsMutex.RLock() defer l.refsMutex.RUnlock() @@ -206,26 +222,28 @@ func (l *Local) RefsCount() (int64, error) { } // SetMetric .. -func (l *Local) SetMetric(m schemas.Metric) error { +func (l *Local) SetMetric(_ context.Context, m schemas.Metric) error { l.metricsMutex.Lock() defer l.metricsMutex.Unlock() l.metrics[m.Key()] = m + return nil } // DelMetric .. -func (l *Local) DelMetric(k schemas.MetricKey) error { +func (l *Local) DelMetric(_ context.Context, k schemas.MetricKey) error { l.metricsMutex.Lock() defer l.metricsMutex.Unlock() delete(l.metrics, k) + return nil } // GetMetric .. -func (l *Local) GetMetric(m *schemas.Metric) error { - exists, _ := l.MetricExists(m.Key()) +func (l *Local) GetMetric(ctx context.Context, m *schemas.Metric) error { + exists, _ := l.MetricExists(ctx, m.Key()) if exists { l.metricsMutex.RLock() @@ -237,35 +255,38 @@ func (l *Local) GetMetric(m *schemas.Metric) error { } // MetricExists .. -func (l *Local) MetricExists(k schemas.MetricKey) (bool, error) { +func (l *Local) MetricExists(_ context.Context, k schemas.MetricKey) (bool, error) { l.metricsMutex.RLock() defer l.metricsMutex.RUnlock() _, ok := l.metrics[k] + return ok, nil } // Metrics .. -func (l *Local) Metrics() (metrics schemas.Metrics, err error) { +func (l *Local) Metrics(_ context.Context) (metrics schemas.Metrics, err error) { metrics = make(schemas.Metrics) + l.metricsMutex.RLock() defer l.metricsMutex.RUnlock() for k, v := range l.metrics { metrics[k] = v } + return } // MetricsCount .. -func (l *Local) MetricsCount() (int64, error) { +func (l *Local) MetricsCount(_ context.Context) (int64, error) { l.metricsMutex.RLock() defer l.metricsMutex.RUnlock() return int64(len(l.metrics)), nil } -// isTaskAlreadyQueued assess if a task is already queued or not +// isTaskAlreadyQueued assess if a task is already queued or not. func (l *Local) isTaskAlreadyQueued(tt schemas.TaskType, uniqueID string) bool { l.tasksMutex.Lock() defer l.tasksMutex.Unlock() @@ -277,6 +298,7 @@ func (l *Local) isTaskAlreadyQueued(tt schemas.TaskType, uniqueID string) bool { taskTypeQueue, ok := l.tasks[tt] if !ok { l.tasks[tt] = make(map[string]interface{}) + return false } @@ -289,29 +311,34 @@ func (l *Local) isTaskAlreadyQueued(tt schemas.TaskType, uniqueID string) bool { // QueueTask registers that we are queueing the task. // It returns true if it managed to schedule it, false if it was already scheduled. -func (l *Local) QueueTask(tt schemas.TaskType, uniqueID, _ string) (bool, error) { +func (l *Local) QueueTask(_ context.Context, tt schemas.TaskType, uniqueID, _ string) (bool, error) { if !l.isTaskAlreadyQueued(tt, uniqueID) { l.tasksMutex.Lock() defer l.tasksMutex.Unlock() + l.tasks[tt][uniqueID] = nil + return true, nil } + return false, nil } -// UnqueueTask removes the task from the tracker -func (l *Local) UnqueueTask(tt schemas.TaskType, uniqueID string) error { +// UnqueueTask removes the task from the tracker. +func (l *Local) UnqueueTask(_ context.Context, tt schemas.TaskType, uniqueID string) error { if l.isTaskAlreadyQueued(tt, uniqueID) { l.tasksMutex.Lock() defer l.tasksMutex.Unlock() + delete(l.tasks[tt], uniqueID) l.executedTasksCount++ } + return nil } // CurrentlyQueuedTasksCount .. -func (l *Local) CurrentlyQueuedTasksCount() (count uint64, err error) { +func (l *Local) CurrentlyQueuedTasksCount(_ context.Context) (count uint64, err error) { l.tasksMutex.RLock() defer l.tasksMutex.RUnlock() @@ -323,8 +350,9 @@ func (l *Local) CurrentlyQueuedTasksCount() (count uint64, err error) { } // ExecutedTasksCount .. -func (l *Local) ExecutedTasksCount() (uint64, error) { +func (l *Local) ExecutedTasksCount(_ context.Context) (uint64, error) { l.tasksMutex.RLock() defer l.tasksMutex.RUnlock() + return l.executedTasksCount, nil } diff --git a/pkg/store/local_test.go b/pkg/store/local_test.go index 9b3fb28f..28fef60d 100644 --- a/pkg/store/local_test.go +++ b/pkg/store/local_test.go @@ -13,42 +13,42 @@ func TestLocalProjectFunctions(t *testing.T) { p.OutputSparseStatusMetrics = false l := NewLocalStore() - l.SetProject(p) + assert.NoError(t, l.SetProject(testCtx, p)) // Set project - projects, err := l.Projects() + projects, err := l.Projects(testCtx) assert.NoError(t, err) assert.Contains(t, projects, p.Key()) assert.Equal(t, p, projects[p.Key()]) // Project exists - exists, err := l.ProjectExists(p.Key()) + exists, err := l.ProjectExists(testCtx, p.Key()) assert.NoError(t, err) assert.True(t, exists) // GetProject should succeed newProject := schemas.NewProject("foo/bar") - assert.NoError(t, l.GetProject(&newProject)) + assert.NoError(t, l.GetProject(testCtx, &newProject)) assert.Equal(t, p, newProject) // Count - count, err := l.ProjectsCount() + count, err := l.ProjectsCount(testCtx) assert.NoError(t, err) assert.Equal(t, int64(1), count) // Delete project - l.DelProject(p.Key()) - projects, err = l.Projects() + assert.NoError(t, l.DelProject(testCtx, p.Key())) + projects, err = l.Projects(testCtx) assert.NoError(t, err) assert.NotContains(t, projects, p.Key()) - exists, err = l.ProjectExists(p.Key()) + exists, err = l.ProjectExists(testCtx, p.Key()) assert.NoError(t, err) assert.False(t, exists) // GetProject should not update the var this time newProject = schemas.NewProject("foo/bar") - assert.NoError(t, l.GetProject(&newProject)) + assert.NoError(t, l.GetProject(testCtx, &newProject)) assert.NotEqual(t, p, newProject) } @@ -59,16 +59,16 @@ func TestLocalEnvironmentFunctions(t *testing.T) { } l := NewLocalStore() - l.SetEnvironment(environment) + assert.NoError(t, l.SetEnvironment(testCtx, environment)) // Set project - environments, err := l.Environments() + environments, err := l.Environments(testCtx) assert.NoError(t, err) assert.Contains(t, environments, environment.Key()) assert.Equal(t, environment, environments[environment.Key()]) // Environment exists - exists, err := l.EnvironmentExists(environment.Key()) + exists, err := l.EnvironmentExists(testCtx, environment.Key()) assert.NoError(t, err) assert.True(t, exists) @@ -77,21 +77,21 @@ func TestLocalEnvironmentFunctions(t *testing.T) { ProjectName: "foo", ID: 1, } - assert.NoError(t, l.GetEnvironment(&newEnvironment)) + assert.NoError(t, l.GetEnvironment(testCtx, &newEnvironment)) assert.Equal(t, environment, newEnvironment) // Count - count, err := l.EnvironmentsCount() + count, err := l.EnvironmentsCount(testCtx) assert.NoError(t, err) assert.Equal(t, int64(1), count) // Delete Environment - l.DelEnvironment(environment.Key()) - environments, err = l.Environments() + assert.NoError(t, l.DelEnvironment(testCtx, environment.Key())) + environments, err = l.Environments(testCtx) assert.NoError(t, err) assert.NotContains(t, environments, environment.Key()) - exists, err = l.EnvironmentExists(environment.Key()) + exists, err = l.EnvironmentExists(testCtx, environment.Key()) assert.NoError(t, err) assert.False(t, exists) @@ -101,7 +101,7 @@ func TestLocalEnvironmentFunctions(t *testing.T) { ID: 1, ExternalURL: "foo", } - assert.NoError(t, l.GetEnvironment(&newEnvironment)) + assert.NoError(t, l.GetEnvironment(testCtx, &newEnvironment)) assert.NotEqual(t, environment, newEnvironment) } @@ -116,14 +116,15 @@ func TestLocalRefFunctions(t *testing.T) { // Set project l := NewLocalStore() - l.SetRef(ref) - projectsRefs, err := l.Refs() + assert.NoError(t, l.SetRef(testCtx, ref)) + + projectsRefs, err := l.Refs(testCtx) assert.NoError(t, err) assert.Contains(t, projectsRefs, ref.Key()) assert.Equal(t, ref, projectsRefs[ref.Key()]) // Ref exists - exists, err := l.RefExists(ref.Key()) + exists, err := l.RefExists(testCtx, ref.Key()) assert.NoError(t, err) assert.True(t, exists) @@ -133,21 +134,21 @@ func TestLocalRefFunctions(t *testing.T) { Kind: schemas.RefKindBranch, Name: "sweet", } - assert.NoError(t, l.GetRef(&newRef)) + assert.NoError(t, l.GetRef(testCtx, &newRef)) assert.Equal(t, ref, newRef) // Count - count, err := l.RefsCount() + count, err := l.RefsCount(testCtx) assert.NoError(t, err) assert.Equal(t, int64(1), count) // Delete Ref - l.DelRef(ref.Key()) - projectsRefs, err = l.Refs() + assert.NoError(t, l.DelRef(testCtx, ref.Key())) + projectsRefs, err = l.Refs(testCtx) assert.NoError(t, err) assert.NotContains(t, projectsRefs, ref.Key()) - exists, err = l.RefExists(ref.Key()) + exists, err = l.RefExists(testCtx, ref.Key()) assert.NoError(t, err) assert.False(t, exists) @@ -157,7 +158,7 @@ func TestLocalRefFunctions(t *testing.T) { Project: schemas.NewProject("foo/bar"), Name: "sweet", } - assert.NoError(t, l.GetRef(&newRef)) + assert.NoError(t, l.GetRef(testCtx, &newRef)) assert.NotEqual(t, ref, newRef) } @@ -171,16 +172,16 @@ func TestLocalMetricFunctions(t *testing.T) { } l := NewLocalStore() - l.SetMetric(m) + assert.NoError(t, l.SetMetric(testCtx, m)) // Set metric - metrics, err := l.Metrics() + metrics, err := l.Metrics(testCtx) assert.NoError(t, err) assert.Contains(t, metrics, m.Key()) assert.Equal(t, m, metrics[m.Key()]) // Metric exists - exists, err := l.MetricExists(m.Key()) + exists, err := l.MetricExists(testCtx, m.Key()) assert.NoError(t, err) assert.True(t, exists) @@ -191,21 +192,21 @@ func TestLocalMetricFunctions(t *testing.T) { "foo": "bar", }, } - assert.NoError(t, l.GetMetric(&newMetric)) + assert.NoError(t, l.GetMetric(testCtx, &newMetric)) assert.Equal(t, m, newMetric) // Count - count, err := l.MetricsCount() + count, err := l.MetricsCount(testCtx) assert.NoError(t, err) assert.Equal(t, int64(1), count) // Delete Metric - l.DelMetric(m.Key()) - metrics, err = l.Metrics() + l.DelMetric(testCtx, m.Key()) + metrics, err = l.Metrics(testCtx) assert.NoError(t, err) assert.NotContains(t, metrics, m.Key()) - exists, err = l.MetricExists(m.Key()) + exists, err = l.MetricExists(testCtx, m.Key()) assert.NoError(t, err) assert.False(t, exists) @@ -216,54 +217,54 @@ func TestLocalMetricFunctions(t *testing.T) { "foo": "bar", }, } - assert.NoError(t, l.GetMetric(&newMetric)) + assert.NoError(t, l.GetMetric(testCtx, &newMetric)) assert.NotEqual(t, m, newMetric) } func TestLocalQueueTask(t *testing.T) { l := NewLocalStore() - ok, err := l.QueueTask(schemas.TaskTypePullMetrics, "foo", "") + ok, err := l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "") assert.True(t, ok) assert.NoError(t, err) - ok, err = l.QueueTask(schemas.TaskTypePullMetrics, "foo", "") + ok, err = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "") assert.False(t, ok) assert.NoError(t, err) - l.QueueTask(schemas.TaskTypePullMetrics, "bar", "") - ok, err = l.QueueTask(schemas.TaskTypePullMetrics, "bar", "") + _, _ = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "bar", "") + ok, err = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "bar", "") assert.False(t, ok) assert.NoError(t, err) } func TestLocalUnqueueTask(t *testing.T) { l := NewLocalStore() - l.QueueTask(schemas.TaskTypePullMetrics, "foo", "") + _, _ = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "") assert.Equal(t, uint64(0), l.(*Local).executedTasksCount) - assert.NoError(t, l.UnqueueTask(schemas.TaskTypePullMetrics, "foo")) + assert.NoError(t, l.UnqueueTask(testCtx, schemas.TaskTypePullMetrics, "foo")) assert.Equal(t, uint64(1), l.(*Local).executedTasksCount) } func TestLocalCurrentlyQueuedTasksCount(t *testing.T) { l := NewLocalStore() - l.QueueTask(schemas.TaskTypePullMetrics, "foo", "") - l.QueueTask(schemas.TaskTypePullMetrics, "bar", "") - l.QueueTask(schemas.TaskTypePullMetrics, "baz", "") + _, _ = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "") + _, _ = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "bar", "") + _, _ = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "baz", "") - count, _ := l.CurrentlyQueuedTasksCount() + count, _ := l.CurrentlyQueuedTasksCount(testCtx) assert.Equal(t, uint64(3), count) - l.UnqueueTask(schemas.TaskTypePullMetrics, "foo") - count, _ = l.CurrentlyQueuedTasksCount() + assert.NoError(t, l.UnqueueTask(testCtx, schemas.TaskTypePullMetrics, "foo")) + count, _ = l.CurrentlyQueuedTasksCount(testCtx) assert.Equal(t, uint64(2), count) } func TestLocalExecutedTasksCount(t *testing.T) { l := NewLocalStore() - l.QueueTask(schemas.TaskTypePullMetrics, "foo", "") - l.QueueTask(schemas.TaskTypePullMetrics, "bar", "") - l.UnqueueTask(schemas.TaskTypePullMetrics, "foo") - l.UnqueueTask(schemas.TaskTypePullMetrics, "foo") + _, _ = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "") + _, _ = l.QueueTask(testCtx, schemas.TaskTypePullMetrics, "bar", "") + _ = l.UnqueueTask(testCtx, schemas.TaskTypePullMetrics, "foo") + _ = l.UnqueueTask(testCtx, schemas.TaskTypePullMetrics, "foo") - count, _ := l.ExecutedTasksCount() + count, _ := l.ExecutedTasksCount(testCtx) assert.Equal(t, uint64(1), count) } diff --git a/pkg/store/redis.go b/pkg/store/redis.go index a50ff429..78515725 100644 --- a/pkg/store/redis.go +++ b/pkg/store/redis.go @@ -24,37 +24,38 @@ const ( // Redis .. type Redis struct { *redis.Client - - ctx context.Context } // SetProject .. -func (r *Redis) SetProject(p schemas.Project) error { +func (r *Redis) SetProject(ctx context.Context, p schemas.Project) error { marshalledProject, err := msgpack.Marshal(p) if err != nil { return err } - _, err = r.HSet(r.ctx, redisProjectsKey, string(p.Key()), marshalledProject).Result() + _, err = r.HSet(ctx, redisProjectsKey, string(p.Key()), marshalledProject).Result() + return err } // DelProject .. -func (r *Redis) DelProject(k schemas.ProjectKey) error { - _, err := r.HDel(r.ctx, redisProjectsKey, string(k)).Result() +func (r *Redis) DelProject(ctx context.Context, k schemas.ProjectKey) error { + _, err := r.HDel(ctx, redisProjectsKey, string(k)).Result() + return err } // GetProject .. -func (r *Redis) GetProject(p *schemas.Project) error { - exists, err := r.ProjectExists(p.Key()) +func (r *Redis) GetProject(ctx context.Context, p *schemas.Project) error { + exists, err := r.ProjectExists(ctx, p.Key()) if err != nil { return err } if exists { k := p.Key() - marshalledProject, err := r.HGet(r.ctx, redisProjectsKey, string(k)).Result() + + marshalledProject, err := r.HGet(ctx, redisProjectsKey, string(k)).Result() if err != nil { return err } @@ -68,14 +69,15 @@ func (r *Redis) GetProject(p *schemas.Project) error { } // ProjectExists .. -func (r *Redis) ProjectExists(k schemas.ProjectKey) (bool, error) { - return r.HExists(r.ctx, redisProjectsKey, string(k)).Result() +func (r *Redis) ProjectExists(ctx context.Context, k schemas.ProjectKey) (bool, error) { + return r.HExists(ctx, redisProjectsKey, string(k)).Result() } // Projects .. -func (r *Redis) Projects() (schemas.Projects, error) { +func (r *Redis) Projects(ctx context.Context) (schemas.Projects, error) { projects := schemas.Projects{} - marshalledProjects, err := r.HGetAll(r.ctx, redisProjectsKey).Result() + + marshalledProjects, err := r.HGetAll(ctx, redisProjectsKey).Result() if err != nil { return projects, err } @@ -86,6 +88,7 @@ func (r *Redis) Projects() (schemas.Projects, error) { if err = msgpack.Unmarshal([]byte(marshalledProject), &p); err != nil { return projects, err } + projects[schemas.ProjectKey(stringProjectKey)] = p } @@ -93,37 +96,40 @@ func (r *Redis) Projects() (schemas.Projects, error) { } // ProjectsCount .. -func (r *Redis) ProjectsCount() (int64, error) { - return r.HLen(r.ctx, redisProjectsKey).Result() +func (r *Redis) ProjectsCount(ctx context.Context) (int64, error) { + return r.HLen(ctx, redisProjectsKey).Result() } // SetEnvironment .. -func (r *Redis) SetEnvironment(e schemas.Environment) error { +func (r *Redis) SetEnvironment(ctx context.Context, e schemas.Environment) error { marshalledEnvironment, err := msgpack.Marshal(e) if err != nil { return err } - _, err = r.HSet(r.ctx, redisEnvironmentsKey, string(e.Key()), marshalledEnvironment).Result() + _, err = r.HSet(ctx, redisEnvironmentsKey, string(e.Key()), marshalledEnvironment).Result() + return err } // DelEnvironment .. -func (r *Redis) DelEnvironment(k schemas.EnvironmentKey) error { - _, err := r.HDel(r.ctx, redisEnvironmentsKey, string(k)).Result() +func (r *Redis) DelEnvironment(ctx context.Context, k schemas.EnvironmentKey) error { + _, err := r.HDel(ctx, redisEnvironmentsKey, string(k)).Result() + return err } // GetEnvironment .. -func (r *Redis) GetEnvironment(e *schemas.Environment) error { - exists, err := r.EnvironmentExists(e.Key()) +func (r *Redis) GetEnvironment(ctx context.Context, e *schemas.Environment) error { + exists, err := r.EnvironmentExists(ctx, e.Key()) if err != nil { return err } if exists { k := e.Key() - marshalledEnvironment, err := r.HGet(r.ctx, redisEnvironmentsKey, string(k)).Result() + + marshalledEnvironment, err := r.HGet(ctx, redisEnvironmentsKey, string(k)).Result() if err != nil { return err } @@ -137,14 +143,15 @@ func (r *Redis) GetEnvironment(e *schemas.Environment) error { } // EnvironmentExists .. -func (r *Redis) EnvironmentExists(k schemas.EnvironmentKey) (bool, error) { - return r.HExists(r.ctx, redisEnvironmentsKey, string(k)).Result() +func (r *Redis) EnvironmentExists(ctx context.Context, k schemas.EnvironmentKey) (bool, error) { + return r.HExists(ctx, redisEnvironmentsKey, string(k)).Result() } // Environments .. -func (r *Redis) Environments() (schemas.Environments, error) { +func (r *Redis) Environments(ctx context.Context) (schemas.Environments, error) { environments := schemas.Environments{} - marshalledProjects, err := r.HGetAll(r.ctx, redisEnvironmentsKey).Result() + + marshalledProjects, err := r.HGetAll(ctx, redisEnvironmentsKey).Result() if err != nil { return environments, err } @@ -155,6 +162,7 @@ func (r *Redis) Environments() (schemas.Environments, error) { if err = msgpack.Unmarshal([]byte(marshalledEnvironment), &p); err != nil { return environments, err } + environments[schemas.EnvironmentKey(stringEnvironmentKey)] = p } @@ -162,37 +170,40 @@ func (r *Redis) Environments() (schemas.Environments, error) { } // EnvironmentsCount .. -func (r *Redis) EnvironmentsCount() (int64, error) { - return r.HLen(r.ctx, redisEnvironmentsKey).Result() +func (r *Redis) EnvironmentsCount(ctx context.Context) (int64, error) { + return r.HLen(ctx, redisEnvironmentsKey).Result() } // SetRef .. -func (r *Redis) SetRef(ref schemas.Ref) error { +func (r *Redis) SetRef(ctx context.Context, ref schemas.Ref) error { marshalledRef, err := msgpack.Marshal(ref) if err != nil { return err } - _, err = r.HSet(r.ctx, redisRefsKey, string(ref.Key()), marshalledRef).Result() + _, err = r.HSet(ctx, redisRefsKey, string(ref.Key()), marshalledRef).Result() + return err } // DelRef .. -func (r *Redis) DelRef(k schemas.RefKey) error { - _, err := r.HDel(r.ctx, redisRefsKey, string(k)).Result() +func (r *Redis) DelRef(ctx context.Context, k schemas.RefKey) error { + _, err := r.HDel(ctx, redisRefsKey, string(k)).Result() + return err } // GetRef .. -func (r *Redis) GetRef(ref *schemas.Ref) error { - exists, err := r.RefExists(ref.Key()) +func (r *Redis) GetRef(ctx context.Context, ref *schemas.Ref) error { + exists, err := r.RefExists(ctx, ref.Key()) if err != nil { return err } if exists { k := ref.Key() - marshalledRef, err := r.HGet(r.ctx, redisRefsKey, string(k)).Result() + + marshalledRef, err := r.HGet(ctx, redisRefsKey, string(k)).Result() if err != nil { return err } @@ -206,14 +217,15 @@ func (r *Redis) GetRef(ref *schemas.Ref) error { } // RefExists .. -func (r *Redis) RefExists(k schemas.RefKey) (bool, error) { - return r.HExists(r.ctx, redisRefsKey, string(k)).Result() +func (r *Redis) RefExists(ctx context.Context, k schemas.RefKey) (bool, error) { + return r.HExists(ctx, redisRefsKey, string(k)).Result() } // Refs .. -func (r *Redis) Refs() (schemas.Refs, error) { +func (r *Redis) Refs(ctx context.Context) (schemas.Refs, error) { refs := schemas.Refs{} - marshalledProjects, err := r.HGetAll(r.ctx, redisRefsKey).Result() + + marshalledProjects, err := r.HGetAll(ctx, redisRefsKey).Result() if err != nil { return refs, err } @@ -224,6 +236,7 @@ func (r *Redis) Refs() (schemas.Refs, error) { if err = msgpack.Unmarshal([]byte(marshalledRef), &p); err != nil { return refs, err } + refs[schemas.RefKey(stringRefKey)] = p } @@ -231,42 +244,45 @@ func (r *Redis) Refs() (schemas.Refs, error) { } // RefsCount .. -func (r *Redis) RefsCount() (int64, error) { - return r.HLen(r.ctx, redisRefsKey).Result() +func (r *Redis) RefsCount(ctx context.Context) (int64, error) { + return r.HLen(ctx, redisRefsKey).Result() } // SetMetric .. -func (r *Redis) SetMetric(m schemas.Metric) error { +func (r *Redis) SetMetric(ctx context.Context, m schemas.Metric) error { marshalledMetric, err := msgpack.Marshal(m) if err != nil { return err } - _, err = r.HSet(r.ctx, redisMetricsKey, string(m.Key()), marshalledMetric).Result() + _, err = r.HSet(ctx, redisMetricsKey, string(m.Key()), marshalledMetric).Result() + return err } // DelMetric .. -func (r *Redis) DelMetric(k schemas.MetricKey) error { - _, err := r.HDel(r.ctx, redisMetricsKey, string(k)).Result() +func (r *Redis) DelMetric(ctx context.Context, k schemas.MetricKey) error { + _, err := r.HDel(ctx, redisMetricsKey, string(k)).Result() + return err } // MetricExists .. -func (r *Redis) MetricExists(k schemas.MetricKey) (bool, error) { - return r.HExists(r.ctx, redisMetricsKey, string(k)).Result() +func (r *Redis) MetricExists(ctx context.Context, k schemas.MetricKey) (bool, error) { + return r.HExists(ctx, redisMetricsKey, string(k)).Result() } // GetMetric .. -func (r *Redis) GetMetric(m *schemas.Metric) error { - exists, err := r.MetricExists(m.Key()) +func (r *Redis) GetMetric(ctx context.Context, m *schemas.Metric) error { + exists, err := r.MetricExists(ctx, m.Key()) if err != nil { return err } if exists { k := m.Key() - marshalledMetric, err := r.HGet(r.ctx, redisMetricsKey, string(k)).Result() + + marshalledMetric, err := r.HGet(ctx, redisMetricsKey, string(k)).Result() if err != nil { return err } @@ -280,9 +296,10 @@ func (r *Redis) GetMetric(m *schemas.Metric) error { } // Metrics .. -func (r *Redis) Metrics() (schemas.Metrics, error) { +func (r *Redis) Metrics(ctx context.Context) (schemas.Metrics, error) { metrics := schemas.Metrics{} - marshalledMetrics, err := r.HGetAll(r.ctx, redisMetricsKey).Result() + + marshalledMetrics, err := r.HGetAll(ctx, redisMetricsKey).Result() if err != nil { return metrics, err } @@ -293,6 +310,7 @@ func (r *Redis) Metrics() (schemas.Metrics, error) { if err := msgpack.Unmarshal([]byte(marshalledMetric), &m); err != nil { return metrics, err } + metrics[schemas.MetricKey(stringMetricKey)] = m } @@ -300,22 +318,19 @@ func (r *Redis) Metrics() (schemas.Metrics, error) { } // MetricsCount .. -func (r *Redis) MetricsCount() (int64, error) { - return r.HLen(r.ctx, redisMetricsKey).Result() +func (r *Redis) MetricsCount(ctx context.Context) (int64, error) { + return r.HLen(ctx, redisMetricsKey).Result() } -func getRedisKeepaliveKey(processUUID string) string { - return fmt.Sprintf("%s:%s", redisKeepaliveKey, processUUID) +// SetKeepalive sets a key with an UUID corresponding to the currently running process. +func (r *Redis) SetKeepalive(ctx context.Context, uuid string, ttl time.Duration) (bool, error) { + return r.SetNX(ctx, fmt.Sprintf("%s:%s", redisKeepaliveKey, uuid), nil, ttl).Result() } -// SetKeepalive sets a key with an UUID corresponding to the currently running process -func (r *Redis) SetKeepalive(uuid string, ttl time.Duration) (bool, error) { - return r.SetNX(r.ctx, fmt.Sprintf("%s:%s", redisKeepaliveKey, uuid), nil, ttl).Result() -} +// KeepaliveExists returns whether a keepalive exists or not for a particular UUID. +func (r *Redis) KeepaliveExists(ctx context.Context, uuid string) (bool, error) { + exists, err := r.Exists(ctx, fmt.Sprintf("%s:%s", redisKeepaliveKey, uuid)).Result() -// KeepaliveExists returns whether a keepalive exists or not for a particular UUID -func (r *Redis) KeepaliveExists(uuid string) (bool, error) { - exists, err := r.Exists(r.ctx, fmt.Sprintf("%s:%s", redisKeepaliveKey, uuid)).Result() return exists == 1, err } @@ -325,73 +340,80 @@ func getRedisQueueKey(tt schemas.TaskType, taskUUID string) string { // QueueTask registers that we are queueing the task. // It returns true if it managed to schedule it, false if it was already scheduled. -func (r *Redis) QueueTask(tt schemas.TaskType, taskUUID, processUUID string) (set bool, err error) { +func (r *Redis) QueueTask(ctx context.Context, tt schemas.TaskType, taskUUID, processUUID string) (set bool, err error) { k := getRedisQueueKey(tt, taskUUID) // We attempt to set the key, if it already exists, we do not overwrite it - set, err = r.SetNX(r.ctx, k, processUUID, 0).Result() - if err != nil { + set, err = r.SetNX(ctx, k, processUUID, 0).Result() + if err != nil || set { return } // If the key already exists, we want to check a couple of things - if !set { - // First, that the associated process UUID is the same as our current one - var tpuuid string - if tpuuid, err = r.Get(r.ctx, k).Result(); err != nil { + // First, that the associated process UUID is the same as our current one + var tpuuid string + + if tpuuid, err = r.Get(ctx, k).Result(); err != nil { + return + } + + // If it is not the case, we assess that the one being associated with the task lock + // is still alive, otherwise we override the key and schedule the task + if tpuuid != processUUID { + var uuidIsAlive bool + + if uuidIsAlive, err = r.KeepaliveExists(ctx, tpuuid); err != nil { return } - // If it is not the case, we assess that the one being associated with the task lock - // is still alive, otherwise we override the key and schedule the task - if tpuuid != processUUID { - var uuidIsAlive bool - if uuidIsAlive, err = r.KeepaliveExists(tpuuid); err != nil { + if !uuidIsAlive { + if _, err = r.Set(ctx, k, processUUID, 0).Result(); err != nil { return } - if !uuidIsAlive { - if _, err = r.Set(r.ctx, k, processUUID, 0).Result(); err != nil { - return - } - return true, nil - } + return true, nil } } + return } -// UnqueueTask removes the task from the tracker -func (r *Redis) UnqueueTask(tt schemas.TaskType, taskUUID string) (err error) { +// UnqueueTask removes the task from the tracker. +func (r *Redis) UnqueueTask(ctx context.Context, tt schemas.TaskType, taskUUID string) (err error) { var matched int64 - matched, err = r.Del(r.ctx, getRedisQueueKey(tt, taskUUID)).Result() + + matched, err = r.Del(ctx, getRedisQueueKey(tt, taskUUID)).Result() if err != nil { return } if matched > 0 { - _, err = r.Incr(r.ctx, redisTasksExecutedCountKey).Result() + _, err = r.Incr(ctx, redisTasksExecutedCountKey).Result() } + return } // CurrentlyQueuedTasksCount .. -func (r *Redis) CurrentlyQueuedTasksCount() (count uint64, err error) { - iter := r.Scan(r.ctx, 0, fmt.Sprintf("%s:*", redisTaskKey), 0).Iterator() - for iter.Next(r.ctx) { +func (r *Redis) CurrentlyQueuedTasksCount(ctx context.Context) (count uint64, err error) { + iter := r.Scan(ctx, 0, fmt.Sprintf("%s:*", redisTaskKey), 0).Iterator() + for iter.Next(ctx) { count++ } + err = iter.Err() + return } // ExecutedTasksCount .. -func (r *Redis) ExecutedTasksCount() (uint64, error) { - countString, err := r.Get(r.ctx, redisTasksExecutedCountKey).Result() +func (r *Redis) ExecutedTasksCount(ctx context.Context) (uint64, error) { + countString, err := r.Get(ctx, redisTasksExecutedCountKey).Result() if err != nil { return 0, err } c, err := strconv.Atoi(countString) + return uint64(c), err } diff --git a/pkg/store/redis_test.go b/pkg/store/redis_test.go index d194613c..c8d509a0 100644 --- a/pkg/store/redis_test.go +++ b/pkg/store/redis_test.go @@ -32,40 +32,40 @@ func TestRedisProjectFunctions(t *testing.T) { p.OutputSparseStatusMetrics = false // Set project - r.SetProject(p) - projects, err := r.Projects() + r.SetProject(testCtx, p) + projects, err := r.Projects(testCtx) assert.NoError(t, err) assert.Contains(t, projects, p.Key()) assert.Equal(t, p, projects[p.Key()]) // Project exists - exists, err := r.ProjectExists(p.Key()) + exists, err := r.ProjectExists(testCtx, p.Key()) assert.NoError(t, err) assert.True(t, exists) // GetProject should succeed newProject := schemas.NewProject("foo/bar") - assert.NoError(t, r.GetProject(&newProject)) + assert.NoError(t, r.GetProject(testCtx, &newProject)) assert.Equal(t, p, newProject) // Count - count, err := r.ProjectsCount() + count, err := r.ProjectsCount(testCtx) assert.NoError(t, err) assert.Equal(t, int64(1), count) // Delete project - r.DelProject(p.Key()) - projects, err = r.Projects() + r.DelProject(testCtx, p.Key()) + projects, err = r.Projects(testCtx) assert.NoError(t, err) assert.NotContains(t, projects, p.Key()) - exists, err = r.ProjectExists(p.Key()) + exists, err = r.ProjectExists(testCtx, p.Key()) assert.NoError(t, err) assert.False(t, exists) // GetProject should not update the var this time newProject = schemas.NewProject("foo/bar") - assert.NoError(t, r.GetProject(&newProject)) + assert.NoError(t, r.GetProject(testCtx, &newProject)) assert.NotEqual(t, p, newProject) } @@ -79,15 +79,15 @@ func TestRedisEnvironmentFunctions(t *testing.T) { } // Set project - r.SetEnvironment(environment) - environments, err := r.Environments() + r.SetEnvironment(testCtx, environment) + environments, err := r.Environments(testCtx) assert.NoError(t, err) assert.Contains(t, environments, environment.Key()) assert.Equal(t, environment.ProjectName, environments[environment.Key()].ProjectName) assert.Equal(t, environment.ID, environments[environment.Key()].ID) // Environment exists - exists, err := r.EnvironmentExists(environment.Key()) + exists, err := r.EnvironmentExists(testCtx, environment.Key()) assert.NoError(t, err) assert.True(t, exists) @@ -96,21 +96,21 @@ func TestRedisEnvironmentFunctions(t *testing.T) { ProjectName: "foo", ID: 1, } - assert.NoError(t, r.GetEnvironment(&newEnvironment)) + assert.NoError(t, r.GetEnvironment(testCtx, &newEnvironment)) assert.Equal(t, environment.ExternalURL, newEnvironment.ExternalURL) // Count - count, err := r.EnvironmentsCount() + count, err := r.EnvironmentsCount(testCtx) assert.NoError(t, err) assert.Equal(t, int64(1), count) // Delete Environment - r.DelEnvironment(environment.Key()) - environments, err = r.Environments() + r.DelEnvironment(testCtx, environment.Key()) + environments, err = r.Environments(testCtx) assert.NoError(t, err) assert.NotContains(t, environments, environment.Key()) - exists, err = r.EnvironmentExists(environment.Key()) + exists, err = r.EnvironmentExists(testCtx, environment.Key()) assert.NoError(t, err) assert.False(t, exists) @@ -119,7 +119,7 @@ func TestRedisEnvironmentFunctions(t *testing.T) { ProjectName: "foo", ID: 1, } - assert.NoError(t, r.GetEnvironment(&newEnvironment)) + assert.NoError(t, r.GetEnvironment(testCtx, &newEnvironment)) assert.NotEqual(t, environment, newEnvironment) } @@ -135,14 +135,14 @@ func TestRedisRefFunctions(t *testing.T) { ) // Set project - r.SetRef(ref) - projectsRefs, err := r.Refs() + r.SetRef(testCtx, ref) + projectsRefs, err := r.Refs(testCtx) assert.NoError(t, err) assert.Contains(t, projectsRefs, ref.Key()) assert.Equal(t, ref, projectsRefs[ref.Key()]) // Ref exists - exists, err := r.RefExists(ref.Key()) + exists, err := r.RefExists(testCtx, ref.Key()) assert.NoError(t, err) assert.True(t, exists) @@ -152,21 +152,21 @@ func TestRedisRefFunctions(t *testing.T) { Kind: schemas.RefKindBranch, Name: "sweet", } - assert.NoError(t, r.GetRef(&newRef)) + assert.NoError(t, r.GetRef(testCtx, &newRef)) assert.Equal(t, ref, newRef) // Count - count, err := r.RefsCount() + count, err := r.RefsCount(testCtx) assert.NoError(t, err) assert.Equal(t, int64(1), count) // Delete Ref - r.DelRef(ref.Key()) - projectsRefs, err = r.Refs() + r.DelRef(testCtx, ref.Key()) + projectsRefs, err = r.Refs(testCtx) assert.NoError(t, err) assert.NotContains(t, projectsRefs, ref.Key()) - exists, err = r.RefExists(ref.Key()) + exists, err = r.RefExists(testCtx, ref.Key()) assert.NoError(t, err) assert.False(t, exists) @@ -176,7 +176,7 @@ func TestRedisRefFunctions(t *testing.T) { Project: schemas.NewProject("foo/bar"), Name: "sweet", } - assert.NoError(t, r.GetRef(&newRef)) + assert.NoError(t, r.GetRef(testCtx, &newRef)) assert.NotEqual(t, ref, newRef) } @@ -192,14 +192,14 @@ func TestRedisMetricFunctions(t *testing.T) { } // Set metric - r.SetMetric(m) - metrics, err := r.Metrics() + r.SetMetric(testCtx, m) + metrics, err := r.Metrics(testCtx) assert.NoError(t, err) assert.Contains(t, metrics, m.Key()) assert.Equal(t, m, metrics[m.Key()]) // Metric exists - exists, err := r.MetricExists(m.Key()) + exists, err := r.MetricExists(testCtx, m.Key()) assert.NoError(t, err) assert.True(t, exists) @@ -210,21 +210,21 @@ func TestRedisMetricFunctions(t *testing.T) { "foo": "bar", }, } - assert.NoError(t, r.GetMetric(&newMetric)) + assert.NoError(t, r.GetMetric(testCtx, &newMetric)) assert.Equal(t, m, newMetric) // Count - count, err := r.MetricsCount() + count, err := r.MetricsCount(testCtx) assert.NoError(t, err) assert.Equal(t, int64(1), count) // Delete Metric - r.DelMetric(m.Key()) - metrics, err = r.Metrics() + r.DelMetric(testCtx, m.Key()) + metrics, err = r.Metrics(testCtx) assert.NoError(t, err) assert.NotContains(t, metrics, m.Key()) - exists, err = r.MetricExists(m.Key()) + exists, err = r.MetricExists(testCtx, m.Key()) assert.NoError(t, err) assert.False(t, exists) @@ -235,28 +235,25 @@ func TestRedisMetricFunctions(t *testing.T) { "foo": "bar", }, } - assert.NoError(t, r.GetMetric(&newMetric)) + assert.NoError(t, r.GetMetric(testCtx, &newMetric)) assert.NotEqual(t, m, newMetric) } -func TestGetRedisKeepaliveKey(t *testing.T) { - assert.Equal(t, "keepalive:foo", getRedisKeepaliveKey("foo")) -} - func TestRedisKeepalive(t *testing.T) { mr, r := newTestRedisStore(t) uuidString := uuid.New().String() - resp, err := r.(*Redis).SetKeepalive(uuidString, time.Second) + resp, err := r.(*Redis).SetKeepalive(testCtx, uuidString, time.Second) assert.True(t, resp) assert.NoError(t, err) - resp, err = r.(*Redis).KeepaliveExists(uuidString) + resp, err = r.(*Redis).KeepaliveExists(testCtx, uuidString) assert.True(t, resp) assert.NoError(t, err) mr.FastForward(2 * time.Second) - resp, err = r.(*Redis).KeepaliveExists(uuidString) + + resp, err = r.(*Redis).KeepaliveExists(testCtx, uuidString) assert.False(t, resp) assert.NoError(t, err) } @@ -268,19 +265,21 @@ func TestGetRedisQueueKey(t *testing.T) { func TestRedisQueueTask(t *testing.T) { mr, r := newTestRedisStore(t) - r.(*Redis).SetKeepalive("controller1", time.Second) - ok, err := r.QueueTask(schemas.TaskTypePullMetrics, "foo", "controller1") + r.(*Redis).SetKeepalive(testCtx, "controller1", time.Second) + + ok, err := r.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "controller1") assert.True(t, ok) assert.NoError(t, err) // The keepalive of controller1 not being expired, we should not requeue the task - ok, err = r.QueueTask(schemas.TaskTypePullMetrics, "foo", "controller2") + ok, err = r.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "controller2") assert.False(t, ok) assert.NoError(t, err) // The keepalive of controller1 being expired, we should requeue the task mr.FastForward(2 * time.Second) - ok, err = r.QueueTask(schemas.TaskTypePullMetrics, "foo", "controller2") + + ok, err = r.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "controller2") assert.True(t, ok) assert.NoError(t, err) } @@ -288,37 +287,37 @@ func TestRedisQueueTask(t *testing.T) { func TestRedisUnqueueTask(t *testing.T) { _, r := newTestRedisStore(t) - r.QueueTask(schemas.TaskTypePullMetrics, "foo", "") - count, _ := r.ExecutedTasksCount() + r.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "") + count, _ := r.ExecutedTasksCount(testCtx) assert.Equal(t, uint64(0), count) - assert.NoError(t, r.UnqueueTask(schemas.TaskTypePullMetrics, "foo")) - count, _ = r.ExecutedTasksCount() + assert.NoError(t, r.UnqueueTask(testCtx, schemas.TaskTypePullMetrics, "foo")) + count, _ = r.ExecutedTasksCount(testCtx) assert.Equal(t, uint64(1), count) } func TestRedisCurrentlyQueuedTasksCount(t *testing.T) { _, r := newTestRedisStore(t) - r.QueueTask(schemas.TaskTypePullMetrics, "foo", "") - r.QueueTask(schemas.TaskTypePullMetrics, "bar", "") - r.QueueTask(schemas.TaskTypePullMetrics, "baz", "") + r.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "") + r.QueueTask(testCtx, schemas.TaskTypePullMetrics, "bar", "") + r.QueueTask(testCtx, schemas.TaskTypePullMetrics, "baz", "") - count, _ := r.CurrentlyQueuedTasksCount() + count, _ := r.CurrentlyQueuedTasksCount(testCtx) assert.Equal(t, uint64(3), count) - r.UnqueueTask(schemas.TaskTypePullMetrics, "foo") - count, _ = r.CurrentlyQueuedTasksCount() + r.UnqueueTask(testCtx, schemas.TaskTypePullMetrics, "foo") + count, _ = r.CurrentlyQueuedTasksCount(testCtx) assert.Equal(t, uint64(2), count) } func TestRedisExecutedTasksCount(t *testing.T) { _, r := newTestRedisStore(t) - r.QueueTask(schemas.TaskTypePullMetrics, "foo", "") - r.QueueTask(schemas.TaskTypePullMetrics, "bar", "") - r.UnqueueTask(schemas.TaskTypePullMetrics, "foo") - r.UnqueueTask(schemas.TaskTypePullMetrics, "foo") + r.QueueTask(testCtx, schemas.TaskTypePullMetrics, "foo", "") + r.QueueTask(testCtx, schemas.TaskTypePullMetrics, "bar", "") + r.UnqueueTask(testCtx, schemas.TaskTypePullMetrics, "foo") + r.UnqueueTask(testCtx, schemas.TaskTypePullMetrics, "foo") - count, _ := r.ExecutedTasksCount() + count, _ := r.ExecutedTasksCount(testCtx) assert.Equal(t, uint64(1), count) } diff --git a/pkg/store/store.go b/pkg/store/store.go index cb379fdf..8d988bb4 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -11,37 +11,37 @@ import ( // Store .. type Store interface { - SetProject(schemas.Project) error - DelProject(schemas.ProjectKey) error - GetProject(*schemas.Project) error - ProjectExists(schemas.ProjectKey) (bool, error) - Projects() (schemas.Projects, error) - ProjectsCount() (int64, error) - SetEnvironment(schemas.Environment) error - DelEnvironment(schemas.EnvironmentKey) error - GetEnvironment(*schemas.Environment) error - EnvironmentExists(schemas.EnvironmentKey) (bool, error) - Environments() (schemas.Environments, error) - EnvironmentsCount() (int64, error) - SetRef(schemas.Ref) error - DelRef(schemas.RefKey) error - GetRef(*schemas.Ref) error - RefExists(schemas.RefKey) (bool, error) - Refs() (schemas.Refs, error) - RefsCount() (int64, error) - SetMetric(schemas.Metric) error - DelMetric(schemas.MetricKey) error - GetMetric(*schemas.Metric) error - MetricExists(schemas.MetricKey) (bool, error) - Metrics() (schemas.Metrics, error) - MetricsCount() (int64, error) + SetProject(context.Context, schemas.Project) error + DelProject(context.Context, schemas.ProjectKey) error + GetProject(context.Context, *schemas.Project) error + ProjectExists(context.Context, schemas.ProjectKey) (bool, error) + Projects(context.Context) (schemas.Projects, error) + ProjectsCount(context.Context) (int64, error) + SetEnvironment(context.Context, schemas.Environment) error + DelEnvironment(context.Context, schemas.EnvironmentKey) error + GetEnvironment(context.Context, *schemas.Environment) error + EnvironmentExists(context.Context, schemas.EnvironmentKey) (bool, error) + Environments(context.Context) (schemas.Environments, error) + EnvironmentsCount(context.Context) (int64, error) + SetRef(context.Context, schemas.Ref) error + DelRef(context.Context, schemas.RefKey) error + GetRef(context.Context, *schemas.Ref) error + RefExists(context.Context, schemas.RefKey) (bool, error) + Refs(context.Context) (schemas.Refs, error) + RefsCount(context.Context) (int64, error) + SetMetric(context.Context, schemas.Metric) error + DelMetric(context.Context, schemas.MetricKey) error + GetMetric(context.Context, *schemas.Metric) error + MetricExists(context.Context, schemas.MetricKey) (bool, error) + Metrics(context.Context) (schemas.Metrics, error) + MetricsCount(context.Context) (int64, error) // Helpers to keep track of currently queued tasks and avoid scheduling them // twice at the risk of ending up with loads of dangling goroutines being locked - QueueTask(schemas.TaskType, string, string) (bool, error) - UnqueueTask(schemas.TaskType, string) error - CurrentlyQueuedTasksCount() (uint64, error) - ExecutedTasksCount() (uint64, error) + QueueTask(context.Context, schemas.TaskType, string, string) (bool, error) + UnqueueTask(context.Context, schemas.TaskType, string) error + CurrentlyQueuedTasksCount(context.Context) (uint64, error) + ExecutedTasksCount(context.Context) (uint64, error) } // NewLocalStore .. @@ -58,12 +58,11 @@ func NewLocalStore() Store { func NewRedisStore(client *redis.Client) Store { return &Redis{ Client: client, - ctx: context.TODO(), } } // New creates a new store and populates it with -// provided []schemas.Project +// provided []schemas.Project. func New( r *redis.Client, projects config.Projects, @@ -77,7 +76,8 @@ func New( // Load all the configured projects in the store for _, p := range projects { sp := schemas.Project{Project: p} - exists, err := s.ProjectExists(sp.Key()) + + exists, err := s.ProjectExists(context.TODO(), sp.Key()) if err != nil { log.WithFields(log.Fields{ "project-name": p.Name, @@ -86,7 +86,7 @@ func New( } if !exists { - if err = s.SetProject(sp); err != nil { + if err = s.SetProject(context.TODO(), sp); err != nil { log.WithFields(log.Fields{ "project-name": p.Name, "error": err.Error(), diff --git a/pkg/store/store_test.go b/pkg/store/store_test.go index 19787faf..6edbe523 100644 --- a/pkg/store/store_test.go +++ b/pkg/store/store_test.go @@ -10,6 +10,8 @@ import ( "github.com/stretchr/testify/assert" ) +var testCtx = context.Background() + func TestNewLocalStore(t *testing.T) { expectedValue := &Local{ projects: make(schemas.Projects), @@ -24,7 +26,6 @@ func TestNewRedisStore(t *testing.T) { redisClient := redis.NewClient(&redis.Options{}) expectedValue := &Redis{ Client: redisClient, - ctx: context.TODO(), } assert.Equal(t, expectedValue, NewRedisStore(redisClient)) @@ -49,6 +50,6 @@ func TestNew(t *testing.T) { Name: "bar", }, }) - count, _ := localStore.ProjectsCount() + count, _ := localStore.ProjectsCount(testCtx) assert.Equal(t, int64(2), count) }