From 90444f156508db4457a4025dcd7851a63ec5aaa8 Mon Sep 17 00:00:00 2001 From: Maxime VISONNEAU Date: Mon, 22 May 2023 11:06:30 +0200 Subject: [PATCH] made the jobs queue bufferSize configurable --- CHANGELOG.md | 24 ++++++++++++++++++++++++ docs/configuration_syntax.md | 11 +++++++++++ pkg/config/config.go | 10 ++++++++++ pkg/config/config_test.go | 1 + pkg/controller/controller.go | 2 +- pkg/controller/scheduler.go | 6 ++---- 6 files changed, 49 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 785f7a93..56e583eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,30 @@ and this project adheres to [0ver](https://0ver.org) (more or less). ## [Unreleased] +### Added + +- new metrics: + - `gitlab_ci_pipeline_test_report_total_time` -> Duration in seconds of all the tests in the most recently finished pipeline + - `gitlab_ci_pipeline_test_report_total_count` -> Number of total tests in the most recently finished pipeline + - `gitlab_ci_pipeline_test_report_success_count` -> Number of successful tests in the most recently finished pipeline + - `gitlab_ci_pipeline_test_report_failed_count` -> Number of failed tests in the most recently finished pipeline + - `gitlab_ci_pipeline_test_report_skipped_count` -> Number of skipped tests in the most recently finished pipeline + - `gitlab_ci_pipeline_test_report_error_count` -> Number of errored tests in the most recently finished pipeline + - `gitlab_ci_pipeline_test_suite_total_time` -> Duration in seconds for the test suite + - `gitlab_ci_pipeline_test_suite_total_count` -> Number of total tests for the test suite + - `gitlab_ci_pipeline_test_suite_success_count` -> Number of successful tests for the test suite + - `gitlab_ci_pipeline_test_suite_failed_count` -> Number of failed tests for the test suite + - `gitlab_ci_pipeline_test_suite_skipped_count` -> Number of skipped tests for the test suite + - `gitlab_ci_pipeline_test_suite_error_count` -> Duration in errored tests for the test suite +- new configuration parameter: `gitlab.burstable_requests_per_second`, introducing a burstable amount of API RPS +- new configuration parameter: `gitlab.maximum_jobs_queue_size`, controlling the queue buffer size + +### Changed + +- Upgraded golang to **v1.20** +- Upgraded most dependencies to their latest versions +- Reduced the amount of data being pulled from the project list API calls + ## [v0.5.4] - 2022-08-25 ### Added diff --git a/docs/configuration_syntax.md b/docs/configuration_syntax.md index 799b36de..a70e8cfe 100644 --- a/docs/configuration_syntax.md +++ b/docs/configuration_syntax.md @@ -94,6 +94,17 @@ gitlab: # (optional, default: 5) burstable_requests_per_second: 5 + # Maximum amount of jobs to keep queue, if this limit is reached + # newly created ones will get dropped. As a best practice you should not change this value. + # Workarounds to avoid hitting the limit are: + # - increase polling intervals + # - increase API rate limit + # - reduce the amount of projects, refs, environments or metrics you are looking into + # - leverage webhooks instead of polling schedules + # + # (optional, default: 1000) + maximum_jobs_queue_size: 1000 + pull: projects_from_wildcards: # Whether to trigger a discovery or not when the diff --git a/pkg/config/config.go b/pkg/config/config.go index e90fb728..4073a8a6 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -114,6 +114,16 @@ type Gitlab struct { // Burstable limit for the GitLab API requests/sec BurstableRequestsPerSecond int `default:"5" validate:"gte=1" yaml:"burstable_requests_per_second"` + + // Maximum amount of jobs to keep queue, if this limit is reached + // newly created ones will get dropped. As a best practice you should not change this value. + // Workarounds to avoid hitting the limit are: + // - increase polling intervals + // - increase API rate limit + // - reduce the amount of projects, refs, environments or metrics you are looking into + // - leverage webhooks instead of polling schedules + // + MaximumJobsQueueSize int `default:"1000" validate:"gte=10" yaml:"maximum_jobs_queue_size"` } // Redis .. diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index d9793ef5..200d8f17 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -24,6 +24,7 @@ func TestNew(t *testing.T) { c.Gitlab.EnableTLSVerify = true c.Gitlab.MaximumRequestsPerSecond = 1 c.Gitlab.BurstableRequestsPerSecond = 5 + c.Gitlab.MaximumJobsQueueSize = 1000 c.Pull.ProjectsFromWildcards.OnInit = true c.Pull.ProjectsFromWildcards.Scheduled = true diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 6bdad16e..8aa123e0 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -52,7 +52,7 @@ func New(ctx context.Context, cfg config.Config, version string) (c Controller, return } - c.TaskController = NewTaskController(ctx, c.Redis) + c.TaskController = NewTaskController(ctx, c.Redis, cfg.Gitlab.MaximumJobsQueueSize) c.registerTasks() c.Store = store.New(ctx, c.Redis, c.Config.Projects) diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index c3d9d8c5..7a3ee763 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -19,8 +19,6 @@ import ( "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/store" ) -const bufferSize = 1000 - // TaskController holds task related clients. type TaskController struct { Factory taskq.Factory @@ -30,7 +28,7 @@ type TaskController struct { } // NewTaskController initializes and returns a new TaskController object. -func NewTaskController(ctx context.Context, r *redis.Client) (t TaskController) { +func NewTaskController(ctx context.Context, r *redis.Client, maximumJobsQueueSize int) (t TaskController) { ctx, span := otel.Tracer(tracerName).Start(ctx, "controller:NewTaskController") defer span.End() @@ -40,7 +38,7 @@ func NewTaskController(ctx context.Context, r *redis.Client) (t TaskController) Name: "default", PauseErrorsThreshold: 3, Handler: t.TaskMap, - BufferSize: bufferSize, + BufferSize: maximumJobsQueueSize, } if r != nil {