diff --git a/.github/workflows/tracks.yml b/.github/workflows/tracks.yml index c78b4150..5185caf3 100644 --- a/.github/workflows/tracks.yml +++ b/.github/workflows/tracks.yml @@ -102,6 +102,10 @@ jobs: track: elastic/logs flags: --test-mode --track-params="number_of_replicas:0" --challenge=many-shards-quantitative stack-version: "8.3.2" + - name: elastic/logs (many-shards-snapshots) + track: elastic/logs + flags: --test-mode --track-params="number_of_replicas:0,snapshot_counts:1" --challenge=many-shards-snapshots + stack-version: "8.3.2" steps: - name: Checkout code diff --git a/elastic/logs/challenges/many-shards-snapshots.json b/elastic/logs/challenges/many-shards-snapshots.json new file mode 100644 index 00000000..03a44299 --- /dev/null +++ b/elastic/logs/challenges/many-shards-snapshots.json @@ -0,0 +1,65 @@ +{% import "rally.helpers" as rally %} +{ + "name": "many-shards-snapshots", + "description": "Indexes a large amount of shards and then sequentially takes a configurable number of snapshots. Used for benchmarks to help identify regressions related to snapshots with high index counts.", + "schedule": [ + {% include "tasks/many-shards-setup.json" %}, + { + "name": "add-many-shards-ilm", + "operation": { + "operation-type": "create-ilm", + "param-source": "add-track-path", + "policies": "ilm/many-shards-quantitative" + } + }, + { + "name": "initial-indices", + "operation": { + "operation-type": "raw-bulk", + "param-source": "initial-indices-source", + "detailed-results": true, + "initial_indices_count": {{ p_initial_indices_count }}, + "name": "auditbeatquantitative", + "timeout": "10m" + }, + "clients": {{ p_bulk_indexing_clients }} + }, + { + "name": "register-snapshot-repository", + "operation": { + "operation-type": "create-snapshot-repository", + "repository": {{ p_snapshot_repo_name | tojson }}, + "body": { + "type": {{ p_snapshot_repo_type | tojson }}, + "settings": {{ p_snapshot_repo_settings | tojson(indent=2)}} + } + } + }, + {# Issue number of back to back snapshots and wait until all have completed #} + {% for snap_iter in range(snapshot_counts | default(100)) %} + { + "name": "create-snapshot-{{ snap_iter }}", + "operation": { + "operation-type": "create-snapshot", + "repository": {{ p_snapshot_repo_name | tojson }}, + "snapshot": "{{ p_snapshot_name }}-{{ snap_iter }}", + "wait-for-completion": false, + "body": { + "indices": {{ p_restore_data_streams | tojson }}, + "ignore_unavailable": true, + "include_global_state": false, + "metadata": {{ p_snapshot_metadata | tojson }} + } + } + }, + {% endfor %} + { + "name": "wait-for-snapshots", + "operation": { + "operation-type": "wait-for-current-snapshots-create", + "repository": {{ p_snapshot_repo_name | tojson }} + } + }, + {% include "tasks/field-caps.json" %} + ] +}