Many shards snapshots challenge for elastic/logs (#294)

This commit adds a new challenge for many-shards that benchmarks ES
performance while taking several (specified by the track parameter
`snapshot_counts`) snapshots.

The intention is to initially use it for tracking regressions by
visualizing the `service_time` of the `wait-for-snapshots` task.

This PR depends uses the Rally operation
`wait-current-snapshots-create` operation (introduced in
elastic/rally#1542).

.github/workflows/tracks.yml

@@ -102,6 +102,10 @@ jobs:
             track: elastic/logs
             flags: --test-mode --track-params="number_of_replicas:0" --challenge=many-shards-quantitative
             stack-version: "8.3.2"
+          - name: elastic/logs (many-shards-snapshots)
+            track: elastic/logs
+            flags: --test-mode --track-params="number_of_replicas:0,snapshot_counts:1" --challenge=many-shards-snapshots
+            stack-version: "8.3.2"
 
     steps:
       - name: Checkout code

elastic/logs/challenges/many-shards-snapshots.json

@@ -0,0 +1,65 @@
+{% import "rally.helpers" as rally %}
+{
+  "name": "many-shards-snapshots",
+  "description": "Indexes a large amount of shards and then sequentially takes a configurable number of snapshots. Used for benchmarks to help identify regressions related to snapshots with high index counts.",
+  "schedule": [
+    {% include "tasks/many-shards-setup.json" %},
+    {
+      "name": "add-many-shards-ilm",
+      "operation": {
+        "operation-type": "create-ilm",
+        "param-source": "add-track-path",
+        "policies": "ilm/many-shards-quantitative"
+      }
+    },
+    {
+      "name": "initial-indices",
+      "operation": {
+        "operation-type": "raw-bulk",
+        "param-source": "initial-indices-source",
+        "detailed-results": true,
+        "initial_indices_count": {{ p_initial_indices_count }},
+        "name": "auditbeatquantitative",
+        "timeout": "10m"
+      },
+      "clients": {{ p_bulk_indexing_clients }}
+    },
+    {
+      "name": "register-snapshot-repository",
+      "operation": {
+        "operation-type": "create-snapshot-repository",
+        "repository": {{ p_snapshot_repo_name | tojson }},
+        "body": {
+          "type": {{ p_snapshot_repo_type | tojson }},
+          "settings": {{ p_snapshot_repo_settings | tojson(indent=2)}}
+        }
+      }
+    },
+    {# Issue <snapshot_counts> number of back to back snapshots and wait until all have completed #}
+      {% for snap_iter in range(snapshot_counts | default(100)) %}
+      {
+        "name": "create-snapshot-{{ snap_iter }}",
+        "operation": {
+          "operation-type": "create-snapshot",
+          "repository": {{ p_snapshot_repo_name | tojson }},
+          "snapshot": "{{ p_snapshot_name  }}-{{ snap_iter }}",
+          "wait-for-completion": false,
+          "body": {
+            "indices": {{ p_restore_data_streams | tojson }},
+            "ignore_unavailable": true,
+            "include_global_state": false,
+            "metadata": {{ p_snapshot_metadata | tojson }}
+          }
+        }
+      },
+      {% endfor %}
+      {
+        "name": "wait-for-snapshots",
+        "operation": {
+          "operation-type": "wait-for-current-snapshots-create",
+          "repository": {{ p_snapshot_repo_name | tojson }}
+        }
+      },
+    {% include "tasks/field-caps.json" %}
+  ]
+}