From f9704fcd5cb8d9e270131d02192d529a8907e2a8 Mon Sep 17 00:00:00 2001
From: dblock <dblock@amazon.com>
Date: Wed, 7 Aug 2024 07:54:17 -0400
Subject: [PATCH] Split test suite with separate docker-compose.

Signed-off-by: dblock <dblock@amazon.com>
---
 .../default/docker-compose.yml                   | 13 +++++++++++++
 .../index_state_management}/docker-compose.yml   |  1 -
 .github/workflows/test-spec.yml                  | 13 ++++++++-----
 .github/workflows/test-tools-integ.yml           |  2 +-
 CHANGELOG.md                                     |  1 +
 TESTING_GUIDE.md                                 | 16 +++++++++-------
 tests/{ => default}/_core/bulk.yaml              |  2 +-
 tests/{ => default}/_core/info.yaml              |  2 +-
 tests/{ => default}/_core/mapping.yaml           |  2 +-
 tests/{ => default}/_core/reindex.yaml           |  2 +-
 tests/{ => default}/_core/reindex/pipeline.yaml  |  2 +-
 tests/{ => default}/_core/search/_source.yaml    |  2 +-
 .../_core/search/cancel_after_time_interval.yaml |  2 +-
 .../{ => default}/_core/search/geo_distance.yaml |  2 +-
 tests/{ => default}/_core/search/match.yaml      |  2 +-
 .../{ => default}/_core/search/multi_match.yaml  |  2 +-
 tests/{ => default}/_core/search/phase_took.yaml |  2 +-
 .../_core/search/rest_total_hits_as_int.yaml     |  2 +-
 .../_core/search/seq_no_primary_term.yaml        |  2 +-
 tests/{ => default}/_core/search/size.yaml       |  2 +-
 tests/{ => default}/cat/aliases.yaml             |  2 +-
 tests/{ => default}/cat/allocation.yaml          |  2 +-
 tests/{ => default}/cat/cluster_manager.yaml     |  2 +-
 tests/{ => default}/cat/count.yaml               |  2 +-
 tests/{ => default}/cat/fielddata.yaml           |  2 +-
 tests/{ => default}/cat/health.yaml              |  2 +-
 tests/{ => default}/cat/index.yaml               |  2 +-
 tests/{ => default}/cat/indices.yaml             |  2 +-
 tests/{ => default}/cat/master.yaml              |  2 +-
 tests/{ => default}/cat/nodeattrs.yaml           |  2 +-
 tests/{ => default}/cat/nodes.yaml               |  2 +-
 tests/{ => default}/cat/pending_tasks.yaml       |  2 +-
 tests/{ => default}/cat/pit_segments/all.yaml    |  2 +-
 .../cat/pit_segments/pit_segments.yaml           |  2 +-
 tests/{ => default}/cat/plugins.yaml             |  2 +-
 tests/{ => default}/cat/recovery.yaml            |  2 +-
 tests/{ => default}/cat/repositories.yaml        |  2 +-
 tests/{ => default}/cat/segment_replication.yaml |  2 +-
 tests/{ => default}/cat/segments.yaml            |  2 +-
 tests/{ => default}/cat/shards.yaml              |  2 +-
 tests/{ => default}/cat/snapshots.yaml           |  2 +-
 tests/{ => default}/cat/tasks.yaml               |  2 +-
 tests/{ => default}/cat/templates.yaml           |  2 +-
 tests/{ => default}/cat/thread_pool.yaml         |  2 +-
 .../cluster/component_template.yaml              |  2 +-
 tests/{ => default}/indices/alias/alias.yaml     |  2 +-
 tests/{ => default}/indices/aliases/aliases.yaml |  2 +-
 .../{ => default}/indices/aliases/put_alias.yaml |  2 +-
 tests/{ => default}/indices/cache.yaml           |  2 +-
 tests/{ => default}/indices/clone.yaml           |  2 +-
 tests/{ => default}/indices/close.yaml           |  2 +-
 .../indices/component_template.yaml              |  2 +-
 tests/{ => default}/indices/dangling.yaml        |  2 +-
 .../indices/data_stream/data_stream.yaml         |  2 +-
 .../indices/data_stream/rollover.yaml            |  2 +-
 .../{ => default}/indices/data_stream/stats.yaml |  2 +-
 tests/{ => default}/indices/doc.yaml             |  2 +-
 tests/{ => default}/indices/flush.yaml           |  2 +-
 tests/{ => default}/indices/forcemerge.yaml      |  2 +-
 tests/{ => default}/indices/index.yaml           |  2 +-
 tests/{ => default}/indices/index_template.yaml  |  2 +-
 tests/{ => default}/indices/mapping.yml          |  2 +-
 tests/{ => default}/indices/open.yaml            |  2 +-
 tests/{ => default}/indices/refresh.yaml         |  2 +-
 tests/{ => default}/indices/settings.yaml        |  2 +-
 tests/{ => default}/indices/shrink.yaml          |  2 +-
 tests/{ => default}/indices/split.yaml           |  2 +-
 tests/{ => default}/indices/stats.yaml           |  2 +-
 tests/{ => default}/indices/update_by_query.yaml |  2 +-
 tests/{ => default}/ingest/pipeline.yaml         |  2 +-
 tests/{ => default}/ml/model_groups.yaml         |  2 +-
 tests/{ => default}/ml/models.yaml               |  2 +-
 tests/{ => default}/nodes/info.yaml              |  2 +-
 tests/{ => default}/nodes/stats.yaml             |  2 +-
 tests/{ => default}/ppl/explain.yaml             |  2 +-
 tests/{ => default}/ppl/query.yaml               |  2 +-
 tests/{ => default}/ppl/stats.yaml               |  2 +-
 tests/{ => default}/sql/close.yaml               |  2 +-
 tests/{ => default}/sql/explain.yaml             |  2 +-
 tests/{ => default}/sql/query.yaml               |  2 +-
 tests/{ => default}/sql/settings.yaml            |  2 +-
 tests/{ => default}/sql/stats.yaml               |  2 +-
 .../nodes/plugins/index_state_management.yaml    |  2 +-
 tools/src/tester/test.ts                         |  2 +-
 84 files changed, 110 insertions(+), 92 deletions(-)
 create mode 100644 .github/opensearch-cluster/default/docker-compose.yml
 rename .github/opensearch-cluster/{ => plugins/index_state_management}/docker-compose.yml (92%)
 rename tests/{ => default}/_core/bulk.yaml (95%)
 rename tests/{ => default}/_core/info.yaml (88%)
 rename tests/{ => default}/_core/mapping.yaml (95%)
 rename tests/{ => default}/_core/reindex.yaml (97%)
 rename tests/{ => default}/_core/reindex/pipeline.yaml (97%)
 rename tests/{ => default}/_core/search/_source.yaml (98%)
 rename tests/{ => default}/_core/search/cancel_after_time_interval.yaml (90%)
 rename tests/{ => default}/_core/search/geo_distance.yaml (95%)
 rename tests/{ => default}/_core/search/match.yaml (95%)
 rename tests/{ => default}/_core/search/multi_match.yaml (94%)
 rename tests/{ => default}/_core/search/phase_took.yaml (90%)
 rename tests/{ => default}/_core/search/rest_total_hits_as_int.yaml (98%)
 rename tests/{ => default}/_core/search/seq_no_primary_term.yaml (93%)
 rename tests/{ => default}/_core/search/size.yaml (95%)
 rename tests/{ => default}/cat/aliases.yaml (93%)
 rename tests/{ => default}/cat/allocation.yaml (79%)
 rename tests/{ => default}/cat/cluster_manager.yaml (81%)
 rename tests/{ => default}/cat/count.yaml (78%)
 rename tests/{ => default}/cat/fielddata.yaml (89%)
 rename tests/{ => default}/cat/health.yaml (98%)
 rename tests/{ => default}/cat/index.yaml (76%)
 rename tests/{ => default}/cat/indices.yaml (97%)
 rename tests/{ => default}/cat/master.yaml (80%)
 rename tests/{ => default}/cat/nodeattrs.yaml (84%)
 rename tests/{ => default}/cat/nodes.yaml (89%)
 rename tests/{ => default}/cat/pending_tasks.yaml (79%)
 rename tests/{ => default}/cat/pit_segments/all.yaml (92%)
 rename tests/{ => default}/cat/pit_segments/pit_segments.yaml (93%)
 rename tests/{ => default}/cat/plugins.yaml (82%)
 rename tests/{ => default}/cat/recovery.yaml (78%)
 rename tests/{ => default}/cat/repositories.yaml (92%)
 rename tests/{ => default}/cat/segment_replication.yaml (81%)
 rename tests/{ => default}/cat/segments.yaml (78%)
 rename tests/{ => default}/cat/shards.yaml (90%)
 rename tests/{ => default}/cat/snapshots.yaml (90%)
 rename tests/{ => default}/cat/tasks.yaml (78%)
 rename tests/{ => default}/cat/templates.yaml (93%)
 rename tests/{ => default}/cat/thread_pool.yaml (89%)
 rename tests/{ => default}/cluster/component_template.yaml (97%)
 rename tests/{ => default}/indices/alias/alias.yaml (96%)
 rename tests/{ => default}/indices/aliases/aliases.yaml (93%)
 rename tests/{ => default}/indices/aliases/put_alias.yaml (96%)
 rename tests/{ => default}/indices/cache.yaml (95%)
 rename tests/{ => default}/indices/clone.yaml (95%)
 rename tests/{ => default}/indices/close.yaml (95%)
 rename tests/{ => default}/indices/component_template.yaml (98%)
 rename tests/{ => default}/indices/dangling.yaml (68%)
 rename tests/{ => default}/indices/data_stream/data_stream.yaml (95%)
 rename tests/{ => default}/indices/data_stream/rollover.yaml (93%)
 rename tests/{ => default}/indices/data_stream/stats.yaml (93%)
 rename tests/{ => default}/indices/doc.yaml (95%)
 rename tests/{ => default}/indices/flush.yaml (94%)
 rename tests/{ => default}/indices/forcemerge.yaml (96%)
 rename tests/{ => default}/indices/index.yaml (97%)
 rename tests/{ => default}/indices/index_template.yaml (97%)
 rename tests/{ => default}/indices/mapping.yml (98%)
 rename tests/{ => default}/indices/open.yaml (96%)
 rename tests/{ => default}/indices/refresh.yaml (92%)
 rename tests/{ => default}/indices/settings.yaml (97%)
 rename tests/{ => default}/indices/shrink.yaml (97%)
 rename tests/{ => default}/indices/split.yaml (97%)
 rename tests/{ => default}/indices/stats.yaml (95%)
 rename tests/{ => default}/indices/update_by_query.yaml (96%)
 rename tests/{ => default}/ingest/pipeline.yaml (94%)
 rename tests/{ => default}/ml/model_groups.yaml (95%)
 rename tests/{ => default}/ml/models.yaml (94%)
 rename tests/{ => default}/nodes/info.yaml (84%)
 rename tests/{ => default}/nodes/stats.yaml (98%)
 rename tests/{ => default}/ppl/explain.yaml (88%)
 rename tests/{ => default}/ppl/query.yaml (88%)
 rename tests/{ => default}/ppl/stats.yaml (78%)
 rename tests/{ => default}/sql/close.yaml (95%)
 rename tests/{ => default}/sql/explain.yaml (88%)
 rename tests/{ => default}/sql/query.yaml (89%)
 rename tests/{ => default}/sql/settings.yaml (94%)
 rename tests/{ => default}/sql/stats.yaml (78%)
 rename tests/{ => plugins/index_state_management}/nodes/plugins/index_state_management.yaml (90%)

diff --git a/.github/opensearch-cluster/default/docker-compose.yml b/.github/opensearch-cluster/default/docker-compose.yml
new file mode 100644
index 000000000..86cb0b9b3
--- /dev/null
+++ b/.github/opensearch-cluster/default/docker-compose.yml
@@ -0,0 +1,13 @@
+version: '3'
+
+services:
+  opensearch-cluster:
+    image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF}
+    ports:
+      - 9200:9200
+      - 9600:9600
+    environment:
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}
+      - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS}
+      - discovery.type=single-node
+      - path.repo=/tmp/opensearch/repo
diff --git a/.github/opensearch-cluster/docker-compose.yml b/.github/opensearch-cluster/plugins/index_state_management/docker-compose.yml
similarity index 92%
rename from .github/opensearch-cluster/docker-compose.yml
rename to .github/opensearch-cluster/plugins/index_state_management/docker-compose.yml
index e2ea4b2de..a9db789bc 100644
--- a/.github/opensearch-cluster/docker-compose.yml
+++ b/.github/opensearch-cluster/plugins/index_state_management/docker-compose.yml
@@ -10,5 +10,4 @@ services:
       - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}
       - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS}
       - discovery.type=single-node
-      - path.repo=/tmp/opensearch/repo
       - plugins.index_state_management.job_interval=1
\ No newline at end of file
diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml
index 0ec43c6e5..b880d93f8 100644
--- a/.github/workflows/test-spec.yml
+++ b/.github/workflows/test-spec.yml
@@ -28,6 +28,8 @@ jobs:
           - version: 2.0.0
             admin_password: admin
           - version: 2.15.0
+          - version: 2.15.0
+            tests: plugins/index_state_management
           - version: 2.16.0
             hub: opensearchstaging
             ref: '@sha256:bcd7f5d5d30231f24f266064248cc8d3306574948190f7bf93016dff29acf17e'
@@ -35,7 +37,7 @@ jobs:
             hub: opensearchstaging
             ref: '@sha256:db1918b2b8f7ef6c22dd6ff54a0640877c3d395a392a53864745024933981e3b'
 
-    name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }})
+    name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}, tests=${{ matrix.entry.tests || 'default' }})
     runs-on: ubuntu-latest
 
     env:
@@ -58,7 +60,7 @@ jobs:
         run: npm ci
 
       - name: Run OpenSearch Cluster
-        working-directory: .github/opensearch-cluster
+        working-directory: .github/opensearch-cluster/${{ matrix.entry.tests || 'default' }}
         run: docker compose up -d
 
       - name: Get Container ID
@@ -71,7 +73,8 @@ jobs:
           npm run test:spec -- \
             --opensearch-insecure \
             --opensearch-version=${{ matrix.entry.version }} \
-            --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}.json
+            --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}-${{ matrix.entry.tests || 'default' }}.json \
+            --tests=tests/${{ matrix.entry.tests || 'default' }}
 
       - name: Get Container Logs
         if: failure() && steps.container.outputs.CONTAINER_ID
@@ -82,8 +85,8 @@ jobs:
       - name: Upload Test Coverage Results
         uses: actions/upload-artifact@v4
         with:
-          name: coverage-${{ matrix.entry.version }}
-          path: coverage/test-spec-coverage-${{ matrix.entry.version }}.json
+          name: coverage-${{ matrix.entry.version }}-${{ matrix.entry.tests || 'default' }}
+          path: coverage/test-spec-coverage-${{ matrix.entry.version }}-${{ matrix.entry.tests || 'default' }}.json
 
   merge-coverage:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/test-tools-integ.yml b/.github/workflows/test-tools-integ.yml
index 915e840cd..d59369f78 100644
--- a/.github/workflows/test-tools-integ.yml
+++ b/.github/workflows/test-tools-integ.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Run OpenSearch Cluster
-        working-directory: .github/opensearch-cluster
+        working-directory: .github/opensearch-cluster/default
         run: |
           docker compose up -d
           sleep 15
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1bebc33e4..d0e0e18a2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -78,6 +78,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Tester tool now provides better context for non-2XX responses when --verbose is used ([#359](https://github.com/opensearch-project/opensearch-api-specification/pull/359))
 - Lock testing for next release of OpenSearch to a specific SHA ([#431](https://github.com/opensearch-project/opensearch-api-specification/pull/431))
 - Replace nullable with null type ([#436](https://github.com/opensearch-project/opensearch-api-specification/pull/436))
+- Split test suite ([#472])(https://github.com/opensearch-project/opensearch-api-specification/pull/472)
 
 ### Deprecated
 
diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md
index a6d4456cf..e65bd3df5 100644
--- a/TESTING_GUIDE.md
+++ b/TESTING_GUIDE.md
@@ -25,7 +25,7 @@ Set up an OpenSearch cluster with Docker:
 (Replace `<<your_password>>` with your desired password. If not provided, the default password inside the `docker-compose.yml` file will be used.)
 ```bash
 export OPENSEARCH_PASSWORD=<<your_password>>
-cd .github/opensearch-cluster
+cd .github/opensearch-cluster/default
 docker compose up -d
 ```
 
@@ -36,7 +36,7 @@ npm run test:spec -- --opensearch-insecure
 
 Run a specific test story:
 ```bash
-npm run test:spec -- --opensearch-insecure --tests tests/_core/info.yaml
+npm run test:spec -- --opensearch-insecure --tests tests/default/_core/info.yaml
 ```
 
 Verbose output:
@@ -69,7 +69,9 @@ curl -k -X PUT --user "admin:${OPENSEARCH_PASSWORD}" https://localhost:9200/_clu
 
 ## Writing Spec Tests
 
-The spec tests reside in the [tests/](tests) directory. Tests are organized in folders that match [namespaces](spec/namespaces). For example, tests for APIs defined in [spec/namespaces/indices.yaml](spec/namespaces/indices.yaml) can be found in [tests/indices/index.yaml](tests/indices/index.yaml) (for `/{index}`), and [tests/indices/doc.yaml](tests/indices/doc.yaml) (for `/{index}/_doc`).
+The spec tests reside in the [tests/](tests) directory. Tests are organized in suites ([default](tests/default/), etc.), and subsequently in folders that match [namespaces](spec/namespaces). For example, tests for APIs defined in [spec/namespaces/indices.yaml](spec/namespaces/indices.yaml) can be found in [tests/default/indices/index.yaml](tests/default/indices/index.yaml) (for `/{index}`), and [tests/default/indices/doc.yaml](tests/default/indices/doc.yaml) (for `/{index}/_doc`). 
+
+Additional suites require custom configuration that is defined in a separate `docker-compose.yml`. For example [.github/opensearch-cluster/plugins/index_state_management/docker-compose.yml](.github/opensearch-cluster/plugins/index_state_management/docker-compose.yml) uses a custom setting of `plugins.index_state_management.job_interval=1` to cause the `/_nodes` API to return plugin information tested in [tests/plugins/index_state_management/nodes/plugins/index_state_management.yaml](tests/plugins/index_state_management/nodes/plugins/index_state_management.yaml).
 
 Each yaml file in the tests directory represents a test story that tests a collection of related operations.
 
@@ -82,9 +84,9 @@ Check the [test_story JSON Schema](json_schemas/test_story.schema.yaml) for the
 
 ### Simple Test Story
 
-Below is the simplified version of the test story that tests the [index operations](tests/indices/index.yaml):
+Below is the simplified version of the test story that tests the [index operations](tests/default/indices/index.yaml):
 ```yaml
-$schema: ../json_schemas/test_story.schema.yaml # The schema of the test story. Include this line so that your editor can validate the test story on the fly.
+$schema: ../../json_schemas/test_story.schema.yaml # The schema of the test story. Include this line so that your editor can validate the test story on the fly.
 
 description: This story tests all endpoints relevant the lifecycle of an index, from creation to deletion.
 
@@ -134,7 +136,7 @@ chapters:
 
 ### Using Output from Previous Chapters
 
-Consider the following chapters in [ml/model_groups](tests/ml/model_groups.yaml) test story:
+Consider the following chapters in [ml/model_groups](tests/default/ml/model_groups.yaml) test story:
 ```yaml
   - synopsis: Create model group.
     id: create_model_group # Only needed if you want to refer to this chapter in another chapter.
@@ -167,7 +169,7 @@ Consider the following chapters in [ml/model_groups](tests/ml/model_groups.yaml)
 ```
 As you can see, the `output` field in the first chapter saves the `model_group_id` from the response body. This value is then used in the subsequent chapters to query and delete the model group.
 
-You can also reuse output in payload expectations. See [tests/nodes/plugins/index_state_management.yaml](tests/nodes/plugins/index_state_management.yaml) for an example.
+You can also reuse output in payload expectations. See [tests/plugins/index_state_management/nodes/plugins/index_state_management.yaml](tests/plugins/index_state_management/nodes/plugins/index_state_management.yaml) for an example.
 
 ### Managing Versions
 
diff --git a/tests/_core/bulk.yaml b/tests/default/_core/bulk.yaml
similarity index 95%
rename from tests/_core/bulk.yaml
rename to tests/default/_core/bulk.yaml
index abcbb438f..3ed172528 100644
--- a/tests/_core/bulk.yaml
+++ b/tests/default/_core/bulk.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test bulk endpoint.
 epilogues:
diff --git a/tests/_core/info.yaml b/tests/default/_core/info.yaml
similarity index 88%
rename from tests/_core/info.yaml
rename to tests/default/_core/info.yaml
index 6635498bc..67e245235 100644
--- a/tests/_core/info.yaml
+++ b/tests/default/_core/info.yaml
@@ -1,5 +1,5 @@
 
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test root endpoint.
 chapters:
diff --git a/tests/_core/mapping.yaml b/tests/default/_core/mapping.yaml
similarity index 95%
rename from tests/_core/mapping.yaml
rename to tests/default/_core/mapping.yaml
index 936cf021e..29c2c335d 100644
--- a/tests/_core/mapping.yaml
+++ b/tests/default/_core/mapping.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test mappings endpoints.
 prologues:
diff --git a/tests/_core/reindex.yaml b/tests/default/_core/reindex.yaml
similarity index 97%
rename from tests/_core/reindex.yaml
rename to tests/default/_core/reindex.yaml
index 2b349d341..86d4fccb4 100644
--- a/tests/_core/reindex.yaml
+++ b/tests/default/_core/reindex.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test reindex.
 epilogues:
diff --git a/tests/_core/reindex/pipeline.yaml b/tests/default/_core/reindex/pipeline.yaml
similarity index 97%
rename from tests/_core/reindex/pipeline.yaml
rename to tests/default/_core/reindex/pipeline.yaml
index cdcae8230..bd63c249d 100644
--- a/tests/_core/reindex/pipeline.yaml
+++ b/tests/default/_core/reindex/pipeline.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test reindex with a Search pipeline.
 epilogues:
diff --git a/tests/_core/search/_source.yaml b/tests/default/_core/search/_source.yaml
similarity index 98%
rename from tests/_core/search/_source.yaml
rename to tests/default/_core/search/_source.yaml
index de34a143b..e0a4c7d04 100644
--- a/tests/_core/search/_source.yaml
+++ b/tests/default/_core/search/_source.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test search endpoint (_source).
 prologues:
diff --git a/tests/_core/search/cancel_after_time_interval.yaml b/tests/default/_core/search/cancel_after_time_interval.yaml
similarity index 90%
rename from tests/_core/search/cancel_after_time_interval.yaml
rename to tests/default/_core/search/cancel_after_time_interval.yaml
index b5bd3b510..464c86a5a 100644
--- a/tests/_core/search/cancel_after_time_interval.yaml
+++ b/tests/default/_core/search/cancel_after_time_interval.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test search endpoint with cancel_after_time_interval.
 prologues:
diff --git a/tests/_core/search/geo_distance.yaml b/tests/default/_core/search/geo_distance.yaml
similarity index 95%
rename from tests/_core/search/geo_distance.yaml
rename to tests/default/_core/search/geo_distance.yaml
index 844df33c2..3beff6865 100644
--- a/tests/_core/search/geo_distance.yaml
+++ b/tests/default/_core/search/geo_distance.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test search endpoint with geo_distance query.
 prologues:
diff --git a/tests/_core/search/match.yaml b/tests/default/_core/search/match.yaml
similarity index 95%
rename from tests/_core/search/match.yaml
rename to tests/default/_core/search/match.yaml
index 2766f3ac1..978493787 100644
--- a/tests/_core/search/match.yaml
+++ b/tests/default/_core/search/match.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test search endpoint with match query.
 prologues:
diff --git a/tests/_core/search/multi_match.yaml b/tests/default/_core/search/multi_match.yaml
similarity index 94%
rename from tests/_core/search/multi_match.yaml
rename to tests/default/_core/search/multi_match.yaml
index b5c7b3c76..be83a9145 100644
--- a/tests/_core/search/multi_match.yaml
+++ b/tests/default/_core/search/multi_match.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test search endpoint with multi_match query.
 prologues:
diff --git a/tests/_core/search/phase_took.yaml b/tests/default/_core/search/phase_took.yaml
similarity index 90%
rename from tests/_core/search/phase_took.yaml
rename to tests/default/_core/search/phase_took.yaml
index 3e061ae0f..5da69c466 100644
--- a/tests/_core/search/phase_took.yaml
+++ b/tests/default/_core/search/phase_took.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test search endpoint with phase_took.
 prologues:
diff --git a/tests/_core/search/rest_total_hits_as_int.yaml b/tests/default/_core/search/rest_total_hits_as_int.yaml
similarity index 98%
rename from tests/_core/search/rest_total_hits_as_int.yaml
rename to tests/default/_core/search/rest_total_hits_as_int.yaml
index 1885bc282..3d01d28da 100644
--- a/tests/_core/search/rest_total_hits_as_int.yaml
+++ b/tests/default/_core/search/rest_total_hits_as_int.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test search endpoint with rest_total_hits_as_int.
 prologues:
diff --git a/tests/_core/search/seq_no_primary_term.yaml b/tests/default/_core/search/seq_no_primary_term.yaml
similarity index 93%
rename from tests/_core/search/seq_no_primary_term.yaml
rename to tests/default/_core/search/seq_no_primary_term.yaml
index 8082de549..8324f9850 100644
--- a/tests/_core/search/seq_no_primary_term.yaml
+++ b/tests/default/_core/search/seq_no_primary_term.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test search endpoint with seq_no_primary_term.
 prologues:
diff --git a/tests/_core/search/size.yaml b/tests/default/_core/search/size.yaml
similarity index 95%
rename from tests/_core/search/size.yaml
rename to tests/default/_core/search/size.yaml
index 64b2b8ca9..7bf68e2c2 100644
--- a/tests/_core/search/size.yaml
+++ b/tests/default/_core/search/size.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test search endpoint with size.
 prologues:
diff --git a/tests/cat/aliases.yaml b/tests/default/cat/aliases.yaml
similarity index 93%
rename from tests/cat/aliases.yaml
rename to tests/default/cat/aliases.yaml
index bef1d6bbc..30f9cebf9 100644
--- a/tests/cat/aliases.yaml
+++ b/tests/default/cat/aliases.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/aliases endpoints.
 epilogues:
diff --git a/tests/cat/allocation.yaml b/tests/default/cat/allocation.yaml
similarity index 79%
rename from tests/cat/allocation.yaml
rename to tests/default/cat/allocation.yaml
index e5d22d0a1..60d603d3b 100644
--- a/tests/cat/allocation.yaml
+++ b/tests/default/cat/allocation.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/allocation endpoints.
 chapters:
diff --git a/tests/cat/cluster_manager.yaml b/tests/default/cat/cluster_manager.yaml
similarity index 81%
rename from tests/cat/cluster_manager.yaml
rename to tests/default/cat/cluster_manager.yaml
index 3f0686d32..e984f2440 100644
--- a/tests/cat/cluster_manager.yaml
+++ b/tests/default/cat/cluster_manager.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/cluster_manager endpoints.
 chapters:
diff --git a/tests/cat/count.yaml b/tests/default/cat/count.yaml
similarity index 78%
rename from tests/cat/count.yaml
rename to tests/default/cat/count.yaml
index e5bc876ab..31e71c580 100644
--- a/tests/cat/count.yaml
+++ b/tests/default/cat/count.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/count endpoints.
 chapters:
diff --git a/tests/cat/fielddata.yaml b/tests/default/cat/fielddata.yaml
similarity index 89%
rename from tests/cat/fielddata.yaml
rename to tests/default/cat/fielddata.yaml
index 35a6721a1..ca8a11cb5 100644
--- a/tests/cat/fielddata.yaml
+++ b/tests/default/cat/fielddata.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/fielddata endpoints.
 chapters:
diff --git a/tests/cat/health.yaml b/tests/default/cat/health.yaml
similarity index 98%
rename from tests/cat/health.yaml
rename to tests/default/cat/health.yaml
index cc18865be..38865e9e5 100644
--- a/tests/cat/health.yaml
+++ b/tests/default/cat/health.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/health endpoints.
 chapters:
diff --git a/tests/cat/index.yaml b/tests/default/cat/index.yaml
similarity index 76%
rename from tests/cat/index.yaml
rename to tests/default/cat/index.yaml
index 7127fe574..ae20cdf3a 100644
--- a/tests/cat/index.yaml
+++ b/tests/default/cat/index.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat endpoints.
 chapters:
diff --git a/tests/cat/indices.yaml b/tests/default/cat/indices.yaml
similarity index 97%
rename from tests/cat/indices.yaml
rename to tests/default/cat/indices.yaml
index c8d3fcb42..c20df6fe8 100644
--- a/tests/cat/indices.yaml
+++ b/tests/default/cat/indices.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/indices endpoints.
 prologues:
diff --git a/tests/cat/master.yaml b/tests/default/cat/master.yaml
similarity index 80%
rename from tests/cat/master.yaml
rename to tests/default/cat/master.yaml
index b1ed7985b..435a27fa1 100644
--- a/tests/cat/master.yaml
+++ b/tests/default/cat/master.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/master endpoints.
 chapters:
diff --git a/tests/cat/nodeattrs.yaml b/tests/default/cat/nodeattrs.yaml
similarity index 84%
rename from tests/cat/nodeattrs.yaml
rename to tests/default/cat/nodeattrs.yaml
index 98e1f8b29..cb9dc328c 100644
--- a/tests/cat/nodeattrs.yaml
+++ b/tests/default/cat/nodeattrs.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/nodeattrs endpoints.
 chapters:
diff --git a/tests/cat/nodes.yaml b/tests/default/cat/nodes.yaml
similarity index 89%
rename from tests/cat/nodes.yaml
rename to tests/default/cat/nodes.yaml
index 919100791..158354e23 100644
--- a/tests/cat/nodes.yaml
+++ b/tests/default/cat/nodes.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/nodes endpoints.
 chapters:
diff --git a/tests/cat/pending_tasks.yaml b/tests/default/cat/pending_tasks.yaml
similarity index 79%
rename from tests/cat/pending_tasks.yaml
rename to tests/default/cat/pending_tasks.yaml
index e76ad3329..c007487ad 100644
--- a/tests/cat/pending_tasks.yaml
+++ b/tests/default/cat/pending_tasks.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/pending_tasks endpoints.
 chapters:
diff --git a/tests/cat/pit_segments/all.yaml b/tests/default/cat/pit_segments/all.yaml
similarity index 92%
rename from tests/cat/pit_segments/all.yaml
rename to tests/default/cat/pit_segments/all.yaml
index 91b4ccdd1..5e6d77632 100644
--- a/tests/cat/pit_segments/all.yaml
+++ b/tests/default/cat/pit_segments/all.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/pit_segments/_all endpoints.
 version: '>= 2.4'
diff --git a/tests/cat/pit_segments/pit_segments.yaml b/tests/default/cat/pit_segments/pit_segments.yaml
similarity index 93%
rename from tests/cat/pit_segments/pit_segments.yaml
rename to tests/default/cat/pit_segments/pit_segments.yaml
index 21ea56af0..d933ba55b 100644
--- a/tests/cat/pit_segments/pit_segments.yaml
+++ b/tests/default/cat/pit_segments/pit_segments.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/pit_segments endpoints.
 version: '>= 2.4'
diff --git a/tests/cat/plugins.yaml b/tests/default/cat/plugins.yaml
similarity index 82%
rename from tests/cat/plugins.yaml
rename to tests/default/cat/plugins.yaml
index e2d665517..383576a7b 100644
--- a/tests/cat/plugins.yaml
+++ b/tests/default/cat/plugins.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/plugins endpoints.
 chapters:
diff --git a/tests/cat/recovery.yaml b/tests/default/cat/recovery.yaml
similarity index 78%
rename from tests/cat/recovery.yaml
rename to tests/default/cat/recovery.yaml
index ab4a9647b..a1341f219 100644
--- a/tests/cat/recovery.yaml
+++ b/tests/default/cat/recovery.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/recovery endpoints.
 chapters:
diff --git a/tests/cat/repositories.yaml b/tests/default/cat/repositories.yaml
similarity index 92%
rename from tests/cat/repositories.yaml
rename to tests/default/cat/repositories.yaml
index 72cd80dce..89d258e23 100644
--- a/tests/cat/repositories.yaml
+++ b/tests/default/cat/repositories.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/repositories endpoints.
 epilogues:
diff --git a/tests/cat/segment_replication.yaml b/tests/default/cat/segment_replication.yaml
similarity index 81%
rename from tests/cat/segment_replication.yaml
rename to tests/default/cat/segment_replication.yaml
index cdc2af500..0495ece5f 100644
--- a/tests/cat/segment_replication.yaml
+++ b/tests/default/cat/segment_replication.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/segment_replication endpoints.
 version: '>= 2.7'
diff --git a/tests/cat/segments.yaml b/tests/default/cat/segments.yaml
similarity index 78%
rename from tests/cat/segments.yaml
rename to tests/default/cat/segments.yaml
index 8d46443ec..b8cbb8f56 100644
--- a/tests/cat/segments.yaml
+++ b/tests/default/cat/segments.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/segments endpoints.
 chapters:
diff --git a/tests/cat/shards.yaml b/tests/default/cat/shards.yaml
similarity index 90%
rename from tests/cat/shards.yaml
rename to tests/default/cat/shards.yaml
index 9bf960b2e..f2ab5c147 100644
--- a/tests/cat/shards.yaml
+++ b/tests/default/cat/shards.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/shards endpoints.
 epilogues:
diff --git a/tests/cat/snapshots.yaml b/tests/default/cat/snapshots.yaml
similarity index 90%
rename from tests/cat/snapshots.yaml
rename to tests/default/cat/snapshots.yaml
index 962827112..1e35ea574 100644
--- a/tests/cat/snapshots.yaml
+++ b/tests/default/cat/snapshots.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/snapshots endpoints.
 epilogues:
diff --git a/tests/cat/tasks.yaml b/tests/default/cat/tasks.yaml
similarity index 78%
rename from tests/cat/tasks.yaml
rename to tests/default/cat/tasks.yaml
index dbde807f1..aedcaab42 100644
--- a/tests/cat/tasks.yaml
+++ b/tests/default/cat/tasks.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/tasks endpoints.
 chapters:
diff --git a/tests/cat/templates.yaml b/tests/default/cat/templates.yaml
similarity index 93%
rename from tests/cat/templates.yaml
rename to tests/default/cat/templates.yaml
index 3f233a76b..330b6c12c 100644
--- a/tests/cat/templates.yaml
+++ b/tests/default/cat/templates.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/templates endpoints.
 epilogues:
diff --git a/tests/cat/thread_pool.yaml b/tests/default/cat/thread_pool.yaml
similarity index 89%
rename from tests/cat/thread_pool.yaml
rename to tests/default/cat/thread_pool.yaml
index 549286655..b38cd8772 100644
--- a/tests/cat/thread_pool.yaml
+++ b/tests/default/cat/thread_pool.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cat/thread_pool endpoints.
 chapters:
diff --git a/tests/cluster/component_template.yaml b/tests/default/cluster/component_template.yaml
similarity index 97%
rename from tests/cluster/component_template.yaml
rename to tests/default/cluster/component_template.yaml
index 783393591..0d8879d4f 100644
--- a/tests/cluster/component_template.yaml
+++ b/tests/default/cluster/component_template.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test component templates.
 epilogues:
diff --git a/tests/indices/alias/alias.yaml b/tests/default/indices/alias/alias.yaml
similarity index 96%
rename from tests/indices/alias/alias.yaml
rename to tests/default/indices/alias/alias.yaml
index 09962aff6..2fbba5383 100644
--- a/tests/indices/alias/alias.yaml
+++ b/tests/default/indices/alias/alias.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test alias endpoints.
 epilogues:
diff --git a/tests/indices/aliases/aliases.yaml b/tests/default/indices/aliases/aliases.yaml
similarity index 93%
rename from tests/indices/aliases/aliases.yaml
rename to tests/default/indices/aliases/aliases.yaml
index 22bcd95f1..805510ae4 100644
--- a/tests/indices/aliases/aliases.yaml
+++ b/tests/default/indices/aliases/aliases.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test aliases endpoints.
 epilogues:
diff --git a/tests/indices/aliases/put_alias.yaml b/tests/default/indices/aliases/put_alias.yaml
similarity index 96%
rename from tests/indices/aliases/put_alias.yaml
rename to tests/default/indices/aliases/put_alias.yaml
index 7a7099efa..d7eaa5c99 100644
--- a/tests/indices/aliases/put_alias.yaml
+++ b/tests/default/indices/aliases/put_alias.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test put_alias variants.
 epilogues:
diff --git a/tests/indices/cache.yaml b/tests/default/indices/cache.yaml
similarity index 95%
rename from tests/indices/cache.yaml
rename to tests/default/indices/cache.yaml
index d88e81497..db4db9dd5 100644
--- a/tests/indices/cache.yaml
+++ b/tests/default/indices/cache.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test index clear cache.
 prologues:
diff --git a/tests/indices/clone.yaml b/tests/default/indices/clone.yaml
similarity index 95%
rename from tests/indices/clone.yaml
rename to tests/default/indices/clone.yaml
index dfc56119c..e2285c536 100644
--- a/tests/indices/clone.yaml
+++ b/tests/default/indices/clone.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test cloning an index.
 prologues:
diff --git a/tests/indices/close.yaml b/tests/default/indices/close.yaml
similarity index 95%
rename from tests/indices/close.yaml
rename to tests/default/indices/close.yaml
index ca0a517d9..c8b9e8989 100644
--- a/tests/indices/close.yaml
+++ b/tests/default/indices/close.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test closing an index.
 prologues:
diff --git a/tests/indices/component_template.yaml b/tests/default/indices/component_template.yaml
similarity index 98%
rename from tests/indices/component_template.yaml
rename to tests/default/indices/component_template.yaml
index d05b88f4c..ff7b2d39f 100644
--- a/tests/indices/component_template.yaml
+++ b/tests/default/indices/component_template.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test component templates.
 epilogues:
diff --git a/tests/indices/dangling.yaml b/tests/default/indices/dangling.yaml
similarity index 68%
rename from tests/indices/dangling.yaml
rename to tests/default/indices/dangling.yaml
index 5accfcce8..94087d494 100644
--- a/tests/indices/dangling.yaml
+++ b/tests/default/indices/dangling.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test dangling indexes.
 chapters:
diff --git a/tests/indices/data_stream/data_stream.yaml b/tests/default/indices/data_stream/data_stream.yaml
similarity index 95%
rename from tests/indices/data_stream/data_stream.yaml
rename to tests/default/indices/data_stream/data_stream.yaml
index 051451286..b03845e21 100644
--- a/tests/indices/data_stream/data_stream.yaml
+++ b/tests/default/indices/data_stream/data_stream.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test data streams.
 prologues:
diff --git a/tests/indices/data_stream/rollover.yaml b/tests/default/indices/data_stream/rollover.yaml
similarity index 93%
rename from tests/indices/data_stream/rollover.yaml
rename to tests/default/indices/data_stream/rollover.yaml
index f8640db10..5e0c637be 100644
--- a/tests/indices/data_stream/rollover.yaml
+++ b/tests/default/indices/data_stream/rollover.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test data streams rollover.
 prologues:
diff --git a/tests/indices/data_stream/stats.yaml b/tests/default/indices/data_stream/stats.yaml
similarity index 93%
rename from tests/indices/data_stream/stats.yaml
rename to tests/default/indices/data_stream/stats.yaml
index 8052ff670..b8cbcd0bf 100644
--- a/tests/indices/data_stream/stats.yaml
+++ b/tests/default/indices/data_stream/stats.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../json_schemas/test_story.schema.yaml
 
 description: Test data streams.
 prologues:
diff --git a/tests/indices/doc.yaml b/tests/default/indices/doc.yaml
similarity index 95%
rename from tests/indices/doc.yaml
rename to tests/default/indices/doc.yaml
index 325602d75..6eb5a8a5d 100644
--- a/tests/indices/doc.yaml
+++ b/tests/default/indices/doc.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test inserting and retrieving a doc.
 epilogues:
diff --git a/tests/indices/flush.yaml b/tests/default/indices/flush.yaml
similarity index 94%
rename from tests/indices/flush.yaml
rename to tests/default/indices/flush.yaml
index a38c4c65b..c2afb3af4 100644
--- a/tests/indices/flush.yaml
+++ b/tests/default/indices/flush.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test flushing an index.
 prologues:
diff --git a/tests/indices/forcemerge.yaml b/tests/default/indices/forcemerge.yaml
similarity index 96%
rename from tests/indices/forcemerge.yaml
rename to tests/default/indices/forcemerge.yaml
index 0e884d1e5..d537e3a7c 100644
--- a/tests/indices/forcemerge.yaml
+++ b/tests/default/indices/forcemerge.yaml
@@ -1,5 +1,5 @@
 
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test force merging an index.
 prologues:
diff --git a/tests/indices/index.yaml b/tests/default/indices/index.yaml
similarity index 97%
rename from tests/indices/index.yaml
rename to tests/default/indices/index.yaml
index 186b7df8f..d30186c8a 100644
--- a/tests/indices/index.yaml
+++ b/tests/default/indices/index.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test endpoints relevant the lifecycle of an index, from creation to deletion.
 epilogues:
diff --git a/tests/indices/index_template.yaml b/tests/default/indices/index_template.yaml
similarity index 97%
rename from tests/indices/index_template.yaml
rename to tests/default/indices/index_template.yaml
index 09a6ce7f4..a3d5aa463 100644
--- a/tests/indices/index_template.yaml
+++ b/tests/default/indices/index_template.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test index templates.
 epilogues:
diff --git a/tests/indices/mapping.yml b/tests/default/indices/mapping.yml
similarity index 98%
rename from tests/indices/mapping.yml
rename to tests/default/indices/mapping.yml
index 00a11407d..ac340f93e 100644
--- a/tests/indices/mapping.yml
+++ b/tests/default/indices/mapping.yml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test mappings endpoints.
 prologues:
diff --git a/tests/indices/open.yaml b/tests/default/indices/open.yaml
similarity index 96%
rename from tests/indices/open.yaml
rename to tests/default/indices/open.yaml
index 61aa6fc4b..2253dddc1 100644
--- a/tests/indices/open.yaml
+++ b/tests/default/indices/open.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test opening an index.
 prologues:
diff --git a/tests/indices/refresh.yaml b/tests/default/indices/refresh.yaml
similarity index 92%
rename from tests/indices/refresh.yaml
rename to tests/default/indices/refresh.yaml
index 587168962..2303ed287 100644
--- a/tests/indices/refresh.yaml
+++ b/tests/default/indices/refresh.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test _refresh.
 epilogues:
diff --git a/tests/indices/settings.yaml b/tests/default/indices/settings.yaml
similarity index 97%
rename from tests/indices/settings.yaml
rename to tests/default/indices/settings.yaml
index a5835fdb0..5bfada4b9 100644
--- a/tests/indices/settings.yaml
+++ b/tests/default/indices/settings.yaml
@@ -1,5 +1,5 @@
 
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test settings for an index.
 prologues:
diff --git a/tests/indices/shrink.yaml b/tests/default/indices/shrink.yaml
similarity index 97%
rename from tests/indices/shrink.yaml
rename to tests/default/indices/shrink.yaml
index 1803889ab..ce35ae605 100644
--- a/tests/indices/shrink.yaml
+++ b/tests/default/indices/shrink.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test shrinking an index.
 prologues:
diff --git a/tests/indices/split.yaml b/tests/default/indices/split.yaml
similarity index 97%
rename from tests/indices/split.yaml
rename to tests/default/indices/split.yaml
index 980feb287..6590cf3e9 100644
--- a/tests/indices/split.yaml
+++ b/tests/default/indices/split.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test splitting an index.
 prologues:
diff --git a/tests/indices/stats.yaml b/tests/default/indices/stats.yaml
similarity index 95%
rename from tests/indices/stats.yaml
rename to tests/default/indices/stats.yaml
index 2874cae76..fa340ab79 100644
--- a/tests/indices/stats.yaml
+++ b/tests/default/indices/stats.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test stats for an index.
 prologues:
diff --git a/tests/indices/update_by_query.yaml b/tests/default/indices/update_by_query.yaml
similarity index 96%
rename from tests/indices/update_by_query.yaml
rename to tests/default/indices/update_by_query.yaml
index a58853f3f..0ce1295d4 100644
--- a/tests/indices/update_by_query.yaml
+++ b/tests/default/indices/update_by_query.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test reindex.
 epilogues:
diff --git a/tests/ingest/pipeline.yaml b/tests/default/ingest/pipeline.yaml
similarity index 94%
rename from tests/ingest/pipeline.yaml
rename to tests/default/ingest/pipeline.yaml
index 5fd90d3ca..3145dcb62 100644
--- a/tests/ingest/pipeline.yaml
+++ b/tests/default/ingest/pipeline.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test the creation of an ingest pipeline with a text embedding processor.
 epilogues:
diff --git a/tests/ml/model_groups.yaml b/tests/default/ml/model_groups.yaml
similarity index 95%
rename from tests/ml/model_groups.yaml
rename to tests/default/ml/model_groups.yaml
index 0e5e68b18..a08efa0ce 100644
--- a/tests/ml/model_groups.yaml
+++ b/tests/default/ml/model_groups.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test the creation of model groups.
 version: '>= 2.11'
diff --git a/tests/ml/models.yaml b/tests/default/ml/models.yaml
similarity index 94%
rename from tests/ml/models.yaml
rename to tests/default/ml/models.yaml
index ecf8c3561..ac386c0a4 100644
--- a/tests/ml/models.yaml
+++ b/tests/default/ml/models.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test the creation of models.
 version: '>= 2.11'
diff --git a/tests/nodes/info.yaml b/tests/default/nodes/info.yaml
similarity index 84%
rename from tests/nodes/info.yaml
rename to tests/default/nodes/info.yaml
index 362a04803..7e352f8a6 100644
--- a/tests/nodes/info.yaml
+++ b/tests/default/nodes/info.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test nodes endpoints.
 chapters:
diff --git a/tests/nodes/stats.yaml b/tests/default/nodes/stats.yaml
similarity index 98%
rename from tests/nodes/stats.yaml
rename to tests/default/nodes/stats.yaml
index ecb6152ea..a52225be7 100644
--- a/tests/nodes/stats.yaml
+++ b/tests/default/nodes/stats.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test nodes stats.
 chapters:
diff --git a/tests/ppl/explain.yaml b/tests/default/ppl/explain.yaml
similarity index 88%
rename from tests/ppl/explain.yaml
rename to tests/default/ppl/explain.yaml
index 70df71d79..339123494 100644
--- a/tests/ppl/explain.yaml
+++ b/tests/default/ppl/explain.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test how a query is executed against OpenSearch.
 
diff --git a/tests/ppl/query.yaml b/tests/default/ppl/query.yaml
similarity index 88%
rename from tests/ppl/query.yaml
rename to tests/default/ppl/query.yaml
index a525a85a0..5e8bfe506 100644
--- a/tests/ppl/query.yaml
+++ b/tests/default/ppl/query.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test send PPL query to index.
 
diff --git a/tests/ppl/stats.yaml b/tests/default/ppl/stats.yaml
similarity index 78%
rename from tests/ppl/stats.yaml
rename to tests/default/ppl/stats.yaml
index ec2786432..fafba95ec 100644
--- a/tests/ppl/stats.yaml
+++ b/tests/default/ppl/stats.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test get SQL Queries stats.
 
diff --git a/tests/sql/close.yaml b/tests/default/sql/close.yaml
similarity index 95%
rename from tests/sql/close.yaml
rename to tests/default/sql/close.yaml
index b82b50be6..bb8e8d06f 100644
--- a/tests/sql/close.yaml
+++ b/tests/default/sql/close.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test to explicitly clear the cursor context.
 
diff --git a/tests/sql/explain.yaml b/tests/default/sql/explain.yaml
similarity index 88%
rename from tests/sql/explain.yaml
rename to tests/default/sql/explain.yaml
index 22a648f58..3ac74d222 100644
--- a/tests/sql/explain.yaml
+++ b/tests/default/sql/explain.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test how a query is executed against OpenSearch.
 
diff --git a/tests/sql/query.yaml b/tests/default/sql/query.yaml
similarity index 89%
rename from tests/sql/query.yaml
rename to tests/default/sql/query.yaml
index 96bf4ee43..fdc5e2c85 100644
--- a/tests/sql/query.yaml
+++ b/tests/default/sql/query.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test send SQL query to index.
 
diff --git a/tests/sql/settings.yaml b/tests/default/sql/settings.yaml
similarity index 94%
rename from tests/sql/settings.yaml
rename to tests/default/sql/settings.yaml
index 6a826806f..4de2b1158 100644
--- a/tests/sql/settings.yaml
+++ b/tests/default/sql/settings.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test updating SQL settings in OS cluster.
 
diff --git a/tests/sql/stats.yaml b/tests/default/sql/stats.yaml
similarity index 78%
rename from tests/sql/stats.yaml
rename to tests/default/sql/stats.yaml
index cbb8d7db1..87a181b44 100644
--- a/tests/sql/stats.yaml
+++ b/tests/default/sql/stats.yaml
@@ -1,4 +1,4 @@
-$schema: ../../json_schemas/test_story.schema.yaml
+$schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test get SQL Queries stats.
 
diff --git a/tests/nodes/plugins/index_state_management.yaml b/tests/plugins/index_state_management/nodes/plugins/index_state_management.yaml
similarity index 90%
rename from tests/nodes/plugins/index_state_management.yaml
rename to tests/plugins/index_state_management/nodes/plugins/index_state_management.yaml
index 4f95866d6..7e22dea77 100644
--- a/tests/nodes/plugins/index_state_management.yaml
+++ b/tests/plugins/index_state_management/nodes/plugins/index_state_management.yaml
@@ -1,4 +1,4 @@
-$schema: ../../../json_schemas/test_story.schema.yaml
+$schema: ../../../../../json_schemas/test_story.schema.yaml
 
 description: Get index_state_management node info settings.
 prologues:
diff --git a/tools/src/tester/test.ts b/tools/src/tester/test.ts
index 601d3ff5d..a05800b37 100644
--- a/tools/src/tester/test.ts
+++ b/tools/src/tester/test.ts
@@ -33,7 +33,7 @@ import TestResults from './TestResults'
 const command = new Command()
   .description('Run test stories against the OpenSearch spec.')
   .addOption(new Option('--spec, --spec-path <path>', 'path to the root folder of the multi-file spec').default('./spec'))
-  .addOption(new Option('--tests, --tests-path <path>', 'path to the root folder of the tests').default('./tests'))
+  .addOption(new Option('--tests, --tests-path <path>', 'path to the root folder of the tests').default('./tests/default'))
   .addOption(
     new Option('--tab-width <size>', 'tab width for displayed results')
       .default(4)