Added distributions/included and excluded.

Signed-off-by: dblock <dblock@amazon.com>

TESTING_GUIDE.md

@@ -228,13 +228,16 @@ OpenSearch consists of plugins that may or may not be present in various distrib
     description: Returns basic information about the cluster.
 ```
 
-Similarly, skip tests that are not applicable to a distribution by listing the distributions that support it.
+Similarly, skip tests that are not applicable to a distribution by listing the distributions that support or do not support it.
 
 ```yaml
 description: Test root endpoint.
 distributions:
-  - amazon-managed
-  - opensearch.org
+  included:
+    - amazon-managed
+    - opensearch.org
+  excluded:
+    - amazon-serverless
 chapters:
   - synopsis: Get server info.
     path: /

json_schemas/test_story.schema.yaml

@@ -111,6 +111,16 @@ definitions:
     type: string
 
   Distributions:
+    description: |
+      The list of distributions that support this API.
+    type: object
+    properties:
+      included:
+        $ref: '#/definitions/DistributionsList'
+      excluded:
+        $ref: '#/definitions/DistributionsList'
+
+  DistributionsList:
     description: |
       The list of distributions that support this API.
     type: array

tests/default/_core/info.yaml

@@ -3,8 +3,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test root endpoint.
 distributions:
-  - amazon-managed
-  - opensearch.org
+  excluded:
+    - amazon-serverless
 chapters:
   - synopsis: Get server info.
     path: /

tests/default/cat/health.yaml

@@ -93,7 +93,9 @@ chapters:
           node.data: '1'
   - synopsis: Cat in different formats (format=cbor).
     distributions:
-      - opensearch.org
+      excluded:
+        - amazon-managed
+        - amazon-serverless
     method: GET
     path: /_cat/health
     parameters:
@@ -106,7 +108,9 @@ chapters:
           node.data: '1'
   - synopsis: Cat in different formats (format=smile).
     distributions:
-      - opensearch.org
+      excluded:
+        - amazon-managed
+        - amazon-serverless
     method: GET
     path: /_cat/health
     parameters:

tests/default/cat/indices.yaml

@@ -72,7 +72,9 @@ chapters:
       content_type: application/yaml
   - synopsis: Cat in different formats (format=cbor).
     distributions:
-      - opensearch.org
+      excluded:
+        - amazon-managed
+        - amazon-serverless
     method: GET
     path: /_cat/indices
     parameters:
@@ -82,7 +84,9 @@ chapters:
       content_type: application/cbor
   - synopsis: Cat in different formats (format=smile).
     distributions:
-      - opensearch.org
+      excluded:
+        - amazon-managed
+        - amazon-serverless
     method: GET
     path: /_cat/indices
     parameters:

tests/default/cat/nodeattrs.yaml

@@ -4,7 +4,9 @@ description: Test cat/nodeattrs endpoints.
 chapters:
   - synopsis: Cat with a json response.
     distributions:
-      - opensearch.org
+      excluded:
+        - amazon-managed
+        - amazon-serverless
     path: /_cat/nodeattrs
     method: GET
     parameters:

tests/default/indices/cache.yaml

@@ -2,7 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test index clear cache.
 distributions:
-  - opensearch.org
+  excluded:
+    - amazon-managed
+    - amazon-serverless
 prologues:
   - path: /movies
     method: PUT

tests/default/indices/dangling.yaml

@@ -2,7 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test dangling indexes.
 distributions:
-  - opensearch.org
+  excluded:
+    - amazon-managed
+    - amazon-serverless
 chapters:
   - synopsis: Get dangling indexes.
     path: /_dangling

tests/default/indices/forcemerge.yaml

@@ -3,7 +3,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test force merging an index.
 distributions:
-  - opensearch.org
+  excluded:
+    - amazon-managed
+    - amazon-serverless
 prologues:
   - path: /movies
     method: PUT

tests/default/indices/segments.yaml

@@ -2,7 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml
 
 description: This story tests the Segments API.
 distributions:
-  - opensearch.org
+  excluded:
+    - amazon-managed
+    - amazon-serverless
 prologues:
   - path: /movies
     method: PUT

tests/default/indices/settings.yaml

@@ -26,7 +26,9 @@ chapters:
       status: 200
   - synopsis: Get global settings (cluster_manager_timeout).
     distributions:
-      - opensearch.org
+      excluded:
+        - amazon-managed
+        - amazon-serverless
     path: /_settings
     method: GET
     version: '>= 2.0'

tests/default/ml/model_groups.yaml

@@ -2,7 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test the creation of model groups.
 distributions:
-  - opensearch.org
+  excluded:
+    - amazon-managed
+    - amazon-serverless
 version: '>= 2.11'
 prologues:
   - path: /_cluster/settings

tests/default/ml/models.yaml

@@ -2,7 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml
 
 description: Test the creation of models.
 distributions:
-  - opensearch.org
+  excluded:
+    - amazon-managed
+    - amazon-serverless
 version: '>= 2.11'
 prologues:
   - path: /_cluster/settings

tools/src/tester/ChapterEvaluator.ts

@@ -87,7 +87,6 @@ export default class ChapterEvaluator {
       path: `${chapter.method} ${chapter.path}`,
       overall: { result: overall_result(evaluations) },
       request: { parameters: params, request },
-      retries,
       response: {
         status,
         payload_body: payload_body_evaluation,
@@ -96,6 +95,10 @@ export default class ChapterEvaluator {
       }
     }
 
+    if (retries !== undefined) {
+      result.retries = retries
+    }
+
     if (output_values_evaluation?.output !== undefined) {
       result.output = output_values_evaluation?.output
     }

tools/src/tester/StoryEvaluator.ts

@@ -37,13 +37,23 @@ export default class StoryEvaluator {
       }
     }
 
-    if (distribution != undefined && story.distributions !== undefined && !story.distributions.includes(distribution)) {
+    if (distribution != undefined && story.distributions?.included !== undefined && story.distributions?.included.length > 0 && !story.distributions.included.includes(distribution)) {
       return {
         result: Result.SKIPPED,
         display_path,
         full_path,
         description: story.description,
-        message: `Skipped because distribution ${distribution} is not ${story.distributions.length > 1 ? 'one of ' : ''}${story.distributions.join(', ')}.`
+        message: `Skipped because distribution ${distribution} is not ${story.distributions.included.length > 1 ? 'one of ' : ''}${story.distributions.included.join(', ')}.`
+      }
+    }
+
+    if (distribution != undefined && story.distributions?.excluded !== undefined && story.distributions?.excluded.length > 0 && story.distributions.excluded.includes(distribution)) {
+      return {
+        result: Result.SKIPPED,
+        display_path,
+        full_path,
+        description: story.description,
+        message: `Skipped because distribution ${distribution} is ${story.distributions.excluded.length > 1 ? 'one of ' : ''}${story.distributions.excluded.join(', ')}.`
       }
     }
 
@@ -55,16 +65,23 @@ export default class StoryEvaluator {
     const { evaluations: prologues, has_errors: prologue_errors } = await this.#evaluate_supplemental_chapters(story.prologues ?? [], dry_run, story_outputs)
     const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run, story_outputs, version, distribution)
     const { evaluations: epilogues } = await this.#evaluate_supplemental_chapters(story.epilogues ?? [], dry_run, story_outputs)
-    return {
+
+    const result: StoryEvaluation = {
       display_path,
       full_path,
       description: story.description,
       chapters,
       prologues,
       epilogues,
       result: overall_result(prologues.concat(chapters).concat(epilogues).concat(prologues).map(e => e.overall)),
-      warnings: this.#chapter_warnings(story.chapters)
     }
+
+    const warnings = this.#chapter_warnings(story.chapters)
+    if (warnings !== undefined) {
+      result.warnings = warnings
+    }
+
+    return result
   }
 
   #chapter_warnings(chapters: Chapter[]): string[] | undefined {
@@ -91,13 +108,16 @@ export default class StoryEvaluator {
     for (const chapter of chapters) {
       if (dry_run) {
         const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
-        evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } })
+        evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run' } })
       } else if (version != undefined && chapter.version !== undefined && !semver.satisfies(version, chapter.version)) {
         const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
-        evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.`, error: undefined } })
-      } else if (distribution != undefined && chapter.distributions !== undefined && !chapter.distributions.includes(distribution)) {
+        evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.` } })
+      } else if (distribution != undefined && chapter.distributions?.included !== undefined && chapter.distributions?.included.length > 0 && !chapter.distributions.included.includes(distribution)) {
+        const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
+        evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is not ${chapter.distributions.included.length > 1 ? 'one of ' : ''}${chapter.distributions.included.join(', ')}.` } })
+      } else if (distribution != undefined && chapter.distributions?.excluded !== undefined && chapter.distributions?.excluded.length > 0 && chapter.distributions.excluded.includes(distribution)) {
         const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
-        evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is not ${chapter.distributions.length > 1 ? 'one of ' : ''}${chapter.distributions.join(', ')}.`, error: undefined } })
+        evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is ${chapter.distributions.excluded.length > 1 ? 'one of ' : ''}${chapter.distributions.excluded.join(', ')}.` } })
       } else {
         const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs)
         has_errors = has_errors || evaluation.overall.result === Result.ERROR
@@ -116,7 +136,7 @@ export default class StoryEvaluator {
     for (const chapter of chapters) {
       const title = `${chapter.method} ${chapter.path}`
       if (dry_run) {
-        evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } })
+        evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run' } })
       } else {
         const { evaluation, evaluation_error } = await this._supplemental_chapter_evaluator.evaluate(chapter, story_outputs)
         has_errors = has_errors || evaluation_error

tools/src/tester/SupplementalChapterEvaluator.ts

@@ -12,7 +12,7 @@ import { ChapterOutput } from "./ChapterOutput";
 import ChapterReader from "./ChapterReader";
 import { StoryOutputs } from "./StoryOutputs";
 import { overall_result } from "./helpers";
-import { ChapterEvaluation, Result } from "./types/eval.types";
+import { ChapterEvaluation, Evaluation, Result } from "./types/eval.types";
 import { SupplementalChapter } from "./types/story.types";
 import { Logger } from "../Logger";
 import { to_json } from "../helpers";
@@ -71,9 +71,11 @@ export default class SupplementalChapterEvaluator {
 
       const message = message_segments.join('\n')
 
+      var overall: Evaluation = { result: Result.ERROR, message }
+      if (response.error !== undefined) overall.error = response.error as Error
+
       var evaluation: ChapterEvaluation = {
-        title,
-        overall: { result: Result.ERROR, message, error: response.error as Error }
+        title, overall
       }
 
       if (output_values_evaluation.output) {

tools/src/tester/types/story.types.ts

@@ -55,9 +55,9 @@ export type Version = string;
  *
  *
  * This interface was referenced by `Story`'s JSON-Schema
- * via the `definition` "Distributions".
+ * via the `definition` "DistributionsList".
  */
-export type Distributions = string[];
+export type DistributionsList = string[];
 /**
  * Number of times to retry on error.
  *
@@ -150,6 +150,17 @@ export interface Request {
 export interface Output {
   [k: string]: string;
 }
+/**
+ * The list of distributions that support this API.
+ *
+ *
+ * This interface was referenced by `Story`'s JSON-Schema
+ * via the `definition` "Distributions".
+ */
+export interface Distributions {
+  included?: DistributionsList;
+  excluded?: DistributionsList;
+}
 /**
  * This interface was referenced by `Story`'s JSON-Schema
  * via the `definition` "ExpectedResponse".

tools/tests/tester/fixtures/evals/skipped/distributions/chapters.yaml

@@ -0,0 +1,16 @@
+display_path: skipped/distributions/chapters.yaml
+full_path: tools/tests/tester/fixtures/stories/skipped/distributions/chapters.yaml
+
+description: This story has chapters with distributions.
+prologues: []
+epilogues: []
+result: SKIPPED
+chapters:
+  - title: This chapter is excluded because of excluded distributions (opensearch.org).
+    overall:
+      message: Skipped because distribution opensearch.org is opensearch.org.
+      result: SKIPPED
+  - title: This chapter is excluded because of included distributions (opensearch.org).
+    overall:
+      message: Skipped because distribution opensearch.org is not amazon-managed.
+      result: SKIPPED

tools/tests/tester/fixtures/evals/skipped/distributions/excluded.yaml

@@ -0,0 +1,6 @@
+display_path: skipped/distributions/excluded.yaml
+full_path: tools/tests/tester/fixtures/stories/skipped/distributions/excluded.yaml
+
+result: SKIPPED
+description: This story should be skipped because of distributions (excluded).
+message: Skipped because distribution opensearch.org is one of opensearch.org, amazon-managed.

tools/tests/tester/fixtures/evals/skipped/distributions/included.yaml

@@ -0,0 +1,6 @@
+display_path: skipped/distributions/included.yaml
+full_path: tools/tests/tester/fixtures/stories/skipped/distributions/included.yaml
+
+result: SKIPPED
+description: This story should be skipped because of distributions (included).
+message: Skipped because distribution opensearch.org is not amazon-managed.

tools/tests/tester/fixtures/stories/skipped/distributions/chapters.yaml

@@ -0,0 +1,22 @@
+$schema: ../../../../../../../json_schemas/test_story.schema.yaml
+
+description: This story has chapters with distributions.
+prologues: []
+epilogues: []
+chapters:
+  - synopsis: This chapter is excluded because of excluded distributions (opensearch.org).
+    distributions:
+      excluded:
+        - opensearch.org
+    path: /{index}
+    method: PUT
+    parameters:
+      index: index
+  - synopsis: This chapter is excluded because of included distributions (opensearch.org).
+    distributions:
+      included:
+        - amazon-managed
+    path: /{index}
+    method: PUT
+    parameters:
+      index: index