From 6ded2997d046d4ad0484b21386c102767c1a420d Mon Sep 17 00:00:00 2001 From: Matthew Evans <7916000+ml-evs@users.noreply.github.com> Date: Mon, 11 Jan 2021 11:34:38 +0000 Subject: [PATCH] Add 'validate_unversioned_path' option, which is false by default (#53) * Add 'validate_unversioned_path' option, which is false by default * Update BATS tests to reflect new defaults * Update descriptions and README * Check for argument invalid values in tests * Update description of all_versioned_paths --- .github/workflows/ci_helper.py | 9 ++++--- .github/workflows/validator_action.yml | 26 +++++++++++++++++++- README.md | 3 ++- action.yml | 16 +++++++++--- entrypoint.sh | 13 ++++++++-- tests/test_all_versioned_paths.bats | 23 ++++++++++++----- tests/test_as_type.bats | 9 +++---- tests/test_create_output.bats | 9 +++---- tests/test_unversioned_path.bats | 34 ++++++++++++++++++++++++++ tests/test_validator_version.bats | 12 +++------ tests/test_verbosity.bats | 12 +++------ 11 files changed, 122 insertions(+), 44 deletions(-) create mode 100644 tests/test_unversioned_path.bats diff --git a/.github/workflows/ci_helper.py b/.github/workflows/ci_helper.py index 98af752..23022cb 100644 --- a/.github/workflows/ci_helper.py +++ b/.github/workflows/ci_helper.py @@ -3,6 +3,7 @@ import json import os + def check_output(expected_keys: list): expected_results_keys = ["success_count", "failure_count"] # not complete list results = json.loads(os.getenv("RESULTS")) @@ -16,15 +17,17 @@ def check_output(expected_keys: list): if __name__ == "__main__": - commands = ["default", "all_versioned_paths"] + commands = ["default", "all_versioned_paths", "validate_unversioned_path"] parser = argparse.ArgumentParser() parser.add_argument("cmd", help="Command to run.", type=str, choices=commands) args = parser.parse_args() if args.cmd == "default": - check_output(["unversioned", "v1"]) + check_output(["v1"]) elif args.cmd == "all_versioned_paths": - check_output(["unversioned", "v1", "v1.0", "v1.0.0"]) + check_output(["v1", "v1.0", "v1.0.0"]) + elif args.cmd == "validate_unversioned_path": + check_output(["unversioned", "v1"]) else: exit(f"Wrong command, it must be one of {commands}") diff --git a/.github/workflows/validator_action.yml b/.github/workflows/validator_action.yml index 2229d52..ff40b21 100644 --- a/.github/workflows/validator_action.yml +++ b/.github/workflows/validator_action.yml @@ -22,11 +22,19 @@ jobs: .github/workflows/wait_for_it.sh localhost:3213 -t 120 sleep 15 - - name: Run action + - name: Run action (validating the unversioned path) uses: ./ with: port: 3213 path: / + validate_unversioned_path: yes + + - name: Run action (without validating the unversioned path) + uses: ./ + with: + port: 3213 + path: / + validate_unversioned_path: no validator_index: runs-on: ubuntu-latest @@ -67,6 +75,7 @@ jobs: port: 3213 path: / all versioned paths: yes + validate_unversioned_path: yes - name: Run action (setting path to empty string) uses: ./ @@ -74,6 +83,7 @@ jobs: port: 3213 path: "" all versioned paths: yes + validate_unversioned_path: yes bats: runs-on: ubuntu-latest @@ -179,3 +189,17 @@ jobs: env: RESULTS: "${{ steps.action_all.outputs.results }}" run: python .github/workflows/ci_helper.py all_versioned_paths + + - name: Run action with validate_unversioned_path to retrieve output + id: action_unversioned + uses: ./ + with: + port: 3213 + path: / + create output: true + validate_unversioned_path: true + + - name: Check "validate_unversioned_path" output + env: + RESULTS: "${{ steps.action_unversioned.outputs.results }}" + run: python .github/workflows/ci_helper.py validate_unversioned_path diff --git a/README.md b/README.md index d2aa9e7..2ac37b2 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,8 @@ This can be chosen using the input `validator_version`. | Input | Description | Usage | Default | Action version | | :---: | :--- | :---: | :---: | :---: | -| `all_versioned_paths` | Whether to test all possible versioned base URLs:

/vMAJOR
/vMAJOR.MINOR
/vMAJOR.MINOR.PATCH

The latter two being optional base URLs according to the specification. | Optional | `false` | `v1+` +| `all_versioned_paths` | Whether to test the optional versioned base URLs:

/vMAJOR
/vMAJOR.MINOR
/vMAJOR.MINOR.PATCH

If `false`, only the mandatory /vMAJOR URL will be tested. | Optional | `false` | `v1+` +| `validate_unversioned_path` | Whether to validate the input URL as a full OPTIMADE implementation without appending any version.

This action assumes that the 'path' parameter will contain an unversioned URL. As it is not mandatory to run a full OPTIMADE implementation from the unversioned URL, by default, this action will not perform any validation of this URL, beyond checking the `/versions` endpoint. | Optional | `false` | `v2.3+` | `as_type` | Validate the request URL with the provided type, rather than scanning the entire implementation.
Example values: `structures`, `reference`. For a full list of values see the [OPTIMADE Python tools documentation](https://www.optimade.org/optimade-python-tools/api_reference/validator/validator/#optimade.validator.validator.ImplementationValidator.__init__). | Optional | - | `v1+` | `domain` | Domain for the OPTIMADE URL (defaults to the GitHub Actions runner host). | Optional | `gh_actions_host` | `v1+` | `fail_fast` | Whether or not to exit and return a non-zero error code on first failure. | Optional | `false` | `v2+` diff --git a/action.yml b/action.yml index 1073f1c..b123a37 100644 --- a/action.yml +++ b/action.yml @@ -9,12 +9,22 @@ inputs: all versioned paths: description: > - Whether to test all possible versioned base URLs: - /vMAJOR; /vMAJOR.MINOR; /vMAJOR.MINOR.PATCH - The latter two being optional base URLs according to the specification. + Whether to test the optional versioned URLs: + /vMAJOR.MINOR; /vMAJOR.MINOR.PATCH + If `false`, only the mandatory /vMAJOR URL will be tested. required: false default: false + validate unversioned paths: + description: > + Whether to validate the input URL as a full OPTIMADE implementation without appending any version. + This action assumes that the `path` parameter will contain an unversioned URL. + As it is not mandatory to run a full OPTIMADE implementation from the unversioned URL, + by default, this action will not perform any validation of this URL, beyond checking + the `/versions` endpoint. + required: false + default: false + as type: description: > Validate the request URL with the provided type, rather than scanning the entire implementation. diff --git a/entrypoint.sh b/entrypoint.sh index c66e6b7..0489e03 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -115,8 +115,17 @@ esac # Run validator for unversioned base URL # Echo line is for testing -echo "run_validator: ${run_validator}${INPUT_PATH}${index}" > ./.entrypoint-run_validator.txt -sh -c "${run_validator}${INPUT_PATH}${index}" | tee "unversioned.json" +case ${INPUT_VALIDATE_UNVERSIONED_PATH} in + y | Y | yes | Yes | YES | true | True | TRUE | on | On | ON) + echo "run_validator: ${run_validator}${INPUT_PATH}${index}" > ./.entrypoint-run_validator.txt + sh -c "${run_validator}${INPUT_PATH}${index}" | tee "unversioned.json" + ;; + n | N | no | No | NO | false | False | FALSE | off | Off | OFF) + ;; + *) + echo "Non-valid input for 'validate unversioned path': ${INPUT_VALIDATE_UNVERSIONED_PATH}. Will use default (false)." + ;; +esac # Run validator for versioned base URL(s) if [ "${INPUT_PATH}" = "/" ]; then diff --git a/tests/test_all_versioned_paths.bats b/tests/test_all_versioned_paths.bats index bbdae59..1870520 100644 --- a/tests/test_all_versioned_paths.bats +++ b/tests/test_all_versioned_paths.bats @@ -8,8 +8,7 @@ load 'test_fixtures' refute_output --partial "ERROR" run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" + assert_output "run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" } @test "all_versioned_paths=True" { @@ -17,6 +16,18 @@ run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" run ${ENTRYPOINT_SH} refute_output --partial "ERROR" + OPTIMADE_VERSION=("v1.0" "v1.0.0") + run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt + assert_output "run_validator: ${TEST_MAJOR_RUN_VALIDATOR} +run_validator: ${TEST_BASE_RUN_VALIDATOR}${OPTIMADE_VERSION[0]} +run_validator: ${TEST_BASE_RUN_VALIDATOR}${OPTIMADE_VERSION[1]}" +} + +@test "all_versioned_paths=True & validate_unversioned_path=True" { + export INPUT_ALL_VERSIONED_PATHS=True INPUT_VALIDATE_UNVERSIONED_PATH=True + run ${ENTRYPOINT_SH} + refute_output --partial "ERROR" + OPTIMADE_VERSION=("v1.0" "v1.0.0") run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} @@ -25,15 +36,16 @@ run_validator: ${TEST_BASE_RUN_VALIDATOR}${OPTIMADE_VERSION[0]} run_validator: ${TEST_BASE_RUN_VALIDATOR}${OPTIMADE_VERSION[1]}" } + @test "all_versioned_paths=invalid_value" { # For an invalid value, it should fallback to the default (false) export INPUT_ALL_VERSIONED_PATHS=invalid_value run ${ENTRYPOINT_SH} refute_output --partial "ERROR" + assert_output --partial "Non-valid input for 'all versioned paths': ${INPUT_ALL_VERSIONED_PATHS}. Will use default (false)." run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" + assert_output "run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" } @test "all_versioned_paths=True for old spec v0.10.1" { @@ -53,8 +65,7 @@ run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" OPTIMADE_VERSION=("v0" "v0.10" "v0.10.1") run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_BASE_RUN_VALIDATOR}${OPTIMADE_VERSION[0]} + assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR}${OPTIMADE_VERSION[0]} run_validator: ${TEST_BASE_RUN_VALIDATOR}${OPTIMADE_VERSION[1]} run_validator: ${TEST_BASE_RUN_VALIDATOR}${OPTIMADE_VERSION[2]}" } diff --git a/tests/test_as_type.bats b/tests/test_as_type.bats index 1652cff..0fe3633 100644 --- a/tests/test_as_type.bats +++ b/tests/test_as_type.bats @@ -9,8 +9,7 @@ load 'test_fixtures' refute_output --partial "ERROR" run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" + assert_output "run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" } @test "as_type='structure'" { @@ -22,8 +21,7 @@ run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" TEST_BASE_RUN_VALIDATOR="optimade-validator $( for i in {1..${INPUT_VERBOSITY}}; do echo '-v '; done; )--as-type ${VALID_AS_TYPE_VALUE} ${INPUT_PROTOCOL}://${INPUT_DOMAIN}${INPUT_PATH}" run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_BASE_RUN_VALIDATOR}v1" + assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR}v1" } @test "as_type='non_valid_input' (invalid value, should fail with status 1 and message)" { @@ -37,6 +35,5 @@ run_validator: ${TEST_BASE_RUN_VALIDATOR}v1" TEST_BASE_RUN_VALIDATOR="optimade-validator $( for i in {1..${INPUT_VERBOSITY}}; do echo '-v '; done; )--as-type ${INVALID_AS_TYPE_VALUE} ${INPUT_PROTOCOL}://${INPUT_DOMAIN}${INPUT_PATH}" run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_BASE_RUN_VALIDATOR}v1" + assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR}v1" } diff --git a/tests/test_create_output.bats b/tests/test_create_output.bats index 3f58dfd..3923722 100644 --- a/tests/test_create_output.bats +++ b/tests/test_create_output.bats @@ -12,8 +12,7 @@ load 'test_fixtures' TEST_BASE_RUN_VALIDATOR="optimade-validator --json ${INPUT_PROTOCOL}://${INPUT_DOMAIN}${INPUT_PATH}" run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_BASE_RUN_VALIDATOR}v1" + assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR}v1" } @test "create_output='test' (invalid value, should use default instead)" { @@ -25,8 +24,7 @@ run_validator: ${TEST_BASE_RUN_VALIDATOR}v1" refute_output --partial "Non-valid input for 'verbosity'" run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" + assert_output "run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" } @test "create_output=True (with validator_version that hasn't got the '--json' option)" { @@ -41,6 +39,5 @@ run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" refute_output --partial "Non-valid input for 'verbosity'" run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" + assert_output "run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" } diff --git a/tests/test_unversioned_path.bats b/tests/test_unversioned_path.bats new file mode 100644 index 0000000..016a66c --- /dev/null +++ b/tests/test_unversioned_path.bats @@ -0,0 +1,34 @@ +#!/usr/bin/env bats + +load 'test_fixtures' + +@test "validate_unversioned_path=False" { + export INPUT_VALIDATE_UNVERSIONED_PATH=False + run ${ENTRYPOINT_SH} + refute_output --partial "ERROR" + + run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt + assert_output "run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" +} + +@test "validate_unversioned_path=True" { + export INPUT_VALIDATE_UNVERSIONED_PATH=True + run ${ENTRYPOINT_SH} + refute_output --partial "ERROR" + + OPTIMADE_VERSION=("v1.0" "v1.0.0") + run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt + assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} +run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" +} + +@test "validate_unversioned_path=invalid_value" { + # For an invalid value, it should fallback to the default (false) + export INPUT_VALIDATE_UNVERSIONED_PATH=invalid_value + run ${ENTRYPOINT_SH} + assert_output --partial "Non-valid input for 'validate unversioned path': ${INPUT_VALIDATE_UNVERSIONED_PATH}. Will use default (false)." + refute_output --partial "ERROR" + + run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt + assert_output "run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" +} diff --git a/tests/test_validator_version.bats b/tests/test_validator_version.bats index 1a5fc99..3b711ed 100644 --- a/tests/test_validator_version.bats +++ b/tests/test_validator_version.bats @@ -9,8 +9,7 @@ load 'test_fixtures' refute_output --partial "ERROR" run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" + assert_output "run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" } @test "validator_version='0.10.0'" { @@ -20,8 +19,7 @@ run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" refute_output --partial "ERROR" run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" + assert_output "run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" } @test "validator_version='v0.10.0'" { @@ -32,8 +30,7 @@ run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" refute_output --partial "ERROR" run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" + assert_output "run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" } @test "validator_version='master'" { @@ -43,8 +40,7 @@ run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" refute_output --partial "ERROR" run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" + assert_output "run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" } @test "validator_version='0.0.0' (invalid value, should fail with status 1 and message)" { diff --git a/tests/test_verbosity.bats b/tests/test_verbosity.bats index 1cbf4f2..2a4f3f3 100644 --- a/tests/test_verbosity.bats +++ b/tests/test_verbosity.bats @@ -10,8 +10,7 @@ load 'test_fixtures' refute_output --partial "Non-valid input for 'verbosity'" run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" + assert_output "run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" } @test "verbosity=0" { @@ -24,8 +23,7 @@ run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" TEST_BASE_RUN_VALIDATOR="optimade-validator ${INPUT_PROTOCOL}://${INPUT_DOMAIN}${INPUT_PATH}" run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_BASE_RUN_VALIDATOR}v1" + assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR}v1" } @test "verbosity='test' (invalid value, should use default instead)" { @@ -37,8 +35,7 @@ run_validator: ${TEST_BASE_RUN_VALIDATOR}v1" assert_output --partial "Using verbosity level: ${VERBOSITY_DEFAULT}" run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" + assert_output "run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" } @test "verbosity=3" { @@ -51,6 +48,5 @@ run_validator: ${TEST_MAJOR_RUN_VALIDATOR}" TEST_BASE_RUN_VALIDATOR="optimade-validator -v -v -v ${INPUT_PROTOCOL}://${INPUT_DOMAIN}${INPUT_PATH}" run cat ${DOCKER_BATS_WORKDIR}/.entrypoint-run_validator.txt - assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR} -run_validator: ${TEST_BASE_RUN_VALIDATOR}v1" + assert_output "run_validator: ${TEST_BASE_RUN_VALIDATOR}v1" }