From 0cd624ed5dd7baafedd7784883063cfdbdc2f39c Mon Sep 17 00:00:00 2001 From: valentin-seitz Date: Tue, 5 Sep 2023 09:23:20 +0200 Subject: [PATCH] Have one Dockerfile instead of multiples (#364) * Added first working prototype * added working fenics adapter * remove comments * added working nutils "adapter" * added not working calclulix -adapter * reorder dockerfile * make autopep8 happy * fix print_case_combinations * WIP * WIP * - Introduced logging instead of prints - refactored into exceptions instead of exit(1) - * - Added Nutils - Added Calculix - Added reference results for some cases - Added heat exchanger tutorial - Added logging and more verbose output - Added timing output * Split building docker and running tutorial * adopt to requested chnages: - fallow-arguments vs std-legacy - no nutils adapter, just install nutils * - Adopt .md to .metadata - generate new reference results * - added overview table * added script to only build dockers to warmup cache * Update tools/tests/components.yaml Co-authored-by: Gerasimos Chourdakis * make autpep8 happy --------- Co-authored-by: Valentin Seitz Co-authored-by: Gerasimos Chourdakis --- .../fluid-openfoam_solid-fenics.tar.gz | 3 + .../fluid-openfoam_solid-nutils.tar.gz | 3 + .../fluid-openfoam_solid-openfoam.tar.gz | 4 +- flow-over-heated-plate/reference_results.md | 21 - .../reference_results.metadata | 29 ++ heat-exchanger-simplified/metadata.yaml | 29 ++ .../fluid-openfoam_solid-calculix.tar.gz | 3 + perpendicular-flap/reference_results.metadata | 27 ++ tools/tests/build_docker_images.py | 96 +++++ .../component-templates/calculix-adapter.yaml | 16 + .../component-templates/fenics-adapter.yaml | 11 +- .../component-templates/nutils-adapter.yaml | 11 +- .../component-templates/openfoam-adapter.yaml | 3 +- tools/tests/components.yaml | 76 +++- tools/tests/docker-compose.template.yaml | 7 +- .../tests/dockerfiles/ubuntu_2204/Dockerfile | 118 +++++ tools/tests/generate_reference_data.py | 139 +++--- tools/tests/print_case_combinations.py | 38 +- tools/tests/print_metadata.py | 27 +- tools/tests/print_test_suites.py | 32 +- ...te => reference_results.metadata.template} | 4 +- tools/tests/reference_versions.yaml | 12 +- tools/tests/systemtests.py | 152 ++++--- tools/tests/systemtests/Systemtest.py | 403 ++++++++++++++---- .../tests/systemtests/SystemtestArguments.py | 3 +- tools/tests/tests.yaml | 22 + 26 files changed, 993 insertions(+), 296 deletions(-) create mode 100644 flow-over-heated-plate/reference-data/fluid-openfoam_solid-fenics.tar.gz create mode 100644 flow-over-heated-plate/reference-data/fluid-openfoam_solid-nutils.tar.gz delete mode 100644 flow-over-heated-plate/reference_results.md create mode 100644 flow-over-heated-plate/reference_results.metadata create mode 100644 heat-exchanger-simplified/metadata.yaml create mode 100644 perpendicular-flap/reference-data/fluid-openfoam_solid-calculix.tar.gz create mode 100644 perpendicular-flap/reference_results.metadata create mode 100644 tools/tests/build_docker_images.py create mode 100644 tools/tests/component-templates/calculix-adapter.yaml create mode 100644 tools/tests/dockerfiles/ubuntu_2204/Dockerfile rename tools/tests/{reference_results.template => reference_results.metadata.template} (85%) diff --git a/flow-over-heated-plate/reference-data/fluid-openfoam_solid-fenics.tar.gz b/flow-over-heated-plate/reference-data/fluid-openfoam_solid-fenics.tar.gz new file mode 100644 index 000000000..fd47ecc01 --- /dev/null +++ b/flow-over-heated-plate/reference-data/fluid-openfoam_solid-fenics.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2565ffc51c8d80fab06ccdda33f62efceb6cdc03318998f5adc9ed0ac84acac2 +size 777932 diff --git a/flow-over-heated-plate/reference-data/fluid-openfoam_solid-nutils.tar.gz b/flow-over-heated-plate/reference-data/fluid-openfoam_solid-nutils.tar.gz new file mode 100644 index 000000000..9c773bcdf --- /dev/null +++ b/flow-over-heated-plate/reference-data/fluid-openfoam_solid-nutils.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b3157902d3ad78593c5471a3a94dd5559fe6970e4cb54658b22ffd270c00297 +size 532549 diff --git a/flow-over-heated-plate/reference-data/fluid-openfoam_solid-openfoam.tar.gz b/flow-over-heated-plate/reference-data/fluid-openfoam_solid-openfoam.tar.gz index 61cf44283..ee340cd80 100644 --- a/flow-over-heated-plate/reference-data/fluid-openfoam_solid-openfoam.tar.gz +++ b/flow-over-heated-plate/reference-data/fluid-openfoam_solid-openfoam.tar.gz @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7a0976d2fc7a421a75036cd5d888aad1526a8e86fa65912c046d96f6b74171ca -size 498892 +oid sha256:c29989e0118ade759e7313330aa8d3d900c660c404eaacd529e979aae6b899c7 +size 498432 diff --git a/flow-over-heated-plate/reference_results.md b/flow-over-heated-plate/reference_results.md deleted file mode 100644 index 8189c99ae..000000000 --- a/flow-over-heated-plate/reference_results.md +++ /dev/null @@ -1,21 +0,0 @@ - - -# Reference Results - -This file contains an overview of the results over the reference results as well as the arguments used to generate them. - -## List of files - -| name | time | sha-1 | -|------|------|-------| -| fluid-openfoam_solid-openfoam.tar.gz | 2023-08-13 14:52:04 | bd8b4d235c8326e75deef3f03a1d8277c429972a | - -## List of arguments used to generate the files - -| name | value | -|------|------| -| OPENFOAM_EXECUTABLE | openfoam2112 | -| PRECICE_TAG | latest | -| OPENFOAM_ADAPTER_REF | master | diff --git a/flow-over-heated-plate/reference_results.metadata b/flow-over-heated-plate/reference_results.metadata new file mode 100644 index 000000000..640389b49 --- /dev/null +++ b/flow-over-heated-plate/reference_results.metadata @@ -0,0 +1,29 @@ + + +# Reference Results + +This file contains an overview of the results over the reference results as well as the arguments used to generate them. + +## List of files + +| name | time | sha256 | +|------|------|-------| +| fluid-openfoam_solid-nutils.tar.gz | 2023-09-04 23:41:09 | 8b3157902d3ad78593c5471a3a94dd5559fe6970e4cb54658b22ffd270c00297 | +| fluid-openfoam_solid-fenics.tar.gz | 2023-09-04 23:41:09 | 2565ffc51c8d80fab06ccdda33f62efceb6cdc03318998f5adc9ed0ac84acac2 | +| fluid-openfoam_solid-openfoam.tar.gz | 2023-09-04 23:41:09 | c29989e0118ade759e7313330aa8d3d900c660c404eaacd529e979aae6b899c7 | + +## List of arguments used to generate the files + +| name | value | +|------|------| +| PRECICE_REF | v2.5.0 | +| OPENFOAM_EXECUTABLE | openfoam2306 | +| OPENFOAM_ADAPTER_REF | v1.2.3 | +| PYTHON_BINDINGS_REF | v2.5.0.4 | +| FENICS_ADAPTER_REF | v1.4.0 | +| TUTORIALS_REF | v202211.0 | +| PLATFORM | ubuntu_2204 | +| CALULIX_VERSION | 2.20 | +| CALULIX_ADAPTER_REF | v2.20.0 | diff --git a/heat-exchanger-simplified/metadata.yaml b/heat-exchanger-simplified/metadata.yaml new file mode 100644 index 000000000..124e76f9d --- /dev/null +++ b/heat-exchanger-simplified/metadata.yaml @@ -0,0 +1,29 @@ +name: Heat Exchanger simplified +path: heat-exchanger-simplified # relative to git repo +url: https://precice.org/tutorials-heat-exchanger-simplified.html + +participants: + - Fluid-Top + - Fluid-Bottom + - Solid + +cases: + fluid-btm-openfoam: + participant: Fluid-Bottom + directory: ./fluid-bottom-openfoam + run: ./run.sh + component: openfoam-adapter + + fluid-top-openfoam: + participant: Fluid-Top + directory: ./fluid-top-openfoam + run: ./run.sh + component: openfoam-adapter + + solid-calculix: + participant: Solid + directory: ./solid-calculix + run: ./run.sh + component: calculix-adapter + + diff --git a/perpendicular-flap/reference-data/fluid-openfoam_solid-calculix.tar.gz b/perpendicular-flap/reference-data/fluid-openfoam_solid-calculix.tar.gz new file mode 100644 index 000000000..63d4135f6 --- /dev/null +++ b/perpendicular-flap/reference-data/fluid-openfoam_solid-calculix.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:596fe1aec3f72fa194c37c485688dea4e308544e3cd773d7c815d6e27c4e55a8 +size 13561235 diff --git a/perpendicular-flap/reference_results.metadata b/perpendicular-flap/reference_results.metadata new file mode 100644 index 000000000..1255609aa --- /dev/null +++ b/perpendicular-flap/reference_results.metadata @@ -0,0 +1,27 @@ + + +# Reference Results + +This file contains an overview of the results over the reference results as well as the arguments used to generate them. + +## List of files + +| name | time | sha256 | +|------|------|-------| +| fluid-openfoam_solid-calculix.tar.gz | 2023-09-04 23:41:09 | 596fe1aec3f72fa194c37c485688dea4e308544e3cd773d7c815d6e27c4e55a8 | + +## List of arguments used to generate the files + +| name | value | +|------|------| +| PRECICE_REF | v2.5.0 | +| OPENFOAM_EXECUTABLE | openfoam2306 | +| OPENFOAM_ADAPTER_REF | v1.2.3 | +| PYTHON_BINDINGS_REF | v2.5.0.4 | +| FENICS_ADAPTER_REF | v1.4.0 | +| TUTORIALS_REF | v202211.0 | +| PLATFORM | ubuntu_2204 | +| CALULIX_VERSION | 2.20 | +| CALULIX_ADAPTER_REF | v2.20.0 | diff --git a/tools/tests/build_docker_images.py b/tools/tests/build_docker_images.py new file mode 100644 index 000000000..c77b8523d --- /dev/null +++ b/tools/tests/build_docker_images.py @@ -0,0 +1,96 @@ + +import argparse +from pathlib import Path +from systemtests.SystemtestArguments import SystemtestArguments +from systemtests.Systemtest import Systemtest, display_systemtestresults_as_table +from systemtests.TestSuite import TestSuites +from metadata_parser.metdata import Tutorials +import logging +import time +from paths import PRECICE_TUTORIAL_DIR, PRECICE_TESTS_RUN_DIR, PRECICE_TESTS_DIR + + +def main(): + parser = argparse.ArgumentParser(description='build docker images') + + # Add an argument for the components + parser.add_argument('--suites', type=str, + help='Comma-separated test-suites to execute') + parser.add_argument( + '--build_args', + type=str, + help='Comma-separated list of arguments provided to the components like openfoam:2102,pythonbindings:latest') + parser.add_argument('--rundir', type=str, help='Directory to run the systemstests in.', + nargs='?', const=PRECICE_TESTS_RUN_DIR, default=PRECICE_TESTS_RUN_DIR) + + parser.add_argument('--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + default='INFO', help='Set the logging level') + + # Parse the command-line arguments + args = parser.parse_args() + + # Configure logging based on the provided log level + logging.basicConfig(level=args.log_level, format='%(levelname)s: %(message)s') + + print(f"Using log-level: {args.log_level}") + + systemtests_to_run = [] + available_tutorials = Tutorials.from_path(PRECICE_TUTORIAL_DIR) + + build_args = SystemtestArguments.from_args(args.build_args) + run_directory = Path(args.rundir) + if args.suites: + test_suites_requested = args.suites.split(',') + available_testsuites = TestSuites.from_yaml( + PRECICE_TESTS_DIR / "tests.yaml", available_tutorials) + test_suites_to_execute = [] + for test_suite_requested in test_suites_requested: + test_suite_found = available_testsuites.get_by_name( + test_suite_requested) + if not test_suite_found: + logging.error(f"Did not find the testsuite with name {test_suite_requested}") + else: + test_suites_to_execute.append(test_suite_found) + if not test_suites_to_execute: + raise RuntimeError( + f"No matching test suites with names {test_suites_requested} found. Use print_test_suites.py to get an overview") + # now convert the test_suites into systemtests + for test_suite in test_suites_to_execute: + tutorials = test_suite.cases_of_tutorial.keys() + for tutorial in tutorials: + for case, reference_result in zip( + test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): + systemtests_to_run.append( + Systemtest(tutorial, build_args, case, reference_result)) + + if not systemtests_to_run: + raise RuntimeError("Did not find any Systemtests to execute.") + + logging.info(f"About to build the images for the following systemtests:\n {systemtests_to_run}") + + results = [] + for number, systemtest in enumerate(systemtests_to_run): + logging.info(f"Started building {systemtest}, {number}/{len(systemtests_to_run)}") + t = time.perf_counter() + result = systemtest.run_only_build(run_directory) + elapsed_time = time.perf_counter() - t + logging.info(f"Building image for {systemtest} took {elapsed_time} seconds") + results.append(result) + + build_docker_success = True + for result in results: + if not result.success: + logging.error(f"Failed to run {result.systemtest}") + build_docker_success = False + else: + logging.info(f"Success running {result.systemtest}") + + display_systemtestresults_as_table(results) + if build_docker_success: + exit(0) + else: + exit(1) + + +if __name__ == '__main__': + main() diff --git a/tools/tests/component-templates/calculix-adapter.yaml b/tools/tests/component-templates/calculix-adapter.yaml new file mode 100644 index 000000000..038a088b7 --- /dev/null +++ b/tools/tests/component-templates/calculix-adapter.yaml @@ -0,0 +1,16 @@ +build: + context: {{ dockerfile_context }} + args: + {% for key, value in build_arguments.items() %} + - {{key}}={{value}} + {% endfor %} + target: calculix_adapter +depends_on: + prepare: + condition: service_completed_successfully +volumes: + - {{ run_directory }}:/runs +command: > + /bin/bash -c "id && + cd '/runs/{{ tutorial_folder }}/{{ case_folder }}' && + {{ run }} | tee {{ case_folder }}.log 2>&1" diff --git a/tools/tests/component-templates/fenics-adapter.yaml b/tools/tests/component-templates/fenics-adapter.yaml index 7b0d41388..b83a23707 100644 --- a/tools/tests/component-templates/fenics-adapter.yaml +++ b/tools/tests/component-templates/fenics-adapter.yaml @@ -1,11 +1,14 @@ -image: precice/fenics-adapter:{{ build_arguments["FENICS_ADAPTER_TAG"] }} -user: ${UID}:${GID} +build: + context: {{ dockerfile_context }} + args: + {% for key, value in build_arguments.items() %} + - {{key}}={{value}} + {% endfor %} + target: fenics_adapter depends_on: prepare: condition: service_completed_successfully volumes: - - /etc/passwd:/etc/passwd:ro - - /etc/group:/etc/group:ro - {{ run_directory }}:/runs command: > /bin/bash -c "id && diff --git a/tools/tests/component-templates/nutils-adapter.yaml b/tools/tests/component-templates/nutils-adapter.yaml index ae37d38f6..9963028ab 100644 --- a/tools/tests/component-templates/nutils-adapter.yaml +++ b/tools/tests/component-templates/nutils-adapter.yaml @@ -1,11 +1,14 @@ -image: "ghcr.io/precice/openfoam-adapter:{{ build_arguments["openfoam-adapter-ref"] }}" # TODO: Update -user: ${UID}:${GID} +build: + context: {{ dockerfile_context }} + args: + {% for key, value in build_arguments.items() %} + - {{key}}={{value}} + {% endfor %} + target: nutils_adapter depends_on: prepare: condition: service_completed_successfully volumes: - - /etc/passwd:/etc/passwd:ro - - /etc/group:/etc/group:ro - {{ run_directory }}:/runs command: > /bin/bash -c "id && diff --git a/tools/tests/component-templates/openfoam-adapter.yaml b/tools/tests/component-templates/openfoam-adapter.yaml index 44847ba27..4657b6e23 100644 --- a/tools/tests/component-templates/openfoam-adapter.yaml +++ b/tools/tests/component-templates/openfoam-adapter.yaml @@ -1,9 +1,10 @@ build: - context: https://github.com/precice/openfoam-adapter.git#add-ci-docker:tools/docker + context: {{ dockerfile_context }} args: {% for key, value in build_arguments.items() %} - {{key}}={{value}} {% endfor %} + target: openfoam_adapter depends_on: prepare: condition: service_completed_successfully diff --git a/tools/tests/components.yaml b/tools/tests/components.yaml index f7aac37dc..0560879e5 100644 --- a/tools/tests/components.yaml +++ b/tools/tests/components.yaml @@ -2,39 +2,77 @@ openfoam-adapter: repository: https://github.com/precice/openfoam-adapter template: component-templates/openfoam-adapter.yaml build_arguments: # these things mean something to the docker-service + PRECICE_REF: + description: Version of preCICE to use + default: "main" + PLATFORM: + description: Dockerfile platform used + default: "ubuntu_2204" + TUTORIALS_REF: + description: Tutorial git reference to use + default: "master" OPENFOAM_EXECUTABLE: - options: ["openfoam2112"] + options: ["openfoam2306","openfoam2212","openfoam2112"] description: exectuable of openfoam to use - default: "openfoam2112" - PRECICE_TAG: - description: Version of preCICE to use - default: "latest" + default: "openfoam2306" OPENFOAM_ADAPTER_REF: description: Reference/tag of the actual OpenFOAM adapter default: "master" + fenics-adapter: repository: https://github.com/precice/fenics-adapter template: component-templates/fenics-adapter.yaml build_arguments: - FENICS_ADAPTER_TAG: - semnantic: Version of the fenics adapter image to use - default: "latest" - -calculix-adapter: - repository: https://github.com/precice/calculix-adapter - template: component-templates/calculix-adapter.yaml - build_arguments: - PRECICE_TAG: + PRECICE_REF: description: Version of preCICE to use - default: "latest" - - + default: "main" + PLATFORM: + description: Dockerfile platform used + default: "ubuntu_2204" + TUTORIALS_REF: + description: Tutorial git reference to use + default: "master" + PYTHON_BINDINGS_REF: + semnantic: Git ref of the pythonbindings to use + default: "master" + FENICS_ADAPTER_REF: + semnantic: Git ref of the fenics adapter to use + default: "master" nutils-adapter: repository: https://github.com/precice/nutils-adapter template: component-templates/nutils-adapter.yaml build_arguments: - PRECICE_TAG: + PRECICE_REF: description: Version of preCICE to use - default: "latest" + default: "main" + PLATFORM: + description: Dockerfile platform used + default: "ubuntu_2204" + TUTORIALS_REF: + description: Tutorial git reference to use + default: "master" + PYTHON_BINDINGS_REF: + semnantic: Git ref of the pythonbindings to use + + +calculix-adapter: + repository: https://github.com/precice/calculix-adapter + template: component-templates/calculix-adapter.yaml + build_arguments: + PRECICE_REF: + description: Version of preCICE to use + default: "main" + PLATFORM: + description: Dockerfile platform used + default: "ubuntu_2204" + TUTORIALS_REF: + description: Tutorial git reference to use + default: "master" + CALULIX_VERSION: + description: Version of Calculix to use + default: "2.20" + CALULIX_ADAPTER_REF: + description: Version of Calculix-Adapter to use + default: "master" \ No newline at end of file diff --git a/tools/tests/docker-compose.template.yaml b/tools/tests/docker-compose.template.yaml index a259eea24..ff6f7d279 100644 --- a/tools/tests/docker-compose.template.yaml +++ b/tools/tests/docker-compose.template.yaml @@ -1,11 +1,12 @@ version: "3.9" services: prepare: - image: ubuntu:latest - user: ${UID}:${GID} + build: + context: {{ dockerfile_context }} + target: base_image volumes: - {{ run_directory }}:/runs - # tar -zxf reference-output.tar.gz && + command: > /bin/bash -c "id && cd '/runs/{{ tutorial_folder }}' && diff --git a/tools/tests/dockerfiles/ubuntu_2204/Dockerfile b/tools/tests/dockerfiles/ubuntu_2204/Dockerfile new file mode 100644 index 000000000..3f34db82e --- /dev/null +++ b/tools/tests/dockerfiles/ubuntu_2204/Dockerfile @@ -0,0 +1,118 @@ +FROM ubuntu:22.04 as base_image +USER root +SHELL ["/bin/bash", "-c"] +ENV DEBIAN_FRONTEND=noninteractive + +RUN useradd -ms /bin/bash precice +ENV PATH="${PATH}:/home/precice/.local/bin" +ENV LD_LIBRARY_PATH="/home/precice/.local/lib:${LD_LIBRARY_PATH}" +ENV CPATH="/home/precice/.local/include:$CPATH" +# Enable detection with pkg-config and CMake +ENV PKG_CONFIG_PATH="/home/precice/.local/lib/pkgconfig:$PKG_CONFIG_PATH" +ENV CMAKE_PREFIX_PATH="/home/precice/.local:$CMAKE_PREFIX_PATH" +USER precice + +FROM base_image as precice_dependecies +USER root +# Installing necessary dependecies for preCICE +RUN apt-get -qq update && \ + apt-get -qq -y install \ + build-essential \ + software-properties-common \ + cmake \ + curl \ + g++ \ + gfortran \ + git \ + libbenchmark-dev \ + libboost-all-dev \ + libeigen3-dev \ + libxml2-dev \ + lsb-release \ + petsc-dev \ + python3-dev \ + python3-numpy \ + python3-pip \ + python3-venv \ + pkg-config \ + wget +USER precice +RUN python3 -m pip install --user --upgrade pip + + +FROM precice_dependecies as precice +# Install & build precice into /home/precice/precice +ARG PRECICE_REF +USER precice +WORKDIR /home/precice +RUN git clone --depth 1 https://github.com/precice/precice.git -b ${PRECICE_REF} precice && \ + cd precice && \ + mkdir build && cd build &&\ + cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/home/precice/.local/ -DPRECICE_PETScMapping=OFF -DBUILD_TESTING=OFF && \ + make all install -j $(nproc) + +FROM precice_dependecies as openfoam_adapter +ARG OPENFOAM_EXECUTABLE +USER root +RUN apt-get update &&\ + wget -q -O - https://dl.openfoam.com/add-debian-repo.sh | bash &&\ + apt-get -qq install ${OPENFOAM_EXECUTABLE}-dev &&\ + ln -s $(which ${OPENFOAM_EXECUTABLE} ) /usr/bin/openfoam +USER precice +COPY --from=precice /home/precice/.local/ /home/precice/.local/ +ARG OPENFOAM_ADAPTER_REF +# Build the OpenFOAM adapter +USER precice +WORKDIR /home/precice +RUN git clone --depth 1 -b ${OPENFOAM_ADAPTER_REF} https://github.com/precice/openfoam-adapter.git &&\ + cd openfoam-adapter && /usr/bin/${OPENFOAM_EXECUTABLE} ./Allwmake -j $(nproc) + + +FROM precice_dependecies as python_bindings +COPY --from=precice /home/precice/.local/ /home/precice/.local/ +ARG PYTHON_BINDINGS_REF +USER precice +WORKDIR /home/precice +# Builds the precice python bindings for python3 +RUN pip3 install --user git+https://github.com/precice/python-bindings.git@${PYTHON_BINDINGS_REF} + + +FROM precice_dependecies as fenics_adapter +COPY --from=python_bindings /home/precice/.local /home/precice/.local +USER root +RUN add-apt-repository -y ppa:fenics-packages/fenics && \ + apt-get -qq update && \ + apt-get -qq install --no-install-recommends fenics +USER precice +RUN pip3 install --user fenics-ufl +ARG FENICS_ADAPTER_REF +# Building fenics-adapter +RUN pip3 install --user git+https://github.com/precice/fenics-adapter.git@${FENICS_ADAPTER_REF} + + +FROM precice_dependecies as nutils_adapter +COPY --from=python_bindings /home/precice/.local /home/precice/.local +USER precice +# Installing nutils - There is no adapter +RUN pip3 install --user nutils + + +FROM precice_dependecies as calculix_adapter +COPY --from=precice /home/precice/.local /home/precice/.local +USER root +RUN apt-get -qq update && \ + apt-get -qq install libarpack2-dev libspooles-dev libyaml-cpp-dev +ARG CALULIX_VERSION +USER precice +#Download Calculix +WORKDIR /home/precice +RUN wget http://www.dhondt.de/ccx_${CALULIX_VERSION}.src.tar.bz2 && \ + tar xvjf ccx_${CALULIX_VERSION}.src.tar.bz2 && \ + rm -fv ccx_${CALULIX_VERSION}.src.tar.bz2 + +ARG CALULIX_ADAPTER_REF +WORKDIR /home/precice +RUN git clone --depth 1 --branch ${CALULIX_ADAPTER_REF} https://github.com/precice/calculix-adapter.git && \ + cd calculix-adapter && \ + make CXX_VERSION=${CALULIX_VERSION} ADDITIONAL_FFLAGS="-fallow-argument-mismatch" -j $(nproc) && \ + ln -s /home/precice/calculix-adapter/bin/ccx_preCICE /home/precice/.local/bin/ccx_preCICE diff --git a/tools/tests/generate_reference_data.py b/tools/tests/generate_reference_data.py index b6272d7b6..ea2457984 100644 --- a/tools/tests/generate_reference_data.py +++ b/tools/tests/generate_reference_data.py @@ -5,19 +5,19 @@ from systemtests.SystemtestArguments import SystemtestArguments from systemtests.Systemtest import Systemtest from pathlib import Path -from typing import List, Tuple, Optional, Dict +from typing import List from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR import hashlib from jinja2 import Environment, FileSystemLoader import tarfile from datetime import datetime - +import logging from paths import PRECICE_TUTORIAL_DIR, PRECICE_TESTS_RUN_DIR, PRECICE_TESTS_DIR, PRECICE_REL_OUTPUT_DIR +import time def create_tar_gz(source_folder: Path, output_filename: Path): with tarfile.open(output_filename, "w:gz") as tar: - print(source_folder, output_filename) tar.add(source_folder, arcname=output_filename.name.replace(".tar.gz", "")) @@ -25,21 +25,14 @@ def render_reference_results_info( reference_results: List[ReferenceResult], arguments_used: SystemtestArguments, time: str): - def calculate_sha1(file_path: Path): - buffer_size = 65536 - sha1_hash = hashlib.sha1() - with open(file_path, "rb") as f: - while True: - data = f.read(buffer_size) - if not data: - break - sha1_hash.update(data) - return sha1_hash.hexdigest() + def sha256sum(filename): + with open(filename, 'rb', buffering=0) as f: + return hashlib.file_digest(f, 'sha256').hexdigest() files = [] for reference_result in reference_results: files.append({ - 'sha1': calculate_sha1(reference_result.path), + 'sha256': sha256sum(reference_result.path), 'time': time, 'name': reference_result.path.name, }) @@ -49,53 +42,77 @@ def calculate_sha1(file_path: Path): 'files': files } jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) - template = jinja_env.get_template("reference_results.template") + template = jinja_env.get_template("reference_results.metadata.template") return template.render(render_dict) -parser = argparse.ArgumentParser(description='generate reference data') -parser.add_argument('--rundir', type=str, help='Directory to run the systemstests in.', - nargs='?', const=PRECICE_TESTS_RUN_DIR, default=PRECICE_TESTS_RUN_DIR) -# Parse the command-line arguments -args = parser.parse_args() - -run_directory = Path(args.rundir) - -available_tutorials = Tutorials.from_path(PRECICE_TUTORIAL_DIR) - -test_suites = TestSuites.from_yaml(PRECICE_TESTS_DIR / "tests.yaml", available_tutorials) - -# Read in parameters -build_args = SystemtestArguments.from_yaml(PRECICE_TESTS_DIR / "reference_versions.yaml") -systemtests_to_run = set() - -for test_suite in test_suites: - tutorials = test_suite.cases_of_tutorial.keys() - for tutorial in tutorials: - for case, reference_result in zip( - test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): - systemtests_to_run.add( - Systemtest(tutorial, build_args, case, reference_result)) - - -reference_result_per_tutorial = {} -current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') - - -print(f"About to run the following tests {systemtests_to_run}") -for systemtest in systemtests_to_run: - systemtest.run_for_reference_results(run_directory) - reference_result_per_tutorial[systemtest.tutorial] = [] - -# Put the tar.gz in there -for systemtest in systemtests_to_run: - reference_result_folder = systemtest.get_system_test_dir() / PRECICE_REL_OUTPUT_DIR - reference_result_per_tutorial[systemtest.tutorial].append(systemtest.reference_result) - create_tar_gz(reference_result_folder, systemtest.reference_result.path) - -# write readme -for tutorial in reference_result_per_tutorial.keys(): - with open(tutorial.path / "reference_results.md", 'w') as file: - ref_results_info = render_reference_results_info( - reference_result_per_tutorial[tutorial], build_args, current_time_string) - file.write(ref_results_info) +def main(): + + parser = argparse.ArgumentParser(description='Generate reference data for systemtests') + parser.add_argument('--rundir', type=str, help='Directory to run the systemstests in.', + nargs='?', const=PRECICE_TESTS_RUN_DIR, default=PRECICE_TESTS_RUN_DIR) + parser.add_argument('--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + default='INFO', help='Set the logging level') + + args = parser.parse_args() + + logging.basicConfig(level=args.log_level, format='%(levelname)s: %(message)s') + + print(f"Using log-level: {args.log_level}") + + run_directory = Path(args.rundir) + + available_tutorials = Tutorials.from_path(PRECICE_TUTORIAL_DIR) + + test_suites = TestSuites.from_yaml(PRECICE_TESTS_DIR / "tests.yaml", available_tutorials) + + # Read in parameters + build_args = SystemtestArguments.from_yaml(PRECICE_TESTS_DIR / "reference_versions.yaml") + systemtests_to_run = set() + + for test_suite in test_suites: + tutorials = test_suite.cases_of_tutorial.keys() + for tutorial in tutorials: + for case, reference_result in zip( + test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): + systemtests_to_run.add( + Systemtest(tutorial, build_args, case, reference_result)) + + reference_result_per_tutorial = {} + current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + + logging.info(f"About to run the following tests {systemtests_to_run}") + for number, systemtest in enumerate(systemtests_to_run): + logging.info(f"Started running {systemtest}, {number}/{len(systemtests_to_run)}") + t = time.perf_counter() + result = systemtest.run_for_reference_results(run_directory) + elapsed_time = time.perf_counter() - t + logging.info(f"Running {systemtest} took {elapsed_time} seconds") + if not result.success: + raise RuntimeError(f"Failed to execute {systemtest}") + reference_result_per_tutorial[systemtest.tutorial] = [] + + # Put the tar.gz in there + for systemtest in systemtests_to_run: + reference_result_folder = systemtest.get_system_test_dir() / PRECICE_REL_OUTPUT_DIR + reference_result_per_tutorial[systemtest.tutorial].append(systemtest.reference_result) + # create folder if needed + systemtest.reference_result.path.parent.mkdir(parents=True, exist_ok=True) + if reference_result_folder.exists(): + create_tar_gz(reference_result_folder, systemtest.reference_result.path) + else: + raise RuntimeError( + f"Error executing: \n {systemtest} \n Could not find result folder {reference_result_folder}\n Probably the tutorial did not run through properly. Please check corresponding logs") + + # write readme + for tutorial in reference_result_per_tutorial.keys(): + with open(tutorial.path / "reference_results.metadata", 'w') as file: + ref_results_info = render_reference_results_info( + reference_result_per_tutorial[tutorial], build_args, current_time_string) + logging.info(f"Writing results for {tutorial.name}") + file.write(ref_results_info) + logging.info(f"Done. Please make sure to manually have a look into the reference results before making a PR.") + + +if __name__ == '__main__': + main() diff --git a/tools/tests/print_case_combinations.py b/tools/tests/print_case_combinations.py index 9176331d5..25e318918 100644 --- a/tools/tests/print_case_combinations.py +++ b/tools/tests/print_case_combinations.py @@ -1,16 +1,32 @@ import yaml -from metadata_parser.metdata import Tutorials, Components -from paths import PRECICE_TUTORIAL_DIR, PRECICE_TESTS_DIR +from metadata_parser.metdata import Tutorials +from paths import PRECICE_TUTORIAL_DIR -available_components = Components.from_yaml( - PRECICE_TESTS_DIR / "components.yaml") +import argparse +import logging -available_tutorials = Tutorials.from_path(PRECICE_TUTORIAL_DIR) -tutorials = {} -for tutorial in available_tutorials: - cases_combinations = [ - f"{combination}" for combination in tutorial.case_combinations] - tutorials[tutorial.path] = cases_combinations +def main(): + parser = argparse.ArgumentParser(description='Prints available Metadata for tutorials') + parser.add_argument('--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + default='INFO', help='Set the logging level') + args = parser.parse_args() -print(yaml.dump(tutorials)) + # Configure logging based on the provided log level + logging.basicConfig(level=args.log_level, format='%(levelname)s: %(message)s') + + print(f"Using log-level: {args.log_level}") + + available_tutorials = Tutorials.from_path(PRECICE_TUTORIAL_DIR) + + tutorials = {} + for tutorial in available_tutorials: + cases_combinations = [ + f"{combination}" for combination in tutorial.case_combinations] + tutorials[tutorial.path.name] = cases_combinations + + print(yaml.dump(tutorials)) + + +if __name__ == '__main__': + main() diff --git a/tools/tests/print_metadata.py b/tools/tests/print_metadata.py index 5c568c841..500e549c9 100644 --- a/tools/tests/print_metadata.py +++ b/tools/tests/print_metadata.py @@ -1,7 +1,26 @@ from metadata_parser.metdata import Tutorials from paths import PRECICE_TUTORIAL_DIR -available_tutorials = Tutorials.from_path(PRECICE_TUTORIAL_DIR) -print("Fount the following tutorials read from the metadata.yaml") -for tutorial in available_tutorials: - print(tutorial) +import argparse +import logging + + +def main(): + parser = argparse.ArgumentParser(description='Prints available Metadata for tutorials') + parser.add_argument('--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + default='INFO', help='Set the logging level') + args = parser.parse_args() + + # Configure logging based on the provided log level + logging.basicConfig(level=args.log_level, format='%(levelname)s: %(message)s') + + print(f"Using log-level: {args.log_level}") + + available_tutorials = Tutorials.from_path(PRECICE_TUTORIAL_DIR) + print("Fount the following tutorials read from the metadata.yaml") + for tutorial in available_tutorials: + print(tutorial) + + +if __name__ == '__main__': + main() diff --git a/tools/tests/print_test_suites.py b/tools/tests/print_test_suites.py index 95ccbcb60..6fc1e4039 100644 --- a/tools/tests/print_test_suites.py +++ b/tools/tests/print_test_suites.py @@ -1,11 +1,29 @@ -from metadata_parser.metdata import Tutorials, Components +from metadata_parser.metdata import Tutorials from systemtests.TestSuite import TestSuites from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR -available_tutorials = Tutorials.from_path(PRECICE_TUTORIAL_DIR) -available_components = Components.from_yaml( - PRECICE_TESTS_DIR / "components.yaml") -available_testsuites = TestSuites.from_yaml( - PRECICE_TESTS_DIR / "tests.yaml", available_tutorials) -print(available_testsuites) +import argparse +import logging + + +def main(): + parser = argparse.ArgumentParser(description='Prints available Test Suites') + parser.add_argument('--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + default='INFO', help='Set the logging level') + args = parser.parse_args() + + # Configure logging based on the provided log level + logging.basicConfig(level=args.log_level, format='%(levelname)s: %(message)s') + + print(f"Using log-level: {args.log_level}") + + available_tutorials = Tutorials.from_path(PRECICE_TUTORIAL_DIR) + available_testsuites = TestSuites.from_yaml( + PRECICE_TESTS_DIR / "tests.yaml", available_tutorials) + + print(available_testsuites) + + +if __name__ == '__main__': + main() diff --git a/tools/tests/reference_results.template b/tools/tests/reference_results.metadata.template similarity index 85% rename from tools/tests/reference_results.template rename to tools/tests/reference_results.metadata.template index e4e1eb7f4..d711cd872 100644 --- a/tools/tests/reference_results.template +++ b/tools/tests/reference_results.metadata.template @@ -8,10 +8,10 @@ This file contains an overview of the results over the reference results as well ## List of files -| name | time | sha-1 | +| name | time | sha256 | |------|------|-------| {% for file in files -%} -| {{ file.name }} | {{ file.time }} | {{ file.sha1 }} | +| {{ file.name }} | {{ file.time }} | {{ file.sha256 }} | {% endfor %} ## List of arguments used to generate the files diff --git a/tools/tests/reference_versions.yaml b/tools/tests/reference_versions.yaml index 61c095293..f5c8be62a 100644 --- a/tools/tests/reference_versions.yaml +++ b/tools/tests/reference_versions.yaml @@ -1,3 +1,9 @@ -OPENFOAM_EXECUTABLE: "openfoam2112" -PRECICE_TAG: "latest" -OPENFOAM_ADAPTER_REF: "master" +PRECICE_REF: "v2.5.0" +OPENFOAM_EXECUTABLE: "openfoam2306" +OPENFOAM_ADAPTER_REF: "v1.2.3" +PYTHON_BINDINGS_REF: "v2.5.0.4" +FENICS_ADAPTER_REF: "v1.4.0" +TUTORIALS_REF: "v202211.0" +PLATFORM: "ubuntu_2204" +CALULIX_VERSION: "2.20" +CALULIX_ADAPTER_REF: "v2.20.0" \ No newline at end of file diff --git a/tools/tests/systemtests.py b/tools/tests/systemtests.py index f3707f886..7a57bbde0 100644 --- a/tools/tests/systemtests.py +++ b/tools/tests/systemtests.py @@ -2,79 +2,95 @@ import argparse from pathlib import Path from systemtests.SystemtestArguments import SystemtestArguments -from systemtests.Systemtest import Systemtest +from systemtests.Systemtest import Systemtest, display_systemtestresults_as_table from systemtests.TestSuite import TestSuites from metadata_parser.metdata import Tutorials, Case +import logging +import time +from paths import PRECICE_TUTORIAL_DIR, PRECICE_TESTS_RUN_DIR, PRECICE_TESTS_DIR -from paths import PRECICE_TUTORIAL_DIR, PRECICE_TESTS_RUN_DIR, PRECICE_TESTS_DIR +def main(): + parser = argparse.ArgumentParser(description='systemtest') + + # Add an argument for the components + parser.add_argument('--suites', type=str, + help='Comma-separated test-suites to execute') + parser.add_argument( + '--build_args', + type=str, + help='Comma-separated list of arguments provided to the components like openfoam:2102,pythonbindings:latest') + parser.add_argument('--rundir', type=str, help='Directory to run the systemstests in.', + nargs='?', const=PRECICE_TESTS_RUN_DIR, default=PRECICE_TESTS_RUN_DIR) + + parser.add_argument('--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], + default='INFO', help='Set the logging level') + + # Parse the command-line arguments + args = parser.parse_args() + + # Configure logging based on the provided log level + logging.basicConfig(level=args.log_level, format='%(levelname)s: %(message)s') + + print(f"Using log-level: {args.log_level}") -parser = argparse.ArgumentParser(description='systemtest') - -# Add an argument for the components -parser.add_argument('--suites', type=str, - help='Comma-separated test-suites to execute') -parser.add_argument( - '--build_args', - type=str, - help='Comma-separated list of arguments provided to the components like openfoam:2102,pythonbindings:latest') -parser.add_argument('--rundir', type=str, help='Directory to run the systemstests in.', - nargs='?', const=PRECICE_TESTS_RUN_DIR, default=PRECICE_TESTS_RUN_DIR) - -# Parse the command-line arguments -args = parser.parse_args() -systemtests_to_run = [] -available_tutorials = Tutorials.from_path(PRECICE_TUTORIAL_DIR) - - -build_args = SystemtestArguments.from_args(args.build_args) -run_directory = Path(args.rundir) -if args.suites: - test_suites_requested = args.suites.split(',') - available_testsuites = TestSuites.from_yaml( - PRECICE_TESTS_DIR / "tests.yaml", available_tutorials) - test_suites_to_execute = [] - for test_suite_requested in test_suites_requested: - test_suite_found = available_testsuites.get_by_name( - test_suite_requested) - if not test_suite_found: - print( - f"Warning: did not find the testsuite with name {test_suite_requested}") + systemtests_to_run = [] + available_tutorials = Tutorials.from_path(PRECICE_TUTORIAL_DIR) + + build_args = SystemtestArguments.from_args(args.build_args) + run_directory = Path(args.rundir) + if args.suites: + test_suites_requested = args.suites.split(',') + available_testsuites = TestSuites.from_yaml( + PRECICE_TESTS_DIR / "tests.yaml", available_tutorials) + test_suites_to_execute = [] + for test_suite_requested in test_suites_requested: + test_suite_found = available_testsuites.get_by_name( + test_suite_requested) + if not test_suite_found: + logging.error(f"Did not find the testsuite with name {test_suite_requested}") + else: + test_suites_to_execute.append(test_suite_found) + if not test_suites_to_execute: + raise RuntimeError( + f"No matching test suites with names {test_suites_requested} found. Use print_test_suites.py to get an overview") + # now convert the test_suites into systemtests + for test_suite in test_suites_to_execute: + tutorials = test_suite.cases_of_tutorial.keys() + for tutorial in tutorials: + for case, reference_result in zip( + test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): + systemtests_to_run.append( + Systemtest(tutorial, build_args, case, reference_result)) + + if not systemtests_to_run: + raise RuntimeError("Did not find any Systemtests to execute.") + + logging.info(f"About to run the following systemtest in the directory {run_directory}:\n {systemtests_to_run}") + + results = [] + for number, systemtest in enumerate(systemtests_to_run): + logging.info(f"Started running {systemtest}, {number}/{len(systemtests_to_run)}") + t = time.perf_counter() + result = systemtest.run(run_directory) + elapsed_time = time.perf_counter() - t + logging.info(f"Running {systemtest} took {elapsed_time} seconds") + results.append(result) + + system_test_success = True + for result in results: + if not result.success: + logging.error(f"Failed to run {result.systemtest}") + system_test_success = False else: - test_suites_to_execute.append(test_suite_found) - if not test_suites_to_execute: - raise Exception( - f"No matching test suites with names {test_suites_requested} found. Use print_test_suites.py to get an overview") - # now convert the test_suites into systemtests - for test_suite in test_suites_to_execute: - tutorials = test_suite.cases_of_tutorial.keys() - for tutorial in tutorials: - for case, reference_result in zip( - test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): - systemtests_to_run.append( - Systemtest(tutorial, build_args, case, reference_result)) - - -if not systemtests_to_run: - raise Exception("Did not find any Systemtests to execute.") - - -print( - f"About to run the following systemtest in the directory {run_directory}: \n{systemtests_to_run}") - -results = [] -for systemtest in systemtests_to_run: - results.append(systemtest.run(run_directory)) - -system_test_success = True -for result in results: - if not result.success: - print(f"Failed to run {result.systemtest}") - system_test_success = False + logging.info(f"Success running {result.systemtest}") + + display_systemtestresults_as_table(results) + if system_test_success: + exit(0) else: - print(f"Success running {result.systemtest}") + exit(1) + -if system_test_success: - exit(0) -else: - exit(1) +if __name__ == '__main__': + main() diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 6cc6be7b5..33056c7bc 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -1,20 +1,24 @@ -import os import subprocess -from typing import List, Dict, Tuple +from typing import List, Dict, Optional from jinja2 import Environment, FileSystemLoader from dataclasses import dataclass, field import shutil from pathlib import Path -from paths import PRECICE_REL_OUTPUT_DIR, PRECICE_TOOLS_DIR, PRECICE_REL_REFERENCE_DIR +from paths import PRECICE_REL_OUTPUT_DIR, PRECICE_TOOLS_DIR, PRECICE_REL_REFERENCE_DIR, PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR from metadata_parser.metdata import Tutorial, CaseCombination, Case, ReferenceResult from .SystemtestArguments import SystemtestArguments from datetime import datetime import tarfile +import time import unicodedata import re +import logging + + +GLOBAL_TIMEOUT = 360 def slugify(value, allow_unicode=False): @@ -35,9 +39,6 @@ def slugify(value, allow_unicode=False): return re.sub(r'[-\s]+', '-', value).strip('-_') -system_test_dir = Path(__file__).parent.parent - - class Systemtest: pass @@ -48,6 +49,7 @@ class DockerComposeResult: stdout_data: List[str] stderr_data: List[str] systemtest: Systemtest + runtime: float # in seconds @dataclass @@ -56,6 +58,7 @@ class FieldCompareResult: stdout_data: List[str] stderr_data: List[str] systemtest: Systemtest + runtime: float # in seconds @dataclass @@ -64,6 +67,32 @@ class SystemtestResult: stdout_data: List[str] stderr_data: List[str] systemtest: Systemtest + build_time: float # in seconds + solver_time: float # in seconds + fieldcompare_time: float # in seconds + + +def display_systemtestresults_as_table(results: List[SystemtestResult]): + """ + Prints the result in a nice tabluated way to get an easy overview + """ + def _get_length_of_name(results: List[SystemtestResult]) -> int: + return max(len(str(result.systemtest)) for result in results) + + max_name_length = _get_length_of_name(results) + + header = f"| {'systemtest':<{max_name_length + 2}} | {'success':^7} | {'building time [s]':^17} | {'solver time [s]':^15} | {'fieldcompare time [s]':^21} |" + separator = "+-" + "-" * (max_name_length + 2) + \ + "-+---------+-------------------+-----------------+-----------------------+" + + print(separator) + print(header) + print(separator) + + for result in results: + row = f"| {str(result.systemtest):<{max_name_length + 2}} | {result.success:^7} | {result.build_time:^17.2f} | {result.solver_time:^15.2f} | {result.fieldcompare_time:^21.2f} |" + print(row) + print(separator) @dataclass @@ -128,6 +157,16 @@ def __get_docker_services(self) -> Dict[str, str]: Returns: A dictionary of rendered services per case name. """ + try: + plaform_requested = self.params_to_use.get("PLATFORM") + except Exception as exc: + raise KeyError("Please specify a PLATFORM argument") from exc + + self.dockerfile_context = PRECICE_TESTS_DIR / "dockerfiles" / Path(plaform_requested) + if not self.dockerfile_context.exists(): + raise ValueError( + f"The path {self.dockerfile_context.resolve()} resulting from argument PLATFORM={plaform_requested} could not be found in the system") + def render_service_template_per_case(case: Case, params_to_use: Dict[str, str]) -> str: render_dict = { 'run_directory': self.run_directory.resolve(), @@ -135,9 +174,10 @@ def render_service_template_per_case(case: Case, params_to_use: Dict[str, str]) 'build_arguments': params_to_use, 'params': params_to_use, 'case_folder': case.path, - 'run': case.run_cmd + 'run': case.run_cmd, + 'dockerfile_context': self.dockerfile_context, } - jinja_env = Environment(loader=FileSystemLoader(system_test_dir)) + jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) template = jinja_env.get_template(case.component.template) return template.render(render_dict) @@ -154,9 +194,10 @@ def __get_docker_compose_file(self): 'tutorial_folder': self.tutorial_folder, 'tutorial': self.tutorial.path.name, 'services': rendered_services, + 'dockerfile_context': self.dockerfile_context, 'precice_output_folder': PRECICE_REL_OUTPUT_DIR, } - jinja_env = Environment(loader=FileSystemLoader(system_test_dir)) + jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) template = jinja_env.get_template("docker-compose.template.yaml") return template.render(render_dict) @@ -167,30 +208,69 @@ def __get_field_compare_compose_file(self): 'precice_output_folder': PRECICE_REL_OUTPUT_DIR, 'reference_output_folder': PRECICE_REL_REFERENCE_DIR + "/" + self.reference_result.path.name.replace(".tar.gz", ""), } - jinja_env = Environment(loader=FileSystemLoader(system_test_dir)) + jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) template = jinja_env.get_template( "docker-compose.field_compare.template.yaml") return template.render(render_dict) + def _get_git_ref(self, repository: Path, abbrev_ref=False) -> Optional[str]: + try: + result = subprocess.run([ + "git", + "-C", repository.resolve(), + "rev-parse", + "--abbrev-ref" if abbrev_ref else + "HEAD"], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, check=True, timeout=60) + current_ref = result.stdout.strip() + return current_ref + except Exception as e: + raise RuntimeError(f"An error occurred while getting the current Git ref: {e}") from e + + def _checkout_ref_in_subfolder(self, repository: Path, subfolder: Path, ref: str): + try: + result = subprocess.run([ + "git", + "-C", repository.resolve(), + "checkout", ref, + "--", subfolder.resolve() + ], check=True, timeout=60) + if result.returncode != 0: + raise RuntimeError(f"git command returned code {result.returncode}") + + except Exception as e: + raise RuntimeError(f"An error occurred while checking out '{ref}' for folder '{repository}': {e}") + def __copy_tutorial_into_directory(self, run_directory: Path): """ - Copies the entire tutorial into a folder to prepare for running. + Checks out the requested tutorial ref and copies the entire tutorial into a folder to prepare for running. """ current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') self.run_directory = run_directory + current_ref = self._get_git_ref(PRECICE_TUTORIAL_DIR) + ref_requested = self.params_to_use.get("TUTORIALS_REF") + if ref_requested: + logging.debug(f"Checking out tutorials {ref_requested} before copying") + self._checkout_ref_in_subfolder(PRECICE_TUTORIAL_DIR, self.tutorial.path, ref_requested) + self.tutorial_folder = slugify(f'{self.tutorial.path.name}_{self.case_combination.cases}_{current_time_string}') destination = run_directory / self.tutorial_folder src = self.tutorial.path self.system_test_dir = destination shutil.copytree(src, destination) + if ref_requested: + with open(destination / "tutorials_ref", 'w') as file: + file.write(ref_requested) + self._checkout_ref_in_subfolder(PRECICE_TUTORIAL_DIR, self.tutorial.path, current_ref) + def __copy_tools(self, run_directory: Path): destination = run_directory / "tools" src = PRECICE_TOOLS_DIR try: shutil.copytree(src, destination) except Exception as e: - print("tools are already copied: ", e) + logging.debug(f"tools are already copied: {e} ") def __put_gitignore(self, run_directory: Path): # Create the .gitignore file with a single asterisk @@ -207,7 +287,7 @@ def __get_uid_gid(self): gid = int(subprocess.check_output(["id", "-g"]).strip()) return uid, gid except Exception as e: - print("Error getting group and user id: ", e) + logging.error("Error getting group and user id: ", e) def __write_env_file(self): with open(self.system_test_dir / ".env", "w") as env_file: @@ -218,7 +298,8 @@ def __unpack_reference_results(self): with tarfile.open(self.reference_result.path) as reference_results_tared: # specify which folder to extract to reference_results_tared.extractall(self.system_test_dir / PRECICE_REL_REFERENCE_DIR) - print(f"extracting {self.reference_result.path} into {self.system_test_dir / PRECICE_REL_REFERENCE_DIR}") + logging.debug( + f"extracting {self.reference_result.path} into {self.system_test_dir / PRECICE_REL_REFERENCE_DIR}") def _run_field_compare(self): """ @@ -230,6 +311,8 @@ def _run_field_compare(self): Returns: A SystemtestResult object containing the state. """ + logging.debug(f"Running fieldcompare for {self}") + time_start = time.perf_counter() self.__unpack_reference_results() docker_compose_content = self.__get_field_compare_compose_file() stdout_data = [] @@ -248,27 +331,75 @@ def _run_field_compare(self): 'field-compare'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True, cwd=self.system_test_dir) - # Read the output in real-time - while True: - output = process.stdout.readline().decode() - if output == '' and process.poll() is not None: - break - if output: - stdout_data.append(output) - print(output, end='') - - # Capture remaining output - stdout, stderr = process.communicate() + try: + stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + except KeyboardInterrupt as k: + process.kill() + # process.send_signal(9) + raise KeyboardInterrupt from k + except Exception as e: + logging.critical( + f"Systemtest {self} had serious issues executin the docker compose command about to kill the docker compose command. Please check the logs! {e}") + process.kill() + stdout, stderr = process.communicate() stdout_data.extend(stdout.decode().splitlines()) stderr_data.extend(stderr.decode().splitlines()) + process.poll() + elapsed_time = time.perf_counter() - time_start + return FieldCompareResult(process.returncode, stdout_data, stderr_data, self, elapsed_time) + except Exception as e: + logging.CRITICAL("Error executing docker compose command:", e) + elapsed_time = time.perf_counter() - time_start + return FieldCompareResult(1, stdout_data, stderr_data, self, elapsed_time) + + def _build_docker(self): + """ + Builds the docker image + """ + logging.debug(f"Building docker image for {self}") + time_start = time.perf_counter() + docker_compose_content = self.__get_docker_compose_file() + with open(self.system_test_dir / "docker-compose.tutorial.yaml", 'w') as file: + file.write(docker_compose_content) + + stdout_data = [] + stderr_data = [] + + try: + # Execute docker-compose command + process = subprocess.Popen(['docker', + 'compose', + '--file', + 'docker-compose.tutorial.yaml', + 'build'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + cwd=self.system_test_dir) - exit_code = process.wait() - return FieldCompareResult(exit_code, stdout_data, stderr_data, self) + try: + stdout, stderr = process.communicate() + except KeyboardInterrupt as k: + process.kill() + # process.send_signal(9) + raise KeyboardInterrupt from k + except Exception as e: + logging.critical( + f"systemtest {self} had serious issues building the docker images via the `docker compose build` command. About to kill the docker compose command. Please check the logs! {e}") + process.kill() + stdout, stderr = process.communicate() + + stdout_data.extend(stdout.decode().splitlines()) + stderr_data.extend(stderr.decode().splitlines()) + elapsed_time = time.perf_counter() - time_start + return DockerComposeResult(process.returncode, stdout_data, stderr_data, self, elapsed_time) except Exception as e: - print("Error executing docker compose command:", e) - return FieldCompareResult(1, stdout_data, stderr_data, self) + logging.critical(f"Error executing docker compose build command: {e}") + elapsed_time = time.perf_counter() - time_start + return DockerComposeResult(1, stdout_data, stderr_data, self, elapsed_time) def _run_tutorial(self): """ @@ -277,104 +408,206 @@ def _run_tutorial(self): Returns: A DockerComposeResult object containing the state. """ - docker_compose_content = self.__get_docker_compose_file() + logging.debug(f"Running tutorial {self}") + time_start = time.perf_counter() stdout_data = [] stderr_data = [] - - with open(self.system_test_dir / "docker-compose.tutorial.yaml", 'w') as file: - file.write(docker_compose_content) try: # Execute docker-compose command process = subprocess.Popen(['docker', 'compose', '--file', 'docker-compose.tutorial.yaml', - 'up', - "--build"], + 'up'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True, cwd=self.system_test_dir) - # Read the output in real-time - while True: - output = process.stdout.readline().decode() - if output == '' and process.poll() is not None: - break - if output: - stdout_data.append(output) - print(output, end='') - - # Capture remaining output - stdout, stderr = process.communicate() + try: + stdout, stderr = process.communicate(timeout=GLOBAL_TIMEOUT) + except KeyboardInterrupt as k: + process.kill() + # process.send_signal(9) + raise KeyboardInterrupt from k + except Exception as e: + logging.critical( + f"Systemtest {self} had serious issues executin the docker compose command about to kill the docker compose command. Please check the logs! {e}") + process.kill() + stdout, stderr = process.communicate() + stdout_data.extend(stdout.decode().splitlines()) stderr_data.extend(stderr.decode().splitlines()) - - exit_code = process.wait() - return DockerComposeResult(exit_code, stdout_data, stderr_data, self) + elapsed_time = time.perf_counter() - time_start + return DockerComposeResult(process.returncode, stdout_data, stderr_data, self, elapsed_time) except Exception as e: - print("Error executing docker compose command:", e) - return DockerComposeResult(1, stdout_data, stderr_data, self) + logging.critical(f"Error executing docker compose up command: {e}") + elapsed_time = time.perf_counter() - time_start + return DockerComposeResult(1, stdout_data, stderr_data, self, elapsed_time) def __repr__(self): return f"{self.tutorial.name} {self.case_combination}" - def __handle_docker_compose_failure(self, result: DockerComposeResult): - print("Docker Compose failed, skipping fieldcompare") + def __write_logs(self, stdout_data: List[str], stderr_data: List[str]): + with open(self.system_test_dir / "stdout.log", 'w') as stdout_file: + stdout_file.write("\n".join(stdout_data)) + with open(self.system_test_dir / "stderr.log", 'w') as stderr_file: + stderr_file.write("\n".join(stderr_data)) - def __handle_field_compare_failure(self, result: FieldCompareResult): - print("Fieldcompare failed") - - def run(self, run_directory: Path): + def __prepare_for_run(self, run_directory: Path): """ - Runs the system test by generating the Docker Compose file, copying everything into a run folder, and executing docker-compose up. + Prepares the run_directory with folders and datastructures needed for every systemtest execution """ self.__copy_tutorial_into_directory(run_directory) self.__copy_tools(run_directory) self.__put_gitignore(run_directory) - std_out: List[str] = [] - std_err: List[str] = [] uid, gid = self.__get_uid_gid() self.env["UID"] = uid self.env["GID"] = gid self.__write_env_file() - docker_compose_result = self._run_tutorial() - std_out.extend(docker_compose_result.stdout_data) - std_err.extend(docker_compose_result.stderr_data) - if docker_compose_result.exit_code == 1: - self.__handle_docker_compose_failure(docker_compose_result) - return SystemtestResult(False, std_out, std_err, self) + + def run(self, run_directory: Path): + """ + Runs the system test by generating the Docker Compose file, copying everything into a run folder, and executing docker-compose up. + """ + self.__prepare_for_run(run_directory) + std_out: List[str] = [] + std_err: List[str] = [] + + docker_build_result = self._build_docker() + std_out.extend(docker_build_result.stdout_data) + std_err.extend(docker_build_result.stderr_data) + if docker_build_result.exit_code != 0: + self.__write_logs(std_out, std_err) + logging.critical(f"Could not build the docker images, {self} failed") + return SystemtestResult( + False, + std_out, + std_err, + self, + build_time=docker_build_result.runtime, + solver_time=0, + fieldcompare_time=0) + + docker_run_result = self._run_tutorial() + std_out.extend(docker_run_result.stdout_data) + std_err.extend(docker_run_result.stderr_data) + if docker_run_result.exit_code != 0: + self.__write_logs(std_out, std_err) + logging.critical(f"Could not run the tutorial, {self} failed") + return SystemtestResult( + False, + std_out, + std_err, + self, + build_time=docker_build_result.runtime, + solver_time=docker_run_result.runtime, + fieldcompare_time=0) fieldcompare_result = self._run_field_compare() std_out.extend(fieldcompare_result.stdout_data) std_err.extend(fieldcompare_result.stderr_data) - if fieldcompare_result.exit_code == 1: - self.__handle_field_compare_failure(fieldcompare_result) - return SystemtestResult(False, std_out, std_err, self) + if fieldcompare_result.exit_code != 0: + self.__write_logs(std_out, std_err) + logging.critical(f"Fieldcompare returned non zero exit code, therefore {self} failed") + return SystemtestResult( + False, + std_out, + std_err, + self, + build_time=docker_build_result.runtime, + solver_time=docker_run_result.runtime, + fieldcompare_time=fieldcompare_result.runtime) # self.__cleanup() - return SystemtestResult(True, std_out, std_err, self) + self.__write_logs(std_out, std_err) + return SystemtestResult( + True, + std_out, + std_err, + self, + build_time=docker_build_result.runtime, + solver_time=docker_run_result.runtime, + fieldcompare_time=fieldcompare_result.runtime) def run_for_reference_results(self, run_directory: Path): """ Runs the system test by generating the Docker Compose files to generate the reference results """ - self.__copy_tutorial_into_directory(run_directory) - self.__copy_tools(run_directory) - self.__put_gitignore(run_directory) + self.__prepare_for_run(run_directory) std_out: List[str] = [] std_err: List[str] = [] - uid, gid = self.__get_uid_gid() - self.env["UID"] = uid - self.env["GID"] = gid - self.__write_env_file() - docker_compose_result = self._run_tutorial() - std_out.extend(docker_compose_result.stdout_data) - std_err.extend(docker_compose_result.stderr_data) - if docker_compose_result.exit_code == 1: - self.__handle_docker_compose_failure(docker_compose_result) - return SystemtestResult(False, std_out, std_err, self) - - return SystemtestResult(True, std_out, std_err, self) + docker_build_result = self._build_docker() + std_out.extend(docker_build_result.stdout_data) + std_err.extend(docker_build_result.stderr_data) + if docker_build_result.exit_code != 0: + self.__write_logs(std_out, std_err) + logging.critical(f"Could not build the docker images, {self} failed") + return SystemtestResult( + False, + std_out, + std_err, + self, + build_time=docker_build_result.runtime, + solver_time=0, + fieldcompare_time=0) + + docker_run_result = self._run_tutorial() + std_out.extend(docker_run_result.stdout_data) + std_err.extend(docker_run_result.stderr_data) + if docker_run_result.exit_code != 0: + self.__write_logs(std_out, std_err) + logging.critical(f"Could not run the tutorial, {self} failed") + return SystemtestResult( + False, + std_out, + std_err, + self, + build_time=docker_build_result.runtime, + solver_time=docker_run_result.runtime, + fieldcompare_time=0) + + self.__write_logs(std_out, std_err) + return SystemtestResult( + True, + std_out, + std_err, + self, + build_time=docker_build_result.runtime, + solver_time=docker_run_result.runtime, + fieldcompare_time=0) + + def run_only_build(self, run_directory: Path): + """ + Runs only the build commmand, for example to preheat the caches of the docker builder. + """ + self.__prepare_for_run(run_directory) + std_out: List[str] = [] + std_err: List[str] = [] + docker_build_result = self._build_docker() + std_out.extend(docker_build_result.stdout_data) + std_err.extend(docker_build_result.stderr_data) + if docker_build_result.exit_code != 0: + self.__write_logs(std_out, std_err) + logging.critical(f"Could not build the docker images, {self} failed") + return SystemtestResult( + False, + std_out, + std_err, + self, + build_time=docker_build_result.runtime, + solver_time=0, + fieldcompare_time=0) + + self.__write_logs(std_out, std_err) + return SystemtestResult( + True, + std_out, + std_err, + self, + build_time=docker_build_result.runtime, + solver_time=0, + fieldcompare_time=0) def get_system_test_dir(self) -> Path: return self.system_test_dir diff --git a/tools/tests/systemtests/SystemtestArguments.py b/tools/tests/systemtests/SystemtestArguments.py index 4afd5e6ba..2b7b3eeec 100644 --- a/tools/tests/systemtests/SystemtestArguments.py +++ b/tools/tests/systemtests/SystemtestArguments.py @@ -1,5 +1,6 @@ from dataclasses import dataclass import yaml +from typing import Optional @dataclass @@ -34,5 +35,5 @@ def __repr__(self): def contains(self, argument_key): return argument_key in self.arguments.keys() - def get(self, argument_key): + def get(self, argument_key) -> Optional[str]: return self.arguments[argument_key] diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 58d92c6b9..827eae38e 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -13,3 +13,25 @@ test_suites: - fluid-openfoam - solid-openfoam reference_result: ./flow-over-heated-plate/reference-data/fluid-openfoam_solid-openfoam.tar.gz + fenics_test: + tutorials: + - path: flow-over-heated-plate + case_combination: + - fluid-openfoam + - solid-fenics + reference_result: ./flow-over-heated-plate/reference-data/fluid-openfoam_solid-fenics.tar.gz + nutils_test: + tutorials: + - path: flow-over-heated-plate + case_combination: + - fluid-openfoam + - solid-nutils + reference_result: ./flow-over-heated-plate/reference-data/fluid-openfoam_solid-nutils.tar.gz + calculix_test: + tutorials: + - path: perpendicular-flap + case_combination: + - fluid-openfoam + - solid-calculix + reference_result: ./perpendicular-flap/reference-data/fluid-openfoam_solid-calculix.tar.gz + timeout: \ No newline at end of file