From df240174dd7605f3606c67d14ff85af47665810d Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Tue, 19 Sep 2023 11:17:33 +0100 Subject: [PATCH 1/2] Add build to readthedocs config (#70) --- .readthedocs.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index aa61218..326f882 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,6 +1,9 @@ version: 2 formats: [] +build: + os: "ubuntu-22.04" + tools: + python: "3.9" python: - version: 3.8 install: - - requirements: docs/requirements.txt + - requirements: docs/requirements.txt From ca832c34eac664161f7b2fc6ed9bc7280289bf8d Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Thu, 19 Oct 2023 16:20:54 +0100 Subject: [PATCH 2/2] Add downstream ci (#68) * Add downstream ci --------- Co-authored-by: Iain Russell Co-authored-by: Iain Russell <40060766+iainrussell@users.noreply.github.com> Co-authored-by: Dusan Figala --- .github/ci-config.yml | 7 +++ .github/ci-hpc-config.yml | 14 +++++ .github/workflows/ci.yml | 76 +++++++++++++++++++++++++++ .github/workflows/label-public-pr-yml | 10 ++++ MANIFEST.in | 1 + setup.py | 2 +- tests/downstream-ci-requirements.txt | 1 + tests/test_40_sample_data.py | 16 ++++-- 8 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 .github/ci-config.yml create mode 100644 .github/ci-hpc-config.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/label-public-pr-yml create mode 100644 tests/downstream-ci-requirements.txt diff --git a/.github/ci-config.yml b/.github/ci-config.yml new file mode 100644 index 0000000..77c9e81 --- /dev/null +++ b/.github/ci-config.yml @@ -0,0 +1,7 @@ +dependencies: | + ecmwf/ecbuild + MathisRosenhauer/libaec@master + ecmwf/eccodes +dependency_branch: develop +parallelism_factor: 8 +self_build: false diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml new file mode 100644 index 0000000..5996da1 --- /dev/null +++ b/.github/ci-hpc-config.yml @@ -0,0 +1,14 @@ +build: + python: '3.10' + modules: + - ninja + dependencies: + - ecmwf/ecbuild@develop + - ecmwf/eccodes@develop + python_dependencies: + - ecmwf/eccodes-python@develop + env: + - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples + - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions + parallel: 64 + requirements: tests/downstream-ci-requirements.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8588c67 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,76 @@ +name: ci + +on: + # Trigger the workflow on push to master or develop, except tag creation + push: + branches: + - 'master' + - 'develop' + tags-ignore: + - '**' + + # Trigger the workflow on pull request + pull_request: ~ + + # Trigger the workflow manually + workflow_dispatch: ~ + + # Trigger after public PR approved for CI + pull_request_target: + types: [labeled] + +jobs: + # Run CI including downstream packages on self-hosted runners + downstream-ci: + name: downstream-ci + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main + with: + pdbufr: ecmwf/pdbufr@${{ github.event.pull_request.head.sha || github.sha }} + codecov_upload: true + secrets: inherit + + # # Run CI of private downstream packages on self-hosted runners + # private-downstream-ci: + # name: private-downstream-ci + # needs: [downstream-ci] + # if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + # runs-on: ubuntu-latest + # permissions: + # pull-requests: write + # steps: + # - name: Dispatch private downstream CI + # uses: ecmwf-actions/dispatch-private-downstream-ci@v1 + # with: + # token: ${{ secrets.GH_REPO_READ_TOKEN }} + # owner: ecmwf-actions + # repository: private-downstream-ci + # event_type: downstream-ci + # payload: '{"metkit": "ecmwf/metkit@${{ github.event.pull_request.head.sha || github.sha }}"}' + + # Build downstream packages on HPC + downstream-ci-hpc: + name: downstream-ci-hpc + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main + with: + pdbufr: ecmwf/pdbufr@${{ github.event.pull_request.head.sha || github.sha }} + secrets: inherit + + # # Run CI of private downstream packages on HPC + # private-downstream-ci-hpc: + # name: private-downstream-ci-hpc + # needs: [downstream-ci-hpc] + # if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + # runs-on: ubuntu-latest + # permissions: + # pull-requests: write + # steps: + # - name: Dispatch private downstream CI + # uses: ecmwf-actions/dispatch-private-downstream-ci@v1 + # with: + # token: ${{ secrets.GH_REPO_READ_TOKEN }} + # owner: ecmwf-actions + # repository: private-downstream-ci + # event_type: downstream-ci-hpc + # payload: '{"metkit": "ecmwf/metkit@${{ github.event.pull_request.head.sha || github.sha }}"}' \ No newline at end of file diff --git a/.github/workflows/label-public-pr-yml b/.github/workflows/label-public-pr-yml new file mode 100644 index 0000000..bda6c82 --- /dev/null +++ b/.github/workflows/label-public-pr-yml @@ -0,0 +1,10 @@ +# Manage labels of pull requests that originate from forks +name: label-public-pr + +on: + pull_request_target: + types: [opened, synchronize] + +jobs: + label: + uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2 diff --git a/MANIFEST.in b/MANIFEST.in index b372f46..32f42de 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,4 +9,5 @@ recursive-include tests *.py recursive-include tests *.yml recursive-include tests *.csv recursive-include docs * +recursive-include tests *.txt diff --git a/setup.py b/setup.py index 7f96c44..de5bee7 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ def parse_version_from(path: str) -> str: url="https://github.com/ecmwf/pdbufr", packages=setuptools.find_packages(), include_package_data=True, - install_requires=["attrs", "eccodes", "pandas"], + install_requires=["attrs", "eccodes", "pandas", "hypothesis"], extras_require={"tests": ["flake8", "pytest", "pytest-cov", "requests"]}, zip_safe=True, keywords="eccodes bufr pandas", diff --git a/tests/downstream-ci-requirements.txt b/tests/downstream-ci-requirements.txt new file mode 100644 index 0000000..663bd1f --- /dev/null +++ b/tests/downstream-ci-requirements.txt @@ -0,0 +1 @@ +requests \ No newline at end of file diff --git a/tests/test_40_sample_data.py b/tests/test_40_sample_data.py index fd1d7c3..251ba2d 100644 --- a/tests/test_40_sample_data.py +++ b/tests/test_40_sample_data.py @@ -734,6 +734,14 @@ def test_ens_compressed() -> None: def test_sat_compressed_1() -> None: + def timestamp(s: str) -> T.Any: + ts = pd.Timestamp(s) + try: + ts = ts.as_unit("ns") + except: + pass + return ts + columns = [ "data_datetime", "latitude", @@ -745,7 +753,7 @@ def test_sat_compressed_1() -> None: ] expected_first_row = { - "data_datetime": pd.Timestamp("2015-08-21 01:59:05"), + "data_datetime": timestamp("2015-08-21 01:59:05"), "latitude": -44.83389, "longitude": 171.1635, "nonCoordinateLatitude": -44.82399, @@ -755,7 +763,7 @@ def test_sat_compressed_1() -> None: } expected_second_row = { - "data_datetime": pd.Timestamp("2015-08-21 01:59:05"), + "data_datetime": timestamp("2015-08-21 01:59:05"), "latitude": -44.83389, "longitude": 171.1635, "nonCoordinateLatitude": -44.82399, @@ -765,7 +773,7 @@ def test_sat_compressed_1() -> None: } expected_12_row = { - "data_datetime": pd.Timestamp("2015-08-21 01:59:05"), + "data_datetime": timestamp("2015-08-21 01:59:05"), "latitude": -44.83389, "longitude": 171.1635, "nonCoordinateLatitude": -44.82399, @@ -775,7 +783,7 @@ def test_sat_compressed_1() -> None: } expected_13_row = { - "data_datetime": pd.Timestamp("2015-08-21 01:59:06"), + "data_datetime": timestamp("2015-08-21 01:59:06"), "latitude": -44.77121, "longitude": 171.1515, "nonCoordinateLatitude": -44.76132,