From 03caeb00d996ff22353031f091570040ae691fa4 Mon Sep 17 00:00:00 2001 From: Steve Arnold Date: Mon, 8 Apr 2024 12:04:27 -0700 Subject: [PATCH 01/17] chg: dev: update python, deps, GH action versions, and tox file * update pybind11 usage and set cmake python vars to Title_CASE * refactor cmake extension build to use pybind11 module bits * move emptygroups test from "differences" Signed-off-by: Steve Arnold --- .github/workflows/ci.yml | 8 +++--- .github/workflows/conda.yml | 10 +++---- .github/workflows/main.yml | 20 +++++++------- .github/workflows/release.yml | 26 +++++++++--------- CMakeLists.txt | 33 ++++++++++++----------- cmake/modules/FindCython.cmake | 6 ++--- pyproject.toml | 1 - setup.py | 2 +- src/CMakeLists.txt | 23 +++++++--------- tests/test_emptygroups.txt | 9 +++---- tox.ini | 48 ++++++++++++++++++++-------------- 11 files changed, 95 insertions(+), 91 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e240970..5a1e8245 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,15 +17,15 @@ jobs: strategy: fail-fast: true matrix: - os: [ubuntu-20.04] - python-version: [3.6, 3.7, 3.8, 3.9] + os: [ubuntu-22.04] + python-version: [3.8, 3.9, '3.10', '3.11', '3.12'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index bf3abd30..2a298903 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -12,8 +12,8 @@ jobs: strategy: fail-fast: false matrix: - platform: [ubuntu-latest, windows-2016, macos-latest] - python-version: [3.8, 3.9] + platform: [ubuntu-latest, windows-2019, macos-latest] + python-version: [3.8, '3.10'] runs-on: ${{ matrix.platform }} @@ -23,18 +23,18 @@ jobs: shell: "bash -l {0}" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Cache conda - uses: actions/cache@v1 + uses: actions/cache@v4 with: path: ~/conda_pkgs_dir key: ${{matrix.os}}-conda-pkgs-${{hashFiles('**/conda.recipe/meta.yaml')}} - name: Get conda - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python-version }} channels: conda-forge diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a9bc2175..702aa5c8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,14 +13,14 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, macos-latest, windows-latest] + os: [ubuntu-22.04, macos-latest, windows-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 name: Install Python with: python-version: '3.8' @@ -40,7 +40,7 @@ jobs: env: CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux2010_x86_64:latest CIBW_MANYLINUX_I686_IMAGE: quay.io/pypa/manylinux2010_i686:latest - CIBW_BUILD: cp36-* cp37-* cp38-* cp39-* + CIBW_BUILD: cp38-* cp39-* cp310-* cp311-* CIBW_SKIP: "*-win32" CIBW_BEFORE_ALL_LINUX: > yum -y -q --enablerepo=extras install epel-release @@ -58,7 +58,7 @@ jobs: run: | python -m cibuildwheel --output-dir wheelhouse - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: path: ./wheelhouse/*.whl @@ -66,9 +66,9 @@ jobs: name: Build source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 name: Install Python with: python-version: '3.7' @@ -78,7 +78,7 @@ jobs: pip install pep517 python -m pep517.build -s . - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: path: dist/*.tar.gz @@ -90,8 +90,8 @@ jobs: name: Check artifacts are correct runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 # note wheels should be in subdirectory - name: Check number of downloaded artifacts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e1251517..3fd96e07 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,14 +12,14 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, macos-latest, windows-latest] + os: [ubuntu-22.04, macos-latest, windows-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 name: Install Python with: python-version: '3.8' @@ -39,7 +39,7 @@ jobs: env: CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux2010_x86_64:latest CIBW_MANYLINUX_I686_IMAGE: quay.io/pypa/manylinux2010_i686:latest - CIBW_BUILD: cp36-* cp37-* cp38-* cp39-* + CIBW_BUILD: cp38-* cp39-* cp310-* cp311-* CIBW_SKIP: "*-win32" CIBW_BEFORE_ALL_LINUX: > yum -y -q --enablerepo=extras install epel-release @@ -57,7 +57,7 @@ jobs: run: | python -m cibuildwheel --output-dir wheelhouse - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: path: ./wheelhouse/*.whl @@ -65,9 +65,9 @@ jobs: name: Build source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 name: Install Python with: python-version: '3.8' @@ -77,13 +77,13 @@ jobs: pip install pep517 python -m pep517.build -s . - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: path: dist/*.tar.gz create_release: needs: [build_sdist, cibw_wheels] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Get version @@ -92,17 +92,17 @@ jobs: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV echo ${{ env.VERSION }} - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 name: Install Python with: python-version: 3.7 # download all artifacts to project dir - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v4 - name: Generate changes file uses: sarnold/gitchangelog-action@master @@ -111,7 +111,7 @@ jobs: - name: Create draft release id: create_release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@main env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/CMakeLists.txt b/CMakeLists.txt index 83eff6e0..d12bc608 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.15...3.18) +cmake_minimum_required(VERSION 3.15...3.28) project(re2 LANGUAGES CXX C) @@ -9,8 +9,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) if(CMAKE_CXX_COMPILER_ID STREQUAL Clang) - set(CLANG_DEFAULT_CXX_STDLIB libc++) - set(CLANG_DEFAULT_RTLIB compiler-rt) + set(CLANG_DEFAULT_CXX_STDLIB libc++) + set(CLANG_DEFAULT_RTLIB compiler-rt) endif() if(NOT CMAKE_BUILD_TYPE) @@ -20,27 +20,30 @@ endif() include(GNUInstallDirs) +# get rid of FindPython old warnings, refactor FindCython module +set(CMP0148 NEW) + find_package(pybind11 CONFIG) if(pybind11_FOUND) - message(STATUS "System pybind11 found") + message(STATUS "System pybind11 found") else() - message(STATUS "Fetching pybind11 from github") - # Fetch pybind11 - include(FetchContent) - - FetchContent_Declare( - pybind11 - GIT_REPOSITORY https://github.com/pybind/pybind11 - GIT_TAG v2.6.1 - ) - FetchContent_MakeAvailable(pybind11) + message(STATUS "Fetching pybind11 from github") + # Fetch pybind11 + include(FetchContent) + + FetchContent_Declare( + pybind11 + GIT_REPOSITORY https://github.com/pybind/pybind11 + GIT_TAG v2.12.0 + ) + FetchContent_MakeAvailable(pybind11) endif() find_package(Threads REQUIRED) if (${PYTHON_IS_DEBUG}) - set(PY_DEBUG ON) + set(PY_DEBUG ON) endif() set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} diff --git a/cmake/modules/FindCython.cmake b/cmake/modules/FindCython.cmake index 04aed1f8..c53e2b83 100644 --- a/cmake/modules/FindCython.cmake +++ b/cmake/modules/FindCython.cmake @@ -24,9 +24,9 @@ # Use the Cython executable that lives next to the Python executable # if it is a local installation. -find_package( PythonInterp ) -if( PYTHONINTERP_FOUND ) - get_filename_component( _python_path ${PYTHON_EXECUTABLE} PATH ) +find_package(Python) +if( Python_FOUND ) + get_filename_component( _python_path ${Python_EXECUTABLE} PATH ) find_program( CYTHON_EXECUTABLE NAMES cython cython.bat cython3 HINTS ${_python_path} diff --git a/pyproject.toml b/pyproject.toml index 18c975e1..d2d6a428 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,6 @@ [build-system] requires = [ "setuptools>=42", - "wheel", "Cython>=0.20", "pybind11>=2.6.0", "ninja; sys_platform != 'Windows'", diff --git a/setup.py b/setup.py index 87c65b23..e76c5e3d 100755 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ def build_extension(self, ext): # from Python. cmake_args = [ "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}".format(extdir), - "-DPYTHON_EXECUTABLE={}".format(sys.executable), + "-DPython_EXECUTABLE={}".format(sys.executable), "-DSCM_VERSION_INFO={}".format(__version__), "-DCMAKE_BUILD_TYPE={}".format(cfg), # not used on MSVC, but no harm ] diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 61d63aa3..8e0372d2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,16 +21,11 @@ add_custom_command(OUTPUT ${cython_output} DEPENDS ${cy_srcs} COMMENT "Cythonizing extension ${cython_src}") -add_library(${cython_module} MODULE ${cython_output}) +pybind11_add_module(${cython_module} MODULE ${cython_output}) -set_target_properties(${cython_module} - PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" - SUFFIX "${PYTHON_MODULE_EXTENSION}") - -target_include_directories(${cython_module} PUBLIC - ${PYTHON_INCLUDE_DIRS}) - -target_compile_definitions(${cython_module} PRIVATE VERSION_INFO=${SCM_VERSION_INFO}) +target_compile_definitions( + ${cython_module} PRIVATE VERSION_INFO=${SCM_VERSION_INFO} +) # here we get to jump through some hoops to find libre2 on the manylinux # docker CI images, etc @@ -57,12 +52,12 @@ endif() if(APPLE) # macos/appleclang needs this - target_link_libraries(${cython_module} PRIVATE pybind11::module) - target_link_libraries(${cython_module} PRIVATE pybind11::python_link_helper) + target_link_libraries(${cython_module} PUBLIC pybind11::module) + target_link_libraries(${cython_module} PUBLIC pybind11::python_link_helper) endif() if(MSVC) - target_compile_options(${cython_module} PRIVATE /utf-8) - target_link_libraries(${cython_module} PRIVATE ${PYTHON_LIBRARIES}) - target_link_libraries(${cython_module} PRIVATE pybind11::windows_extras) + target_compile_options(${cython_module} PUBLIC /utf-8) + target_link_libraries(${cython_module} PUBLIC ${Python_LIBRARIES}) + target_link_libraries(${cython_module} PUBLIC pybind11::windows_extras) endif() diff --git a/tests/test_emptygroups.txt b/tests/test_emptygroups.txt index 4a5bd5bc..b55ca650 100644 --- a/tests/test_emptygroups.txt +++ b/tests/test_emptygroups.txt @@ -20,6 +20,10 @@ Unused vs. empty group: ('a', '') >>> re2.search(r'((.*)+.)', 'a').groups() ('a', '') + >>> re.search(r'((.*)*.)', 'a').groups() + ('a', '') + >>> re2.search(r'((.*)*.)', 'a').groups() + ('a', '') The following show different behavior for re and re2: @@ -28,9 +32,4 @@ The following show different behavior for re and re2: >>> re2.search(r'((.*)*.)', 'Hello').groups() ('Hello', 'Hell') - >>> re.search(r'((.*)*.)', 'a').groups() - ('a', '') - >>> re2.search(r'((.*)*.)', 'a').groups() - ('a', None) - >>> re2.set_fallback_notification(re2.FALLBACK_QUIETLY) diff --git a/tox.ini b/tox.ini index aa388ad0..32143d92 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,23 @@ [tox] -envlist = py3{6,7,8,9} +envlist = py3{7,8,9,10,11,12} skip_missing_interpreters = true isolated_build = true skipsdist=True [gh-actions] -3.6 = py36 -3.7 = py37 -3.8 = py38 -3.9 = py39 +python = + 3.7: py37 + 3.8: py38 + 3.9: py39 + 3.10: py310 + 3.11: py311 + 3.12: py312 + +[gh-actions:env] +PLATFORM = + ubuntu-22.04: linux + macos-latest: macos + windows-latest: windows [testenv] passenv = @@ -22,11 +31,9 @@ passenv = deps = pip>=20.0.1 - path + -e .[test] commands = - python -c "import path; path.Path('build').rmtree_p()" - pip install -e .[test] pytest -v . [testenv:dev] @@ -47,11 +54,9 @@ setenv = deps = pip>=20.0.1 cython>=0.20 - path pytest commands = - python -c "import path; path.Path('build').rmtree_p()" python setup.py build_ext --inplace # use --capture=no to see all the doctest output python -m pytest -v --ignore=tests/test_re.py --doctest-glob=*.txt . @@ -69,14 +74,12 @@ passenv = deps = pip>=20.0.1 - path + .[perf] commands = - python -c "import path; path.Path('build').rmtree_p()" - pip install .[perf] python tests/performance.py -[testenv:deploy] +[testenv:build] passenv = pythonLocation CI @@ -93,14 +96,10 @@ deps = pip>=20.0.1 build twine - path commands = - python -c "import path; path.Path('build').rmtree_p()" python -m build . twine check dist/* - python -m pip install https://github.com/sarnold/gitchangelog/archive/3.0.7.tar.gz - bash -c 'gitchangelog $(git describe --abbrev=0)..' [testenv:check] skip_install = true @@ -111,9 +110,18 @@ allowlist_externals = bash deps = pip>=20.0.1 - pytest commands = pip install pyre2 --force-reinstall --prefer-binary -f dist/ python -m unittest discover -f -s . - #pytest --doctest-glob="*.txt" + +[testenv:clean] +skip_install = true +allowlist_externals = + bash + +deps = + pip>=21.1 + +commands = + bash -c 'rm -rf *.egg-info re2*.so .coverage.* tests/__pycache__ dist/ build/' From fb2d2dff9bfc153997c9f8a8a92722290345a598 Mon Sep 17 00:00:00 2001 From: Steve Arnold Date: Mon, 8 Apr 2024 20:13:48 -0700 Subject: [PATCH 02/17] fix: dev: remove failing subscript test in single match group * cleanup asserts and add groups() test Signed-off-by: Steve Arnold --- tests/test_re.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_re.py b/tests/test_re.py index 57992778..a5644148 100644 --- a/tests/test_re.py +++ b/tests/test_re.py @@ -254,11 +254,10 @@ def test_re_match(self): # A single group m = re.match('(a)', 'a') self.assertEqual(m.group(0), 'a') - self.assertEqual(m.group(0), 'a') self.assertEqual(m.group(1), 'a') + self.assertEqual(m.group(0, 0), ('a', 'a')) self.assertEqual(m.group(1, 1), ('a', 'a')) - self.assertEqual(m[0], 'a') - self.assertEqual(m[1], 'a') + self.assertEqual(m.groups(), ('a',)) pat = re.compile('(?:(?Pa)|(?Pb))(?Pc)?') self.assertEqual(pat.match('a').group(1, 2, 3), ('a', None, None)) From f3d03acdd41eb4f2f9751898ef4d0ad7c52d98eb Mon Sep 17 00:00:00 2001 From: Steve Arnold Date: Mon, 8 Apr 2024 20:18:57 -0700 Subject: [PATCH 03/17] chg: dev: update packaging files, add setuptools_scm support * refactor setup.py after pybind11 upstream changes Signed-off-by: Steve Arnold --- pyproject.toml | 5 ++- setup.cfg | 8 ++--- setup.py | 85 ++++++++++++++++++++++++++++---------------------- tox.ini | 49 ++++++++++++++++++++++------- 4 files changed, 93 insertions(+), 54 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d2d6a428..27348999 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,17 @@ [build-system] requires = [ "setuptools>=42", + "setuptools_scm[toml]>=6.2", "Cython>=0.20", "pybind11>=2.6.0", "ninja; sys_platform != 'Windows'", - "cmake>=3.15", + "cmake>=3.18", ] build-backend = "setuptools.build_meta" +[tool.setuptools_scm] + [tool.pytest.ini_options] minversion = "6.0" testpaths = [ diff --git a/setup.cfg b/setup.cfg index 5223702a..36d2928f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,6 @@ [metadata] name = pyre2 +version = attr: setuptools_scm.get_version author = Andreas van Cranenburgh author_email = andreas@unstable.nl maintainer = Steve Arnold @@ -20,7 +21,8 @@ classifiers = [options] python_requires = >=3.6 -zip_safe = False +setup_requires = + setuptools_scm[toml] [options.extras_require] test = @@ -33,7 +35,3 @@ perf = # these error codes interfere with Black ignore = E203, E231, E501, W503, B950 select = C,E,F,W,B,B9 - -[egg_info] -tag_build = -tag_date = 0 diff --git a/setup.py b/setup.py index e76c5e3d..588ff332 100755 --- a/setup.py +++ b/setup.py @@ -2,16 +2,14 @@ # import os -import sys +import re import subprocess +import sys +from pathlib import Path from setuptools import setup, Extension from setuptools.command.build_ext import build_ext - -# update the release version both here and in conda.recipe/meta.yaml -__version__ = '0.3.6' - # Convert distutils Windows platform specifiers to CMake -A arguments PLAT_TO_CMAKE = { "win32": "Win32", @@ -20,42 +18,39 @@ "win-arm64": "ARM64", } + # A CMakeExtension needs a sourcedir instead of a file list. class CMakeExtension(Extension): - def __init__(self, name, sourcedir=""): - Extension.__init__(self, name, sources=[], libraries=['re2']) - self.sourcedir = os.path.abspath(sourcedir) + def __init__(self, name: str, sourcedir: str = "") -> None: + super().__init__(name, sources=[], libraries=['re2']) + self.sourcedir = os.fspath(Path(sourcedir).resolve()) class CMakeBuild(build_ext): + def build_extension(self, ext: CMakeExtension) -> None: + # Must be in this form due to bug in .resolve() only fixed in Python 3.10+ + ext_fullpath = Path.cwd() / self.get_ext_fullpath(ext.name) + extdir = ext_fullpath.parent.resolve() - def build_extension(self, ext): - extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + # Using this requires trailing slash for auto-detection & inclusion of + # auxiliary "native" libs - # required for auto-detection of auxiliary "native" libs - if not extdir.endswith(os.path.sep): - extdir += os.path.sep - - # Set a sensible default build type for packaging - if "CMAKE_BUILD_OVERRIDE" not in os.environ: - cfg = "Debug" if self.debug else "RelWithDebInfo" - else: - cfg = os.environ.get("CMAKE_BUILD_OVERRIDE", "") + debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug + cfg = "Debug" if debug else "Release" # CMake lets you override the generator - we need to check this. # Can be set with Conda-Build, for example. cmake_generator = os.environ.get("CMAKE_GENERATOR", "") # Set Python_EXECUTABLE instead if you use PYBIND11_FINDPYTHON - # SCM_VERSION_INFO shows you how to pass a value into the C++ code + # EXAMPLE_VERSION_INFO shows you how to pass a value into the C++ code # from Python. cmake_args = [ - "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}".format(extdir), - "-DPython_EXECUTABLE={}".format(sys.executable), - "-DSCM_VERSION_INFO={}".format(__version__), - "-DCMAKE_BUILD_TYPE={}".format(cfg), # not used on MSVC, but no harm + f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}{os.sep}", + f"-DPython_EXECUTABLE={sys.executable}", + f"-DCMAKE_BUILD_TYPE={cfg}", # not used on MSVC, but no harm ] - build_args = ["--verbose"] + build_args = [] # CMake also lets you provide a toolchain file. # Can be set in CI build environments for example. @@ -63,17 +58,27 @@ def build_extension(self, ext): if cmake_toolchain_file: cmake_args += ["-DCMAKE_TOOLCHAIN_FILE={}".format(cmake_toolchain_file)] + cmake_args += [f"-DSCM_VERSION_INFO={self.distribution.get_version()}"] + if self.compiler.compiler_type != "msvc": # Using Ninja-build since it a) is available as a wheel and b) # multithreads automatically. MSVC would require all variables be # exported for Ninja to pick it up, which is a little tricky to do. # Users can override the generator with CMAKE_GENERATOR in CMake # 3.15+. - if not cmake_generator: - cmake_args += ["-GNinja"] + if not cmake_generator or cmake_generator == "Ninja": + try: + import ninja + + ninja_executable_path = Path(ninja.BIN_DIR) / "ninja" + cmake_args += [ + "-GNinja", + f"-DCMAKE_MAKE_PROGRAM:FILEPATH={ninja_executable_path}", + ] + except ImportError: + pass else: - # Single config generators are handled "normally" single_config = any(x in cmake_generator for x in {"NMake", "Ninja"}) @@ -89,10 +94,16 @@ def build_extension(self, ext): # Multi-config generators have a different way to specify configs if not single_config: cmake_args += [ - "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}".format(cfg.upper(), extdir) + f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}" ] build_args += ["--config", cfg] + if sys.platform.startswith("darwin"): + # Cross-compile support for macOS - respect ARCHFLAGS if set + archs = re.findall(r"-arch (\S+)", os.environ.get("ARCHFLAGS", "")) + if archs: + cmake_args += ["-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))] + # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level # across all generators. if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ: @@ -100,21 +111,21 @@ def build_extension(self, ext): # using -j in the build_ext call, not supported by pip or PyPA-build. if hasattr(self, "parallel") and self.parallel: # CMake 3.12+ only. - build_args += ["-j{}".format(self.parallel)] + build_args += [f"-j{self.parallel}"] - if not os.path.exists(self.build_temp): - os.makedirs(self.build_temp) + build_temp = Path(self.build_temp) / ext.name + if not build_temp.exists(): + build_temp.mkdir(parents=True) - subprocess.check_call( - ["cmake", ext.sourcedir] + cmake_args, cwd=self.build_temp + subprocess.run( + ["cmake", ext.sourcedir, *cmake_args], cwd=build_temp, check=True ) - subprocess.check_call( - ["cmake", "--build", "."] + build_args, cwd=self.build_temp + subprocess.run( + ["cmake", "--build", ".", *build_args], cwd=build_temp, check=True ) setup( - version=__version__, ext_modules=[CMakeExtension('re2')], cmdclass={'build_ext': CMakeBuild}, zip_safe=False, diff --git a/tox.ini b/tox.ini index 32143d92..616be0fb 100644 --- a/tox.ini +++ b/tox.ini @@ -19,8 +19,28 @@ PLATFORM = macos-latest: macos windows-latest: windows +[base] +deps = + pip>=21.1 + setuptools_scm[toml] + +[build] +deps = + pip>=21.1 + build + twine + [testenv] +skip_install = true + +setenv = + PYTHONPATH = {toxinidir} + passenv = + HOME + USERNAME + USER + XDG_* CI CC CXX @@ -29,9 +49,12 @@ passenv = CMAKE_GENERATOR PIP_DOWNLOAD_CACHE +allowlist_externals = + bash + deps = - pip>=20.0.1 - -e .[test] + {[base]deps} + .[test] commands = pytest -v . @@ -40,6 +63,10 @@ commands = skip_install = true passenv = + HOME + USERNAME + USER + XDG_* CI CC CXX @@ -49,15 +76,17 @@ passenv = PIP_DOWNLOAD_CACHE setenv = - PYTHONPATH=. + PYTHONPATH = {toxinidir} deps = - pip>=20.0.1 - cython>=0.20 - pytest + {[base]deps} + cython + -r requirements-dev.txt + -e . commands = - python setup.py build_ext --inplace + # this is deprecated => _DeprecatedInstaller warning from setuptools + #python setup.py build_ext --inplace # use --capture=no to see all the doctest output python -m pytest -v --ignore=tests/test_re.py --doctest-glob=*.txt . python -m pytest -v tests/test_re.py @@ -73,7 +102,7 @@ passenv = PIP_DOWNLOAD_CACHE deps = - pip>=20.0.1 + {[base]deps} .[perf] commands = @@ -93,9 +122,7 @@ passenv = allowlist_externals = bash deps = - pip>=20.0.1 - build - twine + {[build]deps} commands = python -m build . From b98576414378cc4f31e838edb18fad3203df0ae6 Mon Sep 17 00:00:00 2001 From: Steve Arnold Date: Mon, 8 Apr 2024 21:20:16 -0700 Subject: [PATCH 04/17] chg: dev: bump cibw version, update workflows and min py version Signed-off-by: Steve Arnold --- .github/workflows/ci.yml | 26 ++++++----- .github/workflows/conda.yml | 87 +++++++++++++++++++++---------------- .github/workflows/main.yml | 22 +++++----- environment.devenv.yml | 14 ++++++ requirements-cibw.txt | 2 +- setup.cfg | 4 +- setup.py | 2 +- 7 files changed, 94 insertions(+), 63 deletions(-) create mode 100644 environment.devenv.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a1e8245..d1422423 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,9 +3,13 @@ name: Smoke on: workflow_dispatch: pull_request: + push: + branches: + - master jobs: python_wheels: + name: Python wheels for ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} defaults: run: @@ -13,12 +17,13 @@ jobs: env: OS: ${{ matrix.os }} PYTHON: ${{ matrix.python-version }} + PYTHONIOENCODING: utf-8 PIP_DOWNLOAD_CACHE: ${{ github.workspace }}/../.pip_download_cache strategy: - fail-fast: true + fail-fast: false matrix: os: [ubuntu-22.04] - python-version: [3.8, 3.9, '3.10', '3.11', '3.12'] + python-version: [3.8, 3.9, '3.10', '3.11'] steps: - uses: actions/checkout@v4 @@ -35,21 +40,20 @@ jobs: pip install tox tox-gh-actions - name: Install Ubuntu build deps - if: runner.os == 'Linux' run: | sudo apt-get -qq update - sudo apt-get install -y software-properties-common - sudo add-apt-repository -y -s ppa:nerdboy/embedded - sudo apt-get install -y pybind11-dev libre2-dev ninja-build + sudo apt-get install -y libre2-dev - - name: Test in place + - name: Test using pip install run: | - tox -e dev + tox + env: + PLATFORM: ${{ matrix.os }} - - name: Build dist pkgs + - name: Build sdist and wheel pkgs run: | - tox -e deploy + tox -e build - - name: Check wheel + - name: Test using built wheel run: | tox -e check diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 2a298903..af361830 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -1,55 +1,68 @@ -name: Conda +name: CondaDev on: workflow_dispatch: - pull_request: push: branches: - master + pull_request: jobs: build: + name: Test on Python ${{ matrix.python-version }} and ${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - platform: [ubuntu-latest, windows-2019, macos-latest] - python-version: [3.8, '3.10'] - - runs-on: ${{ matrix.platform }} - - # The setup-miniconda action needs this to activate miniconda - defaults: - run: - shell: "bash -l {0}" + os: ['macos-11', 'ubuntu-22.04'] + python-version: ['3.8', '3.11'] + env: + OS: ${{ matrix.os }} + PYTHON: ${{ matrix.python-version }} + PYTHONIOENCODING: utf-8 steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Cache conda - uses: actions/cache@v4 - with: - path: ~/conda_pkgs_dir - key: ${{matrix.os}}-conda-pkgs-${{hashFiles('**/conda.recipe/meta.yaml')}} - - - name: Get conda - uses: conda-incubator/setup-miniconda@v3 - with: - python-version: ${{ matrix.python-version }} - channels: conda-forge - channel-priority: strict - use-only-tar-bz2: true - auto-activate-base: true + - uses: actions/checkout@v4 + with: + fetch-depth: 0 - - name: Prepare - run: conda install conda-build conda-verify + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + channels: conda-forge + channel-priority: strict + use-only-tar-bz2: true - - name: Build - run: conda build conda.recipe + - name: Cache conda packages + id: cache + uses: actions/cache@v4 + env: + # Increase this value to reset cache and rebuild the env during the PR + CACHE_NUMBER: 3 + with: + path: /home/runner/conda_pkgs_dir + key: + ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-${{hashFiles('environment.devenv.yml') }} + restore-keys: | + ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}- + ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - name: Install - run: conda install -c ${CONDA_PREFIX}/conda-bld/ pyre2 + - name: Configure condadev environment + shell: bash -l {0} + env: + PY_VER: ${{ matrix.python-version }} + run: | + conda config --set always_yes yes --set changeps1 no + conda config --add channels conda-forge + conda install conda-devenv + conda devenv - - name: Test - run: python -m unittest discover -f -s tests/ + - name: Build and test + shell: bash -l {0} + env: + PY_VER: ${{ matrix.python-version }} + run: | + source activate pyre2 + python -m pip install .[test] -vv + python -m pytest -v . diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 702aa5c8..21855fbc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,17 +3,17 @@ name: Build on: workflow_dispatch: pull_request: - push: - branches: - - master + #push: + #branches: + #- master jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} for Python + cibw_wheels: runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: - os: [ubuntu-22.04, macos-latest, windows-latest] + os: [ubuntu-22.04, macos-11, windows-2019] steps: - uses: actions/checkout@v4 @@ -71,19 +71,19 @@ jobs: - uses: actions/setup-python@v5 name: Install Python with: - python-version: '3.7' + python-version: '3.8' - name: Build sdist run: | - pip install pep517 - python -m pep517.build -s . + pip install build + python -m build -s . - uses: actions/upload-artifact@v4 with: path: dist/*.tar.gz check_artifacts: - needs: [build_sdist, build_wheels] + needs: [build_sdist, cibw_wheels] defaults: run: shell: bash @@ -95,4 +95,4 @@ jobs: # note wheels should be in subdirectory - name: Check number of downloaded artifacts - run: ls -R + run: ls -l artifact diff --git a/environment.devenv.yml b/environment.devenv.yml new file mode 100644 index 00000000..dcdee670 --- /dev/null +++ b/environment.devenv.yml @@ -0,0 +1,14 @@ +name: pyre2 + +dependencies: + - python ==3.11 + - cmake >=3.18 + - ninja + - cython + - cxx-compiler + - pybind11 + - pip + - re2 + - pytest + - regex + - six diff --git a/requirements-cibw.txt b/requirements-cibw.txt index 932364dd..3d34eb3b 100644 --- a/requirements-cibw.txt +++ b/requirements-cibw.txt @@ -1 +1 @@ -cibuildwheel==1.7.4 +cibuildwheel==2.11.3 diff --git a/setup.cfg b/setup.cfg index 36d2928f..f8c4c6bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,12 +14,12 @@ license_files = LICENSE classifiers = License :: OSI Approved :: BSD License Programming Language :: Cython - Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.8 Intended Audience :: Developers Topic :: Software Development :: Libraries :: Python Modules [options] -python_requires = >=3.6 +python_requires = >=3.8 setup_requires = setuptools_scm[toml] diff --git a/setup.py b/setup.py index 588ff332..814368ae 100755 --- a/setup.py +++ b/setup.py @@ -128,5 +128,5 @@ def build_extension(self, ext: CMakeExtension) -> None: setup( ext_modules=[CMakeExtension('re2')], cmdclass={'build_ext': CMakeBuild}, - zip_safe=False, + package_dir={'': 'src'}, ) From 7fea650c89fe60f77522ee70eb079017cef21a0f Mon Sep 17 00:00:00 2001 From: Steve Arnold Date: Tue, 9 Apr 2024 18:49:21 -0700 Subject: [PATCH 05/17] chg: dev: add/update cfgs, modernize cibw build workflow Signed-off-by: Steve Arnold --- .github/workflows/ci.yml | 6 +----- .github/workflows/main.yml | 42 ++++++++++++++++++-------------------- .pep8speaks.yml | 15 ++++++++++++++ pyproject.toml | 2 +- setup.cfg | 22 ++++++++++++++++++-- 5 files changed, 57 insertions(+), 30 deletions(-) create mode 100644 .pep8speaks.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1422423..95c25204 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,8 +52,4 @@ jobs: - name: Build sdist and wheel pkgs run: | - tox -e build - - - name: Test using built wheel - run: | - tox -e check + tox -e build,check diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 21855fbc..0bdccb53 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,38 +25,36 @@ jobs: with: python-version: '3.8' - - name: Prepare compiler environment for Windows - if: runner.os == 'Windows' - uses: ilammy/msvc-dev-cmd@v1 + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 with: - arch: amd64 - - - name: Install cibuildwheel - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements-cibw.txt + platforms: all - name: Build wheels + uses: pypa/cibuildwheel@v2.17 env: - CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux2010_x86_64:latest - CIBW_MANYLINUX_I686_IMAGE: quay.io/pypa/manylinux2010_i686:latest - CIBW_BUILD: cp38-* cp39-* cp310-* cp311-* - CIBW_SKIP: "*-win32" + # configure cibuildwheel to build native archs ('auto'), and some + # emulated ones, plus cross-compile on macos + CIBW_ARCHS_LINUX: auto aarch64 + CIBW_ARCHS_MACOS: auto arm64 + CIBW_TEST_SKIP: "*_arm64 *universal2:arm64 *linux_i686" + CIBW_ARCHS_WINDOWS: auto64 + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 + CIBW_MANYLINUX_I686_IMAGE: manylinux2010 + CIBW_BUILD: cp37-* cp38-* cp39-* cp310-* cp311-* + CIBW_SKIP: "*musllinux* cp311-*i686" CIBW_BEFORE_ALL_LINUX: > - yum -y -q --enablerepo=extras install epel-release - && yum install -y re2-devel ninja-build - CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel show {wheel} && auditwheel repair -w {dest_dir} {wheel}" + yum -y install epel-release && yum install -y re2-devel ninja-build CIBW_BEFORE_ALL_MACOS: > - brew install re2 pybind11 ninja - CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=10.09 - CIBW_REPAIR_WHEEL_COMMAND_MACOS: "pip uninstall -y delocate && pip install git+https://github.com/Chia-Network/delocate.git && delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} -v {wheel}" + brew install re2 pybind11 + #CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=10.14 CIBW_BEFORE_ALL_WINDOWS: > vcpkg install re2:x64-windows && vcpkg integrate install CIBW_ENVIRONMENT_WINDOWS: 'CMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake' - CIBW_TEST_COMMAND: python -c "import re2" - run: | - python -m cibuildwheel --output-dir wheelhouse + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: pytest -v {package}/tests/test_re.py - uses: actions/upload-artifact@v4 with: diff --git a/.pep8speaks.yml b/.pep8speaks.yml new file mode 100644 index 00000000..4f120b0d --- /dev/null +++ b/.pep8speaks.yml @@ -0,0 +1,15 @@ +scanner: + diff_only: True # If False, the entire file touched by the Pull Request is scanned for errors. If True, only the diff is scanned. + linter: flake8 # Other option is pycodestyle + +no_blank_comment: False # If True, no comment is made on PR without any errors. +descending_issues_order: True # If True, PEP 8 issues in message will be displayed in descending order of line numbers in the file + +pycodestyle: # Same as scanner.linter value. Other option is flake8 + max-line-length: 90 # Default is 79 in PEP 8 + +flake8: + max-line-length: 90 # Default is 79 in PEP 8 + exclude: + - tests + - docs diff --git a/pyproject.toml b/pyproject.toml index 27348999..02531078 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "setuptools>=42", "setuptools_scm[toml]>=6.2", "Cython>=0.20", - "pybind11>=2.6.0", + "pybind11>=2.11.1", "ninja; sys_platform != 'Windows'", "cmake>=3.18", ] diff --git a/setup.cfg b/setup.cfg index f8c4c6bc..f4bf3678 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,7 +31,25 @@ test = perf = regex +[aliases] +test=pytest + +[check] +metadata = true +restructuredtext = true +strict = false + +[check-manifest] +ignore = + .gitattributes + .gitchangelog.rc + .gitignore + conda/** + [flake8] # these error codes interfere with Black -ignore = E203, E231, E501, W503, B950 -select = C,E,F,W,B,B9 +#ignore = E203, E231, E501, W503, B950, +ignore = E225,E226,E227,E402,E703,E999 +max-line-length = 90 +filename = *.pyx, *.px* +exclude = .git, .eggs, *.egg, .tox, build From 9e6daec3b312195c2d2ae19a4d1279f3ab382f4a Mon Sep 17 00:00:00 2001 From: Steve Arnold Date: Tue, 9 Apr 2024 21:43:56 -0700 Subject: [PATCH 06/17] chg: dev: update cmake and workflow files, delete unused bits * cleanup ci workflow, remove crufty makefile with deprecated setup.py commands * remove the package_dir bit from setup.py Signed-off-by: Steve Arnold --- .github/workflows/ci.yml | 2 +- .github/workflows/conda.yml | 8 ++++---- CMakeLists.txt | 2 +- MANIFEST.in | 8 -------- Makefile | 30 ------------------------------ cmake/modules/FindCython.cmake | 10 +++++----- setup.py | 1 - src/CMakeLists.txt | 2 +- tox.ini | 2 +- 9 files changed, 13 insertions(+), 52 deletions(-) delete mode 100644 MANIFEST.in delete mode 100644 Makefile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95c25204..5a21d080 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-22.04] - python-version: [3.8, 3.9, '3.10', '3.11'] + python-version: [3.9, '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index af361830..6312fc24 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -2,10 +2,10 @@ name: CondaDev on: workflow_dispatch: - push: - branches: - - master - pull_request: + #push: + #branches: + #- master + #pull_request: jobs: build: diff --git a/CMakeLists.txt b/CMakeLists.txt index d12bc608..87627f90 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,7 +21,7 @@ endif() include(GNUInstallDirs) # get rid of FindPython old warnings, refactor FindCython module -set(CMP0148 NEW) +#set(CMP0148 NEW) find_package(pybind11 CONFIG) diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 305a4445..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,8 +0,0 @@ -global-include CMakeLists.txt *.cmake -include AUTHORS README.rst HISTORY CHANGELOG.rst LICENSE -graft src -graft tests -global-exclude *.py[cod] __pycache__ -recursive-exclude .tox * -recursive-exclude .github * -recursive-exclude vcpkg * diff --git a/Makefile b/Makefile deleted file mode 100644 index 3785bdce..00000000 --- a/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -install: - python3 setup.py install --user - -test: install - pytest - -install2: - python2 setup.py install --user - -test2: install2 - python2 -m pytest - -clean: - rm -rf build pyre2.egg-info &>/dev/null - rm -f *.so src/*.so src/re2.cpp src/*.html &>/dev/null - -distclean: clean - rm -rf .tox/ dist/ .pytest_cache/ - -valgrind: - python3-dbg setup.py install --user && \ - (cd tests && valgrind --tool=memcheck --suppressions=../valgrind-python.supp \ - --leak-check=full --show-leak-kinds=definite \ - python3-dbg test_re.py) - -valgrind2: - python2-dbg setup.py install --user && \ - (cd tests && valgrind --tool=memcheck --suppressions=../valgrind-python.supp \ - --leak-check=full --show-leak-kinds=definite \ - python2-dbg re2_test.py) diff --git a/cmake/modules/FindCython.cmake b/cmake/modules/FindCython.cmake index c53e2b83..83ac106e 100644 --- a/cmake/modules/FindCython.cmake +++ b/cmake/modules/FindCython.cmake @@ -2,7 +2,7 @@ # # This code sets the following variables: # -# CYTHON_EXECUTABLE +# Cython_EXECUTABLE # # See also UseCython.cmake @@ -27,18 +27,18 @@ find_package(Python) if( Python_FOUND ) get_filename_component( _python_path ${Python_EXECUTABLE} PATH ) - find_program( CYTHON_EXECUTABLE + find_program( Cython_EXECUTABLE NAMES cython cython.bat cython3 HINTS ${_python_path} ) else() - find_program( CYTHON_EXECUTABLE + find_program( Cython_EXECUTABLE NAMES cython cython.bat cython3 ) endif() include( FindPackageHandleStandardArgs ) -FIND_PACKAGE_HANDLE_STANDARD_ARGS( Cython REQUIRED_VARS CYTHON_EXECUTABLE ) +FIND_PACKAGE_HANDLE_STANDARD_ARGS( Cython REQUIRED_VARS Cython_EXECUTABLE ) -mark_as_advanced( CYTHON_EXECUTABLE ) +mark_as_advanced( Cython_EXECUTABLE ) diff --git a/setup.py b/setup.py index 814368ae..73c87fc9 100755 --- a/setup.py +++ b/setup.py @@ -128,5 +128,4 @@ def build_extension(self, ext: CMakeExtension) -> None: setup( ext_modules=[CMakeExtension('re2')], cmdclass={'build_ext': CMakeBuild}, - package_dir={'': 'src'}, ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8e0372d2..d28e325c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,7 +12,7 @@ file(GLOB cy_srcs *.pyx *.pxi *.h) # .pyx -> .cpp add_custom_command(OUTPUT ${cython_output} - COMMAND ${CYTHON_EXECUTABLE} + COMMAND ${Cython_EXECUTABLE} -a -3 --fast-fail --cplus -I ${re2_include_dir} diff --git a/tox.ini b/tox.ini index 616be0fb..67eb8056 100644 --- a/tox.ini +++ b/tox.ini @@ -151,4 +151,4 @@ deps = pip>=21.1 commands = - bash -c 'rm -rf *.egg-info re2*.so .coverage.* tests/__pycache__ dist/ build/' + bash -c 'rm -rf src/*.egg-info re2*.so src/re2*.so src/re2.cpp *coverage.* tests/__pycache__ dist/ build/' From 7a329427c711312ecf2bff7b8b091cb250cafe80 Mon Sep 17 00:00:00 2001 From: Steve Arnold Date: Fri, 12 Apr 2024 19:44:49 -0700 Subject: [PATCH 07/17] chg: dev: cleanup metadata and test imports, disable platform whl tests * check if find_package py3 works across all CI runners Signed-off-by: Steve Arnold --- .github/workflows/main.yml | 6 +++--- cmake/modules/FindCython.cmake | 2 +- setup.cfg | 2 +- tests/test_re.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0bdccb53..374e5acb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, macos-11, windows-2019] + os: [ubuntu-22.04, macos-11, windows-latest] steps: - uses: actions/checkout@v4 @@ -53,8 +53,8 @@ jobs: vcpkg install re2:x64-windows && vcpkg integrate install CIBW_ENVIRONMENT_WINDOWS: 'CMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake' - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: pytest -v {package}/tests/test_re.py + CIBW_TEST_REQUIRES: "" + CIBW_TEST_COMMAND: "" - uses: actions/upload-artifact@v4 with: diff --git a/cmake/modules/FindCython.cmake b/cmake/modules/FindCython.cmake index 83ac106e..fde6edde 100644 --- a/cmake/modules/FindCython.cmake +++ b/cmake/modules/FindCython.cmake @@ -24,7 +24,7 @@ # Use the Cython executable that lives next to the Python executable # if it is a local installation. -find_package(Python) +find_package(Python3) if( Python_FOUND ) get_filename_component( _python_path ${Python_EXECUTABLE} PATH ) find_program( Cython_EXECUTABLE diff --git a/setup.cfg b/setup.cfg index f4bf3678..7bf19496 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ author = Andreas van Cranenburgh author_email = andreas@unstable.nl maintainer = Steve Arnold maintainer_email = nerdboy@gentoo.org -description = Python wrapper for Google\'s RE2 using Cython +description = Python wrapper for Google RE2 library using Cython long_description = file: README.rst long_description_content_type = text/x-rst; charset=UTF-8 url = https://github.com/andreasvc/pyre2 diff --git a/tests/test_re.py b/tests/test_re.py index a5644148..567e54fa 100644 --- a/tests/test_re.py +++ b/tests/test_re.py @@ -689,9 +689,9 @@ def test_dealloc(self): def test_re_suite(): try: - from tests.re_utils import benchmarks, tests, SUCCEED, FAIL, SYNTAX_ERROR + from tests.re_utils import tests, SUCCEED, FAIL, SYNTAX_ERROR except ImportError: - from re_utils import benchmarks, tests, SUCCEED, FAIL, SYNTAX_ERROR + from re_utils import tests, SUCCEED, FAIL, SYNTAX_ERROR if verbose: print('\nRunning test_re_suite ...') From 816705b6bf49ea653d6ad6bddde4767981bc50b7 Mon Sep 17 00:00:00 2001 From: Steve Arnold Date: Fri, 12 Apr 2024 20:55:52 -0700 Subject: [PATCH 08/17] chg: dev: enable findpython policy, use matrix uploads * no epel pkgs for linux aarch64, enable PYBIND11_FINDPYTHON * set macos deployment target to 10.9 Signed-off-by: Steve Arnold --- .github/workflows/main.yml | 7 ++++--- CMakeLists.txt | 3 ++- cmake/modules/FindCython.cmake | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 374e5acb..56deffd4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, macos-11, windows-latest] + os: [ubuntu-20.04, macos-11, windows-latest] steps: - uses: actions/checkout@v4 @@ -36,7 +36,7 @@ jobs: env: # configure cibuildwheel to build native archs ('auto'), and some # emulated ones, plus cross-compile on macos - CIBW_ARCHS_LINUX: auto aarch64 + CIBW_ARCHS_LINUX: auto CIBW_ARCHS_MACOS: auto arm64 CIBW_TEST_SKIP: "*_arm64 *universal2:arm64 *linux_i686" CIBW_ARCHS_WINDOWS: auto64 @@ -48,7 +48,7 @@ jobs: yum -y install epel-release && yum install -y re2-devel ninja-build CIBW_BEFORE_ALL_MACOS: > brew install re2 pybind11 - #CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=10.14 + CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=10.9 CIBW_BEFORE_ALL_WINDOWS: > vcpkg install re2:x64-windows && vcpkg integrate install @@ -58,6 +58,7 @@ jobs: - uses: actions/upload-artifact@v4 with: + name: wheels-${{ matrix.os }} path: ./wheelhouse/*.whl build_sdist: diff --git a/CMakeLists.txt b/CMakeLists.txt index 87627f90..9b805e97 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,8 +21,9 @@ endif() include(GNUInstallDirs) # get rid of FindPython old warnings, refactor FindCython module -#set(CMP0148 NEW) +set(CMP0148 NEW) +set(PYBIND11_FINDPYTHON ON) find_package(pybind11 CONFIG) if(pybind11_FOUND) diff --git a/cmake/modules/FindCython.cmake b/cmake/modules/FindCython.cmake index fde6edde..83ac106e 100644 --- a/cmake/modules/FindCython.cmake +++ b/cmake/modules/FindCython.cmake @@ -24,7 +24,7 @@ # Use the Cython executable that lives next to the Python executable # if it is a local installation. -find_package(Python3) +find_package(Python) if( Python_FOUND ) get_filename_component( _python_path ${Python_EXECUTABLE} PATH ) find_program( Cython_EXECUTABLE From 621888a399f87dd0eb9a2abde82207d50f36e9ba Mon Sep 17 00:00:00 2001 From: Steve Arnold Date: Sat, 13 Apr 2024 15:18:40 -0700 Subject: [PATCH 09/17] chg: dev: try pkgconf on windows CI runner Signed-off-by: Steve Arnold --- .github/workflows/main.yml | 2 +- src/CMakeLists.txt | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 56deffd4..f25c147f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,7 +50,7 @@ jobs: brew install re2 pybind11 CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=10.9 CIBW_BEFORE_ALL_WINDOWS: > - vcpkg install re2:x64-windows + vcpkg install pkgconf re2:x64-windows && vcpkg integrate install CIBW_ENVIRONMENT_WINDOWS: 'CMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake' CIBW_TEST_REQUIRES: "" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d28e325c..18b42fc8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -29,12 +29,14 @@ target_compile_definitions( # here we get to jump through some hoops to find libre2 on the manylinux # docker CI images, etc -find_package(re2 CONFIG NAMES re2) +if(NOT MSVC) + find_package(re2 CONFIG NAMES re2) +endif() if(re2_FOUND) message(STATUS "System re2 found") target_link_libraries(${cython_module} PRIVATE re2::re2) -elseif(NOT MSVC) +else() message(STATUS "Trying PkgConfig") find_package(PkgConfig REQUIRED) pkg_check_modules(RE2 IMPORTED_TARGET re2) From df921a8681dace056dae0c42c132a9899333534f Mon Sep 17 00:00:00 2001 From: Stephen L Arnold Date: Thu, 5 Sep 2024 19:57:06 -0700 Subject: [PATCH 10/17] fix: dev: bump CMake to c++17 with extensions to build against re2-0.2024.07.02 Signed-off-by: Stephen L Arnold --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b805e97..eea96df3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,9 +4,9 @@ project(re2 LANGUAGES CXX C) option(PY_DEBUG "Set if python being linked is a Py_DEBUG build" OFF) -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_CXX_EXTENSIONS ON) if(CMAKE_CXX_COMPILER_ID STREQUAL Clang) set(CLANG_DEFAULT_CXX_STDLIB libc++) From c3ebb3056465cddf3da8e8725515f6c8eb431836 Mon Sep 17 00:00:00 2001 From: Stephen Arnold Date: Thu, 5 Sep 2024 20:54:39 -0700 Subject: [PATCH 11/17] fix: dev: bump ubuntu and mac workflow runners * revert to macos-13 with the same version as target * In Theory this should get us full c++17 Signed-off-by: Stephen Arnold --- .github/workflows/main.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f25c147f..53d4ff36 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, macos-11, windows-latest] + os: [ubuntu-22.04, windows-latest, macos-13] steps: - uses: actions/checkout@v4 @@ -23,7 +23,7 @@ jobs: - uses: actions/setup-python@v5 name: Install Python with: - python-version: '3.8' + python-version: '3.9' - name: Set up QEMU if: runner.os == 'Linux' @@ -32,7 +32,7 @@ jobs: platforms: all - name: Build wheels - uses: pypa/cibuildwheel@v2.17 + uses: pypa/cibuildwheel@v2.20 env: # configure cibuildwheel to build native archs ('auto'), and some # emulated ones, plus cross-compile on macos @@ -42,13 +42,14 @@ jobs: CIBW_ARCHS_WINDOWS: auto64 CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 CIBW_MANYLINUX_I686_IMAGE: manylinux2010 - CIBW_BUILD: cp37-* cp38-* cp39-* cp310-* cp311-* - CIBW_SKIP: "*musllinux* cp311-*i686" + CIBW_BUILD: cp38-* cp39-* cp310-* cp311-* cp312-* + CIBW_SKIP: "*musllinux* *i686" CIBW_BEFORE_ALL_LINUX: > - yum -y install epel-release && yum install -y re2-devel ninja-build + yum -y update && yum -y install epel-release && yum install -y re2-devel ninja-build CIBW_BEFORE_ALL_MACOS: > brew install re2 pybind11 - CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=10.9 + # macos target should be 10.14 to get c++17 + CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=13.0 CIBW_BEFORE_ALL_WINDOWS: > vcpkg install pkgconf re2:x64-windows && vcpkg integrate install @@ -70,7 +71,7 @@ jobs: - uses: actions/setup-python@v5 name: Install Python with: - python-version: '3.8' + python-version: '3.9' - name: Build sdist run: | From 8fcfdedc68f4f31563fd4c9d376dbc2c469ed45a Mon Sep 17 00:00:00 2001 From: Stephen Arnold Date: Sat, 7 Sep 2024 12:06:30 -0700 Subject: [PATCH 12/17] chg: swap out flake8 for cython-lint, update setup files, remove pep8 cfg Signed-off-by: Stephen Arnold --- .pep8speaks.yml | 15 --------------- pyproject.toml | 7 ++++++- setup.cfg | 8 -------- setup.py | 22 ++++++++++++++++------ tox.ini | 12 ++++++++++++ 5 files changed, 34 insertions(+), 30 deletions(-) delete mode 100644 .pep8speaks.yml diff --git a/.pep8speaks.yml b/.pep8speaks.yml deleted file mode 100644 index 4f120b0d..00000000 --- a/.pep8speaks.yml +++ /dev/null @@ -1,15 +0,0 @@ -scanner: - diff_only: True # If False, the entire file touched by the Pull Request is scanned for errors. If True, only the diff is scanned. - linter: flake8 # Other option is pycodestyle - -no_blank_comment: False # If True, no comment is made on PR without any errors. -descending_issues_order: True # If True, PEP 8 issues in message will be displayed in descending order of line numbers in the file - -pycodestyle: # Same as scanner.linter value. Other option is flake8 - max-line-length: 90 # Default is 79 in PEP 8 - -flake8: - max-line-length: 90 # Default is 79 in PEP 8 - exclude: - - tests - - docs diff --git a/pyproject.toml b/pyproject.toml index 02531078..2b24a200 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "setuptools>=42", "setuptools_scm[toml]>=6.2", "Cython>=0.20", - "pybind11>=2.11.1", + "pybind11>=2.12", "ninja; sys_platform != 'Windows'", "cmake>=3.18", ] @@ -17,3 +17,8 @@ minversion = "6.0" testpaths = [ "tests", ] + +[tool.cython-lint] +max-line-length = 110 +ignore = ['E225','E226','E227','E402','E703','E999'] +# exclude = 'my_project/excluded_cython_file.pyx' diff --git a/setup.cfg b/setup.cfg index 7bf19496..b086a46a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,11 +45,3 @@ ignore = .gitchangelog.rc .gitignore conda/** - -[flake8] -# these error codes interfere with Black -#ignore = E203, E231, E501, W503, B950, -ignore = E225,E226,E227,E402,E703,E999 -max-line-length = 90 -filename = *.pyx, *.px* -exclude = .git, .eggs, *.egg, .tox, build diff --git a/setup.py b/setup.py index 73c87fc9..b89812b5 100755 --- a/setup.py +++ b/setup.py @@ -32,11 +32,17 @@ def build_extension(self, ext: CMakeExtension) -> None: ext_fullpath = Path.cwd() / self.get_ext_fullpath(ext.name) extdir = ext_fullpath.parent.resolve() - # Using this requires trailing slash for auto-detection & inclusion of - # auxiliary "native" libs + # Set a sensible default build type for packaging + if "CMAKE_BUILD_OVERRIDE" not in os.environ: + cfg = "Debug" if self.debug else "RelWithDebInfo" + else: + cfg = os.environ.get("CMAKE_BUILD_OVERRIDE", "") - debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug - cfg = "Debug" if debug else "Release" + # Set a coverage flag if provided + if "WITH_COVERAGE" not in os.environ: + coverage = "OFF" + else: + coverage = os.environ.get("WITH_COVERAGE", "") # CMake lets you override the generator - we need to check this. # Can be set with Conda-Build, for example. @@ -50,7 +56,11 @@ def build_extension(self, ext: CMakeExtension) -> None: f"-DPython_EXECUTABLE={sys.executable}", f"-DCMAKE_BUILD_TYPE={cfg}", # not used on MSVC, but no harm ] - build_args = [] + build_args = ["--verbose"] + + # Add CMake arguments set as environment variable + if "CMAKE_ARGS" in os.environ: + cmake_args += [item for item in os.environ["CMAKE_ARGS"].split(" ") if item] # CMake also lets you provide a toolchain file. # Can be set in CI build environments for example. @@ -68,7 +78,7 @@ def build_extension(self, ext: CMakeExtension) -> None: # 3.15+. if not cmake_generator or cmake_generator == "Ninja": try: - import ninja + import ninja # noqa: F401 ninja_executable_path = Path(ninja.BIN_DIR) / "ninja" cmake_args += [ diff --git a/tox.ini b/tox.ini index 67eb8056..92528038 100644 --- a/tox.ini +++ b/tox.ini @@ -142,6 +142,18 @@ commands = pip install pyre2 --force-reinstall --prefer-binary -f dist/ python -m unittest discover -f -s . +[testenv:style] +envdir = {toxworkdir}/tests + +passenv = + {[testenv:tests]passenv} + +deps = + pip>=23.1 + cython-lint + +commands = + cython-lint src/ [testenv:clean] skip_install = true allowlist_externals = From e975b7cf9692a715c9103c67bbaa6f7943687801 Mon Sep 17 00:00:00 2001 From: Stephen Arnold Date: Sat, 7 Sep 2024 16:37:52 -0700 Subject: [PATCH 13/17] fix: cleanup tests and fix a raw string test, enable more win32 * split all runners into separate arch via matrix * macos does need macos-14 to get a proper arm64 build Signed-off-by: Stephen Arnold --- .github/workflows/main.yml | 32 +++++++++++++++++++++++--------- tests/re_utils.py | 5 +++-- tests/test_re.py | 17 +++++++++++------ 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 53d4ff36..1c2beded 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,23 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, windows-latest, macos-13] + include: + - os: "ubuntu-22.04" + arch: "x86_64" + - os: "ubuntu-22.04" + arch: "aarch64" + - os: "macos-13" + arch: "x86_64" + macosx_deployment_target: "13.0" + - os: "macos-14" + arch: "arm64" + macosx_deployment_target: "14.0" + - os: "windows-latest" + arch: "auto64" + triplet: "x64-windows" + - os: "windows-latest" + arch: "auto32" + triplet: "x86-windows" steps: - uses: actions/checkout@v4 @@ -36,10 +52,8 @@ jobs: env: # configure cibuildwheel to build native archs ('auto'), and some # emulated ones, plus cross-compile on macos - CIBW_ARCHS_LINUX: auto - CIBW_ARCHS_MACOS: auto arm64 + CIBW_ARCHS: ${{ matrix.arch }} CIBW_TEST_SKIP: "*_arm64 *universal2:arm64 *linux_i686" - CIBW_ARCHS_WINDOWS: auto64 CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 CIBW_MANYLINUX_I686_IMAGE: manylinux2010 CIBW_BUILD: cp38-* cp39-* cp310-* cp311-* cp312-* @@ -48,10 +62,10 @@ jobs: yum -y update && yum -y install epel-release && yum install -y re2-devel ninja-build CIBW_BEFORE_ALL_MACOS: > brew install re2 pybind11 - # macos target should be 10.14 to get c++17 - CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=13.0 + # macos target should be at least 10.13 to get full c++17 + CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=${{ matrix.macosx_deployment_target }} CIBW_BEFORE_ALL_WINDOWS: > - vcpkg install pkgconf re2:x64-windows + vcpkg install pkgconf:${{ matrix.triplet }} re2:${{ matrix.triplet }} && vcpkg integrate install CIBW_ENVIRONMENT_WINDOWS: 'CMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake' CIBW_TEST_REQUIRES: "" @@ -59,7 +73,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.os }} + name: wheels-${{ matrix.os }}-${{ matrix.arch }} path: ./wheelhouse/*.whl build_sdist: @@ -88,7 +102,7 @@ jobs: run: shell: bash name: Check artifacts are correct - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 diff --git a/tests/re_utils.py b/tests/re_utils.py index 348c3ce9..6ddecd9b 100644 --- a/tests/re_utils.py +++ b/tests/re_utils.py @@ -550,8 +550,9 @@ # lookbehind: split by : but not if it is escaped by -. ('(? updated for py311+ + # by removing one backslash from each set of 3 + (r'(? 2: unicode = str unichr = chr From 1465367ab228e1ceae8bb28f624bf2bb602fcffa Mon Sep 17 00:00:00 2001 From: Stephen Arnold Date: Sat, 7 Sep 2024 22:12:24 -0700 Subject: [PATCH 14/17] chg: switch conda workflow to condadev environment Signed-off-by: Stephen Arnold --- .github/workflows/{conda.yml => conda-dev.yml} | 17 +++++++---------- environment.devenv.yml | 17 ++++++++++++++--- 2 files changed, 21 insertions(+), 13 deletions(-) rename .github/workflows/{conda.yml => conda-dev.yml} (87%) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda-dev.yml similarity index 87% rename from .github/workflows/conda.yml rename to .github/workflows/conda-dev.yml index 6312fc24..41e5da74 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda-dev.yml @@ -2,10 +2,10 @@ name: CondaDev on: workflow_dispatch: - #push: - #branches: - #- master - #pull_request: + push: + branches: + - master + pull_request: jobs: build: @@ -14,8 +14,8 @@ jobs: strategy: fail-fast: false matrix: - os: ['macos-11', 'ubuntu-22.04'] - python-version: ['3.8', '3.11'] + os: ['ubuntu-22.04', 'macos-13'] + python-version: ['3.9', '3.10', '3.11', '3.12'] env: OS: ${{ matrix.os }} PYTHON: ${{ matrix.python-version }} @@ -23,15 +23,12 @@ jobs: steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: ${{ matrix.python-version }} channels: conda-forge - channel-priority: strict use-only-tar-bz2: true - name: Cache conda packages @@ -39,7 +36,7 @@ jobs: uses: actions/cache@v4 env: # Increase this value to reset cache and rebuild the env during the PR - CACHE_NUMBER: 3 + CACHE_NUMBER: 0 with: path: /home/runner/conda_pkgs_dir key: diff --git a/environment.devenv.yml b/environment.devenv.yml index dcdee670..1e45b2b6 100644 --- a/environment.devenv.yml +++ b/environment.devenv.yml @@ -1,11 +1,22 @@ name: pyre2 +{% set python_version = os.environ.get("PY_VER", "3.11") %} + +channels: + - conda-forge + dependencies: - - python ==3.11 - - cmake >=3.18 + - cmake>=3.18 - ninja + - ccache + - clangxx_osx-64 # [osx] + - gxx_linux-64 # [linux] + - pybind11-abi + - pybind11-stubgen + - vs2019_win-64 # [win] + - pkgconfig # [win] + - python={{ python_version }} - cython - - cxx-compiler - pybind11 - pip - re2 From db22ef17fe8d73eb9871d2c529325a2d130c232f Mon Sep 17 00:00:00 2001 From: Stephen Arnold Date: Sun, 8 Sep 2024 16:16:57 -0700 Subject: [PATCH 15/17] chg: dev: make sure conda-devenv installs specific pkgs with pip * this is essentially a workaround for non-pypi pkg cruft Signed-off-by: Stephen Arnold --- environment.devenv.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/environment.devenv.yml b/environment.devenv.yml index 1e45b2b6..f961082c 100644 --- a/environment.devenv.yml +++ b/environment.devenv.yml @@ -9,6 +9,7 @@ dependencies: - cmake>=3.18 - ninja - ccache + - re2 - clangxx_osx-64 # [osx] - gxx_linux-64 # [linux] - pybind11-abi @@ -18,8 +19,9 @@ dependencies: - python={{ python_version }} - cython - pybind11 - - pip - - re2 - - pytest - - regex - - six + - pip: + # these two need to be newer than broken runner packages + - pytest + - regex + - urllib3 + - six From f254daa2e95de75122f42746a6ead3600592692d Mon Sep 17 00:00:00 2001 From: Stephen Arnold Date: Sun, 8 Sep 2024 16:51:11 -0700 Subject: [PATCH 16/17] chg: dev: switch conda env to use mamba with newer workflow cmds Signed-off-by: Stephen Arnold --- .github/workflows/conda-dev.yml | 20 ++++++++++++-------- environment.devenv.yml | 12 ++++++------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/conda-dev.yml b/.github/workflows/conda-dev.yml index 41e5da74..12642de4 100644 --- a/.github/workflows/conda-dev.yml +++ b/.github/workflows/conda-dev.yml @@ -28,8 +28,11 @@ jobs: with: auto-update-conda: true python-version: ${{ matrix.python-version }} - channels: conda-forge - use-only-tar-bz2: true + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true + activate-environment: pyre2 + environment-file: environment.devenv.yml - name: Cache conda packages id: cache @@ -46,20 +49,21 @@ jobs: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - name: Configure condadev environment - shell: bash -l {0} + shell: bash -el {0} env: PY_VER: ${{ matrix.python-version }} run: | conda config --set always_yes yes --set changeps1 no - conda config --add channels conda-forge - conda install conda-devenv - conda devenv + conda info + conda list + conda config --show-sources + conda config --show + printenv | sort - name: Build and test - shell: bash -l {0} + shell: bash -el {0} env: PY_VER: ${{ matrix.python-version }} run: | - source activate pyre2 python -m pip install .[test] -vv python -m pytest -v . diff --git a/environment.devenv.yml b/environment.devenv.yml index f961082c..e660e76c 100644 --- a/environment.devenv.yml +++ b/environment.devenv.yml @@ -19,9 +19,9 @@ dependencies: - python={{ python_version }} - cython - pybind11 - - pip: - # these two need to be newer than broken runner packages - - pytest - - regex - - urllib3 - - six + - pip + - pytest + - regex + # these two need to be newer than broken runner packages + - urllib3 + - six From ae58b0b531cf537905915efa8bf94d22dee45fd2 Mon Sep 17 00:00:00 2001 From: Stephen Arnold Date: Sun, 8 Sep 2024 17:16:38 -0700 Subject: [PATCH 17/17] chg: dev: one more conda-devenv refactor based on latest docs * also cleanup the wheel artifact check, download to artifacts/ Signed-off-by: Stephen Arnold --- .github/workflows/conda-dev.yml | 21 ++++++++++----------- .github/workflows/main.yml | 7 ++++--- environment.devenv.yml | 9 ++------- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/.github/workflows/conda-dev.yml b/.github/workflows/conda-dev.yml index 12642de4..3e3cb9d3 100644 --- a/.github/workflows/conda-dev.yml +++ b/.github/workflows/conda-dev.yml @@ -25,14 +25,14 @@ jobs: - uses: actions/checkout@v4 - uses: conda-incubator/setup-miniconda@v3 + env: + PY_VER: ${{ matrix.python-version }} with: - auto-update-conda: true - python-version: ${{ matrix.python-version }} - mamba-version: "*" - channels: conda-forge,defaults - channel-priority: true activate-environment: pyre2 - environment-file: environment.devenv.yml + channels: conda-forge + allow-softlinks: true + channel-priority: flexible + show-channel-urls: true - name: Cache conda packages id: cache @@ -54,16 +54,15 @@ jobs: PY_VER: ${{ matrix.python-version }} run: | conda config --set always_yes yes --set changeps1 no - conda info - conda list - conda config --show-sources - conda config --show - printenv | sort + conda config --add channels conda-forge + conda install conda-devenv + conda devenv - name: Build and test shell: bash -el {0} env: PY_VER: ${{ matrix.python-version }} run: | + source activate pyre2 python -m pip install .[test] -vv python -m pytest -v . diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1c2beded..1ec3f0cf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -104,9 +104,10 @@ jobs: name: Check artifacts are correct runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 + with: + path: artifacts - # note wheels should be in subdirectory + # note wheels should be in subdirectories named - name: Check number of downloaded artifacts - run: ls -l artifact + run: ls -l artifacts/* diff --git a/environment.devenv.yml b/environment.devenv.yml index e660e76c..95fd733c 100644 --- a/environment.devenv.yml +++ b/environment.devenv.yml @@ -1,10 +1,5 @@ name: pyre2 -{% set python_version = os.environ.get("PY_VER", "3.11") %} - -channels: - - conda-forge - dependencies: - cmake>=3.18 - ninja @@ -16,12 +11,12 @@ dependencies: - pybind11-stubgen - vs2019_win-64 # [win] - pkgconfig # [win] - - python={{ python_version }} + - python ={{ get_env("PY_VER", default="3.9") }} - cython - pybind11 - pip - pytest - regex - # these two need to be newer than broken runner packages + # these two need to be newer than broken runner packages, 3.12 only - urllib3 - six