Merge branch 'master' into arith_object_flux

neuronsimulator · Jan 24, 2024 · 408fbb1 · 408fbb1
2 parents 1653ff0 + 9db73b4
commit 408fbb1
Show file tree

Hide file tree

Showing 98 changed files with 1,368 additions and 2,515 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,7 +1,7 @@
 version: 2.1
 
 orbs:
-  python: circleci/python@0.3.2
+  python: circleci/python@2.1.1
 
 jobs:
   manylinux2014-aarch64:
@@ -13,7 +13,7 @@ jobs:
         type: string
 
     machine:
-      image: ubuntu-2004:202101-01
+      image: default
 
     resource_class: arm.medium
 
@@ -49,25 +49,18 @@ jobs:
             # choose available python versions from pyenv
             pyenv_py_ver=""
             case << parameters.NRN_PYTHON_VERSION >> in
-              38) pyenv_py_ver="3.8.7" ;;
-              39) pyenv_py_ver="3.9.1" ;;
-              310) pyenv_py_ver="3.10.1" ;;
-              311) pyenv_py_ver="3.11.0" ;;
+              38) pyenv_py_ver="3.8" ;;
+              39) pyenv_py_ver="3.9" ;;
+              310) pyenv_py_ver="3.10" ;;
+              311) pyenv_py_ver="3.11" ;;
+              312) pyenv_py_ver="3.12" ;;
               *) echo "Error: pyenv python version not specified!" && exit 1;;
             esac
 
-            # install python dependencies: .10 is not available pyenv
-            if [ "<< parameters.NRN_PYTHON_VERSION >>" == "310" ]; then
-              sudo apt install software-properties-common -y
-              sudo add-apt-repository ppa:deadsnakes/ppa -y
-              sudo apt install python3.10 libpython3.10 python3.10-venv
-              export PYTHON_EXE=$(which python3.10)
-            else
-              cd /opt/circleci/.pyenv/plugins/python-build/../.. && git pull && cd -
-              env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install $pyenv_py_ver --force
-              pyenv global $pyenv_py_ver
-              export PYTHON_EXE=$(which python)
-            fi
+            cd /opt/circleci/.pyenv/plugins/python-build/../.. && git fetch --all && git checkout -B master origin/master && cd -
+            env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install $pyenv_py_ver --force
+            pyenv global $pyenv_py_ver
+            export PYTHON_EXE=$(which python)
 
             # test wheel
             packaging/python/test_wheels.sh $PYTHON_EXE $(ls -t wheelhouse/*.whl)
@@ -95,7 +88,7 @@ workflows:
                 - /circleci\/.*/
           matrix:
             parameters:
-              NRN_PYTHON_VERSION: ["311"]
+              NRN_PYTHON_VERSION: ["312"]
               NRN_NIGHTLY_UPLOAD: ["false"]
 
   nightly:
@@ -110,5 +103,5 @@ workflows:
       - manylinux2014-aarch64:
           matrix:
             parameters:
-              NRN_PYTHON_VERSION: ["38", "39", "310", "311"]
+              NRN_PYTHON_VERSION: ["38", "39", "310", "311", "312"]
               NRN_NIGHTLY_UPLOAD: ["true"]
diff --git a/.github/workflows/neuron-ci.yml b/.github/workflows/neuron-ci.yml
@@ -40,11 +40,11 @@ jobs:
       PY_MIN_VERSION: ${{ matrix.config.python_min_version || '3.8' }}
       PY_MAX_VERSION: ${{ matrix.config.python_max_version || '3.11' }}
       MUSIC_INSTALL_DIR: /opt/MUSIC
-      MUSIC_VERSION: 1.2.0
+      MUSIC_VERSION: 1.2.1
 
     strategy:
       matrix:
-        os: [ macOS-11, ubuntu-20.04]
+        os: [ macOS-12, ubuntu-20.04]
         config:
           - { matrix_eval : "CC=gcc-9 CXX=g++-9",   build_mode: "setuptools"}
           - { matrix_eval : "CC=gcc-10 CXX=g++-10", build_mode: "cmake", music: ON}
@@ -78,7 +78,7 @@ jobs:
               cmake_option: -DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_MPI=OFF
                 -DCORENRN_ENABLE_OPENMP=OFF -DNRN_ENABLE_RX3D=OFF
               sanitizer: thread
-          - os: macOS-12
+          - os: macOS-13
             config:
               build_mode: cmake
               # TODO: investigate rxd test timeouts in this build and re-enable them
@@ -96,7 +96,11 @@ jobs:
       - name: Install homebrew packages
         if: startsWith(matrix.os, 'macOS')
         run: |
+          # Unlink and re-link to prevent errors when GitHub macOS runner images
+          # install Python outside of brew; See actions/setup-python#577 and BlueBrain/libsonata/pull/317
+          brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done
           brew install ccache coreutils doxygen flex bison mpich ninja xz autoconf autoconf automake libtool
+          # We use both for dynamic mpi in nrn
           brew unlink mpich
           brew install openmpi
           brew install --cask xquartz
@@ -169,7 +173,8 @@ jobs:
           curl -L -o MUSIC.zip https://github.com/INCF/MUSIC/archive/refs/tags/${MUSIC_VERSION}.zip
           unzip MUSIC.zip && mv MUSIC-* MUSIC && cd MUSIC
           ./autogen.sh
-          ./configure --with-python-sys-prefix --prefix=$MUSIC_INSTALL_DIR --disable-anysource
+          # on some systems MPI library detection fails, provide exact flags/compilers
+          ./configure --with-python-sys-prefix --prefix=$MUSIC_INSTALL_DIR --disable-anysource MPI_CXXFLAGS="-g -O3" MPI_CFLAGS="-g -O3" MPI_LDFLAGS=" " CC=mpicc CXX=mpicxx
           make -j install
           deactivate
         working-directory: ${{runner.temp}}

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -50,7 +50,7 @@ jobs:
           name: Release ${{ env.REL_TAG }}
           prerelease: true
 
-  full-src-package:
+  nrn-full-src-package:
     runs-on: ubuntu-latest
     needs: tag-n-release
     steps:
@@ -72,13 +72,13 @@ jobs:
         cmake -DNRN_ENABLE_PYTHON=OFF -DNRN_ENABLE_RX3D=OFF -DNRN_ENABLE_MPI=OFF -DNRN_ENABLE_INTERVIEWS=OFF ../nrn
         make nrnversion_h VERBOSE=1
 
-    - name: Create full-src-package
+    - name: Create nrn-full-src-package
       id: tar
       run: |
-        tar -czvf full-src-package-${REL_TAG}.tar.gz nrn
-        echo "asset_file=full-src-package-${REL_TAG}.tar.gz" >> $GITHUB_OUTPUT
+        tar -czvf nrn-full-src-package-${REL_TAG}.tar.gz nrn
+        echo "asset_file=nrn-full-src-package-${REL_TAG}.tar.gz" >> $GITHUB_OUTPUT
 
-    - name: Upload full-src-package to release
+    - name: Upload nrn-full-src-package to release
       run: |
         gh release upload ${{ needs.tag-n-release.outputs.rel_tag }} ${{ steps.tar.outputs.asset_file }}
       env:

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -53,6 +53,7 @@ mac_m1_cmake_build:
     - real_python=$(python3 resolve_shim.py)
     - echo "python3=$(command -v python3) is really ${real_python}"
     - PYTHONEXECUTABLE=${real_python} ${real_python} -mvenv venv
+    - venv/bin/python3 -m ensurepip --upgrade --default-pip
     - venv/bin/pip install --upgrade pip -r nrn_requirements.txt
     - git submodule update --init --recursive --force --depth 1 -- external/nmodl
     - venv/bin/pip install --upgrade -r external/nmodl/requirements.txt
@@ -187,6 +188,7 @@ simulation_stack:
     bb5_constraint: volta
     bb5_cpus_per_task: 2
     bb5_partition: prod # assume this is a good source of GPU nodes
+    bb5_exclusive: user # allocate gpu node exclusively for the CI user (to avoid errors from oversubscription)
 .test_neuron:
   extends: [.ctest]
   variables:

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -74,6 +74,8 @@ option(NRN_ENABLE_RX3D "Enable rx3d support" ${NRN_ENABLE_RX3D_DEFAULT})
 option(NRN_ENABLE_CORENEURON "Enable CoreNEURON support" ${NRN_ENABLE_CORENEURON_DEFAULT})
 option(NRN_ENABLE_BACKTRACE "Enable pretty-printed backtraces" ${NRN_ENABLE_BACKTRACE_DEFAULT})
 option(NRN_ENABLE_TESTS "Enable unit tests" ${NRN_ENABLE_TESTS_DEFAULT})
+option(NRN_ENABLE_MATH_OPT "Enable extra math optimisations (to enable SIMD)"
+       ${NRN_ENABLE_MATH_OPT_DEFAULT})
 set(NRN_ENABLE_MODEL_TESTS
     "${NRN_ENABLE_MODEL_TESTS_DEFAULT}"
     CACHE STRING "Comma-separated list of detailed models to enable tests of.")
@@ -94,11 +96,11 @@ option(NRN_ENABLE_MOD_COMPATIBILITY "Enable CoreNEURON compatibility for MOD fil
        ${NRN_ENABLE_MOD_COMPATIBILITY_DEFAULT})
 option(NRN_ENABLE_REL_RPATH "Use relative RPATH in binaries. for relocatable installs/Python"
        ${NRN_ENABLE_REL_RPATH_DEFAULT})
-option(NRN_WHEEL_BUILD ${NRN_WHEEL_BUILD_DEFAULT})
+option(NRN_BINARY_DIST_BUILD ${NRN_BINARY_DIST_BUILD_DEFAULT})
 option(NRN_WHEEL_STATIC_READLINE "Use static readline libraries for the wheels."
        ${NRN_WHEEL_STATIC_READLINE_DEFAULT})
 mark_as_advanced(NRN_ENABLE_REL_RPATH)
-mark_as_advanced(NRN_WHEEL_BUILD)
+mark_as_advanced(NRN_BINARY_DIST_BUILD)
 
 # =============================================================================
 # Build options (string)
@@ -214,7 +216,7 @@ include(cmake/modules/FindPythonModule.cmake)
 include(cmake/Coverage.cmake)
 
 # set CMAKE_BUILD_TYPE and associated flags using allowableBuildTypes and CMAKE_BUILD_TYPE_DEFAULT
-set(allowableBuildTypes Custom Debug Release RelWithDebInfo Fast)
+set(allowableBuildTypes Custom Debug Release RelWithDebInfo Fast FastDebug)
 include(ReleaseDebugAutoFlags)
 
 # Try and emit an intelligent warning if the version number currently set in the CMake project(...)
@@ -929,7 +931,7 @@ if(BUILD_TYPE_UPPER MATCHES "CUSTOM")
 else()
   set(COMPILER_FLAGS "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}}")
 endif()
-string(JOIN " " COMPILER_FLAGS "${COMPILER_FLAGS}" ${NRN_COMPILE_FLAGS})
+string(JOIN " " COMPILER_FLAGS "${COMPILER_FLAGS}" ${NRN_COMPILE_FLAGS} ${CMAKE_CXX_FLAGS})
 
 message(STATUS "")
 message(STATUS "Configured NEURON ${PROJECT_VERSION}")

diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-[![Build Status](https://dev.azure.com/neuronsimulator/nrn/_apis/build/status/neuronsimulator.nrn?branchName=master)](https://dev.azure.com/neuronsimulator/nrn/_build/latest?definitionId=1&branchName=master) [![Actions Status](https://github.com/neuronsimulator/nrn/workflows/Windows%20Installer/badge.svg)](https://github.com/neuronsimulator/nrn/actions) [![Actions Status](https://github.com/neuronsimulator/nrn/workflows/NEURON%20CI/badge.svg)](https://github.com/neuronsimulator/nrn/actions) [![codecov](https://codecov.io/gh/neuronsimulator/nrn/branch/master/graph/badge.svg?token=T7PIDw6LrC)](https://codecov.io/gh/neuronsimulator/nrn) [![Documentation Status](https://readthedocs.org/projects/nrn/badge/?version=latest)](http://nrn.readthedocs.io/?badge=latest)
+[![Build Status](https://dev.azure.com/neuronsimulator/nrn/_apis/build/status/neuronsimulator.nrn?branchName=master)](https://dev.azure.com/neuronsimulator/nrn/_build/latest?definitionId=1&branchName=master) [![Actions Status](https://github.com/neuronsimulator/nrn/actions/workflows/windows.yml/badge.svg?branch=master)](https://github.com/neuronsimulator/nrn/actions) [![Actions Status](https://github.com/neuronsimulator/nrn/workflows/NEURON%20CI/badge.svg)](https://github.com/neuronsimulator/nrn/actions) [![codecov](https://codecov.io/gh/neuronsimulator/nrn/branch/master/graph/badge.svg?token=T7PIDw6LrC)](https://codecov.io/gh/neuronsimulator/nrn) [![Documentation Status](https://readthedocs.org/projects/nrn/badge/?version=latest)](http://nrn.readthedocs.io/?badge=latest)
 
 # NEURON
 NEURON is a simulator for models of neurons and networks of neuron. See [http://neuron.yale.edu](http://neuron.yale.edu) for installers, source code, documentation, tutorials, announcements of

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -42,6 +42,8 @@ stages:
             python.version: '3.10'
           Python311:
             python.version: '3.11'
+          Python312:
+            python.version: '3.12'
       steps:
 
       # Secure files documentation:
@@ -81,26 +83,30 @@ stages:
 
     # Jobs to build OSX wheels natively
     - job: 'MacOSWheels'
-      timeoutInMinutes: 45
+      timeoutInMinutes: 60
       pool:
-        vmImage: 'macOS-11'
+        vmImage: 'macOS-12'
       strategy:
         matrix:
           Python38:
             python.version: '3.8'
             python.org.version: '3.8.10'
-            python.installer.name: 'macosx10.9.pkg'
+            python.installer.name: 'macos11.pkg'
           Python39:
             python.version: '3.9'
             python.org.version: '3.9.13'
-            python.installer.name: 'macosx10.9.pkg'
+            python.installer.name: 'macos11.pkg'
           Python310:
             python.version: '3.10'
-            python.org.version: '3.10.5'
+            python.org.version: '3.10.11'
             python.installer.name: 'macos11.pkg'
           Python311:
             python.version: '3.11'
-            python.org.version: '3.11.1'
+            python.org.version: '3.11.7'
+            python.installer.name: 'macos11.pkg'
+          Python312:
+            python.version: '3.12'
+            python.org.version: '3.12.0'
             python.installer.name: 'macos11.pkg'
 
       steps:

diff --git a/bin/CMakeLists.txt b/bin/CMakeLists.txt
@@ -37,8 +37,7 @@ include(CMakeListsNrnMech)
 # nrnmech_makefile (based on coreneuron Configure templates)
 # =============================================================================
 nrn_configure_file(nrngui bin)
-cpp_cc_build_time_copy(INPUT ${CMAKE_CURRENT_SOURCE_DIR}/sortspike
-                       OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/sortspike)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/sortspike ${CMAKE_CURRENT_BINARY_DIR}/sortspike COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nrnivmodl_makefile_cmake.in
                ${PROJECT_BINARY_DIR}/bin/nrnmech_makefile @ONLY)
 string(JOIN " " NRN_PYTHON_VERSIONS_STRING ${NRN_PYTHON_VERSIONS})

diff --git a/bin/nrnivmodl.in b/bin/nrnivmodl.in
@@ -46,6 +46,8 @@ LinkCoreNEURON=false
 UserINCFLAGS=""
 UserLDFLAGS=""
 UserCOREFLAGS=()
+UserNMODLBIN=""
+UserNMODLFLAGS=""
 
 # - options come first but can be in any order.
 while [ "$1" ] ; do
@@ -68,6 +70,17 @@ while [ "$1" ] ; do
         UserCOREFLAGS+=(-l "${2}")
         shift
         shift;;
+    -nmodl)
+        echo "[NMODL][warning] Code generation with NMODL is pre-alpha, lacks features and is intended only for development use"
+        UserNMODLBIN="$2"
+        UserNMODLFLAGS="--neuron $UserNMODLFLAGS"
+        shift
+        shift;;
+    -nmodlflags)
+        echo "[NMODL][warning] If sympy is enabled, NMODL needs to be found in PYTHONPATH"
+        UserNMODLFLAGS="$UserNMODLFLAGS $2"
+        shift
+        shift;;
     -*)
         echo "$1 unrecognized"
         exit 1;;
@@ -159,7 +172,7 @@ for i in "${files[@]}" ; do
   echo "\
 ./${base_name// /\\ }.cpp: ${f}.mod
 	@printf \" -> \$(C_GREEN)NMODL\$(C_RESET) \$<\\\n\"
-	(cd \"$dir_name\"; @NRN_NOCMODL_SANITIZER_ENVIRONMENT_STRING@ MODLUNIT=\$(NRNUNITS) \$(NOCMODL) \"$base_name.mod\" -o \"$mdir\")
+	(cd \"$dir_name\"; @NRN_NOCMODL_SANITIZER_ENVIRONMENT_STRING@ MODLUNIT=\$(NRNUNITS) \$(NOCMODL) \"$base_name.mod\" -o \"$mdir\" $UserNMODLFLAGS)
 
 ./${base_name// /\\ }.o: ./${base_name// /\\ }.cpp
 	@printf \" -> \$(C_GREEN)Compiling\$(C_RESET) \$<\\\n\"
@@ -246,5 +259,5 @@ if [ "$LinkCoreNEURON" = true ] ; then
   fi
 fi
 
-make -j 4 -f "${bindir}/nrnmech_makefile" "ROOT=${prefix}" "MODOBJFILES=$MODOBJS" "UserLDFLAGS=$UserLDFLAGS" "UserINCFLAGS=$UserINCFLAGS" "LinkCoreNEURON=$LinkCoreNEURON" special &&
+make -j 4 -f "${bindir}/nrnmech_makefile" "ROOT=${prefix}" "MODOBJFILES=$MODOBJS" "UserLDFLAGS=$UserLDFLAGS" "UserINCFLAGS=$UserINCFLAGS" "LinkCoreNEURON=$LinkCoreNEURON" "UserNMODLBIN=$UserNMODLBIN" "UserNMODLFLAGS=$UserNMODLFLAGS" special &&
 echo "Successfully created $MODSUBDIR/special"
diff --git a/bin/nrnivmodl_makefile_cmake.in b/bin/nrnivmodl_makefile_cmake.in
@@ -4,6 +4,8 @@
 # UserLDFLAGS
 # UserINCFLAGS
 # LinkCoreNEURON
+# UserNMODLBIN
+# UserNMODLFLAGS
 # Rules to build MODOBJFILES from mod files are found in makemod2c_inc
 
 # Mechanisms version are by default 0.0, but should be overriden
@@ -15,6 +17,7 @@ OUTPUT = .
 DESTDIR =
 UserINCFLAGS =
 UserLDFLAGS =
+UserNMODLBIN =
 
 # install dirs
 bindir := ${ROOT}/@CMAKE_INSTALL_BINDIR@
@@ -67,7 +70,11 @@ CCOMPILE = $(CC) $(CFLAGS) @NRN_COMPILE_DEFS_STRING@ @NRN_COMPILE_FLAGS_STRING@
 CXX_LINK_EXE = $(CXX) $(CXXFLAGS) @CMAKE_EXE_LINKER_FLAGS@ @NRN_LINK_FLAGS_STRING@
 CXX_LINK_SHARED = $(CXX) $(CXXFLAGS) @CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS@ @CMAKE_SHARED_LIBRARY_CXX_FLAGS@ @CMAKE_SHARED_LINKER_FLAGS@ @NRN_LINK_FLAGS_STRING@
 
+ifeq ($(UserNMODLBIN), )
 NOCMODL = $(bindir)/nocmodl
+else
+NOCMODL = $(UserNMODLBIN)
+endif
 NRNUNITS = $(datadir_lib)/nrnunits.lib
 
 # File path config (internal)