Merge branch 'master' into cornu/remove_regexp

.github/workflows/neuron-ci.yml

@@ -44,7 +44,7 @@ jobs:
 
     strategy:
       matrix:
-        os: [ macOS-11, ubuntu-20.04]
+        os: [ macOS-12, ubuntu-20.04]
         config:
           - { matrix_eval : "CC=gcc-9 CXX=g++-9",   build_mode: "setuptools"}
           - { matrix_eval : "CC=gcc-10 CXX=g++-10", build_mode: "cmake", music: ON}
@@ -78,7 +78,7 @@ jobs:
               cmake_option: -DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_MPI=OFF
                 -DCORENRN_ENABLE_OPENMP=OFF -DNRN_ENABLE_RX3D=OFF
               sanitizer: thread
-          - os: macOS-12
+          - os: macOS-13
             config:
               build_mode: cmake
               # TODO: investigate rxd test timeouts in this build and re-enable them
@@ -96,7 +96,11 @@ jobs:
       - name: Install homebrew packages
         if: startsWith(matrix.os, 'macOS')
         run: |
+          # Unlink and re-link to prevent errors when GitHub macOS runner images
+          # install Python outside of brew; See actions/setup-python#577 and BlueBrain/libsonata/pull/317
+          brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done
           brew install ccache coreutils doxygen flex bison mpich ninja xz autoconf autoconf automake libtool
+          # We use both for dynamic mpi in nrn
           brew unlink mpich
           brew install openmpi
           brew install --cask xquartz

.github/workflows/release.yml

@@ -50,7 +50,7 @@ jobs:
           name: Release ${{ env.REL_TAG }}
           prerelease: true
 
-  full-src-package:
+  nrn-full-src-package:
     runs-on: ubuntu-latest
     needs: tag-n-release
     steps:
@@ -72,13 +72,13 @@ jobs:
         cmake -DNRN_ENABLE_PYTHON=OFF -DNRN_ENABLE_RX3D=OFF -DNRN_ENABLE_MPI=OFF -DNRN_ENABLE_INTERVIEWS=OFF ../nrn
         make nrnversion_h VERBOSE=1
 
-    - name: Create full-src-package
+    - name: Create nrn-full-src-package
       id: tar
       run: |
-        tar -czvf full-src-package-${REL_TAG}.tar.gz nrn
-        echo "asset_file=full-src-package-${REL_TAG}.tar.gz" >> $GITHUB_OUTPUT
+        tar -czvf nrn-full-src-package-${REL_TAG}.tar.gz nrn
+        echo "asset_file=nrn-full-src-package-${REL_TAG}.tar.gz" >> $GITHUB_OUTPUT
 
-    - name: Upload full-src-package to release
+    - name: Upload nrn-full-src-package to release
       run: |
         gh release upload ${{ needs.tag-n-release.outputs.rel_tag }} ${{ steps.tar.outputs.asset_file }}
       env:

azure-pipelines.yml

@@ -42,6 +42,8 @@ stages:
             python.version: '3.10'
           Python311:
             python.version: '3.11'
+          Python312:
+            python.version: '3.12'
       steps:
 
       # Secure files documentation:
@@ -83,24 +85,28 @@ stages:
     - job: 'MacOSWheels'
       timeoutInMinutes: 60
       pool:
-        vmImage: 'macOS-11'
+        vmImage: 'macOS-12'
       strategy:
         matrix:
           Python38:
             python.version: '3.8'
             python.org.version: '3.8.10'
-            python.installer.name: 'macosx10.9.pkg'
+            python.installer.name: 'macos11.pkg'
           Python39:
             python.version: '3.9'
             python.org.version: '3.9.13'
-            python.installer.name: 'macosx10.9.pkg'
+            python.installer.name: 'macos11.pkg'
           Python310:
             python.version: '3.10'
-            python.org.version: '3.10.5'
+            python.org.version: '3.10.11'
             python.installer.name: 'macos11.pkg'
           Python311:
             python.version: '3.11'
-            python.org.version: '3.11.1'
+            python.org.version: '3.11.7'
+            python.installer.name: 'macos11.pkg'
+          Python312:
+            python.version: '3.12'
+            python.org.version: '3.12.0'
             python.installer.name: 'macos11.pkg'
 
       steps:

bin/CMakeLists.txt

@@ -37,8 +37,7 @@ include(CMakeListsNrnMech)
 # nrnmech_makefile (based on coreneuron Configure templates)
 # =============================================================================
 nrn_configure_file(nrngui bin)
-cpp_cc_build_time_copy(INPUT ${CMAKE_CURRENT_SOURCE_DIR}/sortspike
-                       OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/sortspike)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/sortspike ${CMAKE_CURRENT_BINARY_DIR}/sortspike COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nrnivmodl_makefile_cmake.in
                ${PROJECT_BINARY_DIR}/bin/nrnmech_makefile @ONLY)
 string(JOIN " " NRN_PYTHON_VERSIONS_STRING ${NRN_PYTHON_VERSIONS})

bin/nrnivmodl.in

@@ -46,6 +46,8 @@ LinkCoreNEURON=false
 UserINCFLAGS=""
 UserLDFLAGS=""
 UserCOREFLAGS=()
+UserNMODLBIN=""
+UserNMODLFLAGS=""
 
 # - options come first but can be in any order.
 while [ "$1" ] ; do
@@ -68,6 +70,17 @@ while [ "$1" ] ; do
         UserCOREFLAGS+=(-l "${2}")
         shift
         shift;;
+    -nmodl)
+        echo "[NMODL][warning] Code generation with NMODL is pre-alpha, lacks features and is intended only for development use"
+        UserNMODLBIN="$2"
+        UserNMODLFLAGS="--neuron $UserNMODLFLAGS"
+        shift
+        shift;;
+    -nmodlflags)
+        echo "[NMODL][warning] If sympy is enabled, NMODL needs to be found in PYTHONPATH"
+        UserNMODLFLAGS="$UserNMODLFLAGS $2"
+        shift
+        shift;;
     -*)
         echo "$1 unrecognized"
         exit 1;;
@@ -159,7 +172,7 @@ for i in "${files[@]}" ; do
   echo "\
 ./${base_name// /\\ }.cpp: ${f}.mod
 	@printf \" -> \$(C_GREEN)NMODL\$(C_RESET) \$<\\\n\"
-	(cd \"$dir_name\"; @NRN_NOCMODL_SANITIZER_ENVIRONMENT_STRING@ MODLUNIT=\$(NRNUNITS) \$(NOCMODL) \"$base_name.mod\" -o \"$mdir\")
+	(cd \"$dir_name\"; @NRN_NOCMODL_SANITIZER_ENVIRONMENT_STRING@ MODLUNIT=\$(NRNUNITS) \$(NOCMODL) \"$base_name.mod\" -o \"$mdir\" $UserNMODLFLAGS)
 
 ./${base_name// /\\ }.o: ./${base_name// /\\ }.cpp
 	@printf \" -> \$(C_GREEN)Compiling\$(C_RESET) \$<\\\n\"
@@ -246,5 +259,5 @@ if [ "$LinkCoreNEURON" = true ] ; then
   fi
 fi
 
-make -j 4 -f "${bindir}/nrnmech_makefile" "ROOT=${prefix}" "MODOBJFILES=$MODOBJS" "UserLDFLAGS=$UserLDFLAGS" "UserINCFLAGS=$UserINCFLAGS" "LinkCoreNEURON=$LinkCoreNEURON" special &&
+make -j 4 -f "${bindir}/nrnmech_makefile" "ROOT=${prefix}" "MODOBJFILES=$MODOBJS" "UserLDFLAGS=$UserLDFLAGS" "UserINCFLAGS=$UserINCFLAGS" "LinkCoreNEURON=$LinkCoreNEURON" "UserNMODLBIN=$UserNMODLBIN" "UserNMODLFLAGS=$UserNMODLFLAGS" special &&
 echo "Successfully created $MODSUBDIR/special"

bin/nrnivmodl_makefile_cmake.in

@@ -4,6 +4,8 @@
 # UserLDFLAGS
 # UserINCFLAGS
 # LinkCoreNEURON
+# UserNMODLBIN
+# UserNMODLFLAGS
 # Rules to build MODOBJFILES from mod files are found in makemod2c_inc
 
 # Mechanisms version are by default 0.0, but should be overriden
@@ -15,6 +17,7 @@ OUTPUT = .
 DESTDIR =
 UserINCFLAGS =
 UserLDFLAGS =
+UserNMODLBIN =
 
 # install dirs
 bindir := ${ROOT}/@CMAKE_INSTALL_BINDIR@
@@ -67,7 +70,11 @@ CCOMPILE = $(CC) $(CFLAGS) @NRN_COMPILE_DEFS_STRING@ @NRN_COMPILE_FLAGS_STRING@
 CXX_LINK_EXE = $(CXX) $(CXXFLAGS) @CMAKE_EXE_LINKER_FLAGS@ @NRN_LINK_FLAGS_STRING@
 CXX_LINK_SHARED = $(CXX) $(CXXFLAGS) @CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS@ @CMAKE_SHARED_LIBRARY_CXX_FLAGS@ @CMAKE_SHARED_LINKER_FLAGS@ @NRN_LINK_FLAGS_STRING@
 
+ifeq ($(UserNMODLBIN), )
 NOCMODL = $(bindir)/nocmodl
+else
+NOCMODL = $(UserNMODLBIN)
+endif
 NRNUNITS = $(datadir_lib)/nrnunits.lib
 
 # File path config (internal)

bldnrnmacpkgcmake.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -ex
 
-default_pythons="python3.8 python3.9 python3.10"
+default_pythons="python3.8 python3.9 python3.10 python3.11"
 # distribution built with
 # bash bldnrnmacpkgcmake.sh
 # without args, default are the pythons above.
@@ -11,6 +11,10 @@ default_pythons="python3.8 python3.9 python3.10"
 # All pythons must have the same macos version and that will become
 # the MACOSX_DEPLOYMENT_TARGET
 
+# On my machine, to build nrn-x.x.x-macosx-10.9-universal2-py-38-39-310-311.pkg
+# I built my own versions of 3.8 in $HOME/soft/python3.8, and
+export PATH=$HOME/soft/python3.8/bin:$PATH
+
 CPU=`uname -m`
 
 universal="yes" # changes to "no" if any python not universal
@@ -45,11 +49,15 @@ if test "$archs" != "universal2" ; then
   universal=no
 fi
 
+# Arrgh. Recent changes to nrn source require at least 10.15!
+macosver=10.15
+mac_platform=macosx-$macosver-$archs
+
 export MACOSX_DEPLOYMENT_TARGET=$macosver
 echo "MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET"
 
 if test "$NRN_SRC" == "" ; then
-  NRN_SRC=$HOME/neuron/nrn
+  NRN_SRC=`pwd`
 fi
 NRN_BLD=$NRN_SRC/build
 NSRC=$NRN_SRC
@@ -64,7 +72,7 @@ mkdir -p $NRN_BLD
 rm -r -f $NRN_BLD/*
 cd $NRN_BLD
 
-PYVS="py" # will be part of package file name, eg. py-37-38-39-310
+PYVS="py" # will be part of package file name, eg. py-38-39-310-311
 pythons="" # will be arg value of NRN_PYTHON_DYNAMIC
 for i in $args ; do
   PYVER=`$i -c 'from sys import version_info as v ; print (str(v.major) + str(v.minor)); quit()'`
@@ -83,7 +91,7 @@ fi
 # from brew install gedit). User installations are expected to have the
 # former and would only accidentally have the latter.
 
-cmake .. -DCMAKE_INSTALL_PREFIX=$NRN_INSTALL \
+cmake .. -G Ninja -DCMAKE_INSTALL_PREFIX=$NRN_INSTALL \
   -DNRN_ENABLE_MPI_DYNAMIC=ON \
   -DPYTHON_EXECUTABLE=`which python3` -DNRN_ENABLE_PYTHON_DYNAMIC=ON \
   -DNRN_PYTHON_DYNAMIC="$pythons" \
@@ -94,7 +102,7 @@ cmake .. -DCMAKE_INSTALL_PREFIX=$NRN_INSTALL \
   -DCMAKE_PREFIX_PATH=/usr/X11 \
   -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++
 
-make -j install
+ninja install
 
 if test "$universal" = "yes" ; then
   _temp="`lipo -archs $NRN_INSTALL/share/nrn/demo/release/$CPU/special`"
@@ -121,32 +129,29 @@ chk () {
   )
 }
 
+# test basic functionality
+for i in $args ; do
+  chk $i
+done
+
 #/Applications/Packages.app from
 # http://s.sudre.free.fr/Software/Packages/about.html
 # For mac to do a productsign, need my developerID_installer.cer
 # and Neurondev.p12 file. To add to the keychain, double click each
 # of those files. By default, I added my certificates to the login keychain.
-make macpkg # will sign the binaries, construct below
-            # mentioned PACKAGE_FILE_NAME, and request notarization from
-            # Apple. At the end it will print a stapling request that you
-            # should run manually after receiving a "success" email from
-            # Apple.
+ninja macpkg # will sign the binaries, construct below
+            # mentioned PACKAGE_FILE_NAME, request notarization from
+            # Apple, and staple the package.
 
-# test basic functionality
-for i in $args ; do
-  chk $i
-done
-
-# upload package to neuron.yale.edu
-ALPHADIR='hines@neuron.yale.edu:/home/htdocs/ftp/neuron/versions/alpha'
+# Copy the package to $HOME/$PACKAGE_FULL_NAME
+# You should then manually upload that to github.
 describe="`sh $NRN_SRC/nrnversion.sh describe`"
 macos=macos${MACOSX_DEPLOYMENT_TARGET}
 PACKAGE_FULL_NAME=nrn-${describe}-${mac_platform}-${PYVS}.pkg
-PACKATE_DOWNLOAD_NAME=$ALPHADIR/$PACKAGE_FULL_NAME
 PACKAGE_FILE_NAME=$NRN_BLD/src/mac/build/NEURON.pkg
+
+cp $PACKAGE_FILE_NAME $HOME/$PACKAGE_FULL_NAME
+
 echo "
-  Until we figure out how to automatically staple the notarization
-  the following two commands must be executed manually.
-  xcrun stapler staple $PACKAGE_FILE_NAME
-  cp $PACKAGE_FILE_NAME $HOME/$PACKAGE_FULL_NAME
+  Manually upload $HOME/$PACKAGE_FULL_NAME to github
 "

cmake/CompilerHelper.cmake

@@ -37,16 +37,13 @@ if(CMAKE_C_COMPILER_ID MATCHES "PGI" OR CMAKE_C_COMPILER_ID MATCHES "NVHPC")
     # "src/modlunit/units.cpp", warning #170-D: pointer points outside of underlying object
     # "src/nrnpython/grids.cpp", warning #174-D: expression has no effect
     # "src/nmodl/nocpout.cpp", warning #177-D: variable "j" was declared but never referenced
-    # "src/mesch/conjgrad.c", warning #180-D: argument is incompatible with formal parameter
     # "src/nrniv/partrans.cpp", warning #186-D: pointless comparison of unsigned integer with zero
-    # "src/mesch/machine.h", warning #301-D: typedef name has already been declared (with same type)
     # "src/nrnpython/rxdmath.cpp", warning #541-D: allowing all exceptions is incompatible with previous function
     # "src/nmodl/nocpout.cpp", warning #550-D: variable "sion" was set but never used
     # "src/gnu/neuron_gnu_builtin.h", warning #816-D: type qualifier on return type is meaningless"
     # "src/modlunit/consist.cpp", warning #2465-D: conversion from a string literal to "char *" is deprecated
     # ~~~
-    list(APPEND NRN_COMPILE_FLAGS
-         --diag_suppress=1,47,111,128,170,174,177,180,186,301,541,550,816,2465)
+    list(APPEND NRN_COMPILE_FLAGS --diag_suppress=1,47,111,128,170,174,177,186,541,550,816,2465)
   endif()
   list(APPEND NRN_COMPILE_FLAGS -noswitcherror)
   list(APPEND NRN_LINK_FLAGS -noswitcherror)

cmake/NeuronFileLists.cmake

@@ -427,7 +427,7 @@ set(NMODL_FILES_LIST
     units.cpp
     version.cpp)
 
-set(IVOS_FILES_LIST listimpl.cpp observe.cpp regexp.cpp resource.cpp)
+set(IVOS_FILES_LIST observe.cpp regexp.cpp resource.cpp)
 
 set(MPI_DYNAMIC_INCLUDE nrnmpi_dynam.h nrnmpi_dynam_cinc nrnmpi_dynam_wrappers.inc)
 

cmake/SanitizerHelper.cmake

@@ -48,8 +48,8 @@ if(NRN_SANITIZERS)
   # directory
   foreach(sanitizer ${nrn_sanitizers})
     if(EXISTS "${PROJECT_SOURCE_DIR}/.sanitizers/${sanitizer}.supp")
-      cpp_cc_build_time_copy(INPUT "${PROJECT_SOURCE_DIR}/.sanitizers/${sanitizer}.supp"
-                             OUTPUT "${PROJECT_BINARY_DIR}/share/nrn/sanitizers/${sanitizer}.supp")
+      configure_file("${PROJECT_SOURCE_DIR}/.sanitizers/${sanitizer}.supp"
+                     "${PROJECT_BINARY_DIR}/share/nrn/sanitizers/${sanitizer}.supp" COPYONLY)
       install(FILES "${PROJECT_BINARY_DIR}/share/nrn/sanitizers/${sanitizer}.supp"
               DESTINATION "${CMAKE_INSTALL_PREFIX}/share/nrn/sanitizers")
     endif()

docs/guide/porting_mechanisms_to_cpp.rst

@@ -324,3 +324,27 @@ If your MOD files produce these deprecation warnings, make sure that the
 relevant method (``vector_capacity`` in this example) is being called with an
 argument of the correct type (``IvocVect*``), and not a type that is implicitly
 converted to ``void*``.
+
+Legacy random number generators and API
+---------------------------------------
+
+Various changes have also been done in the API of NEURON functions related to random 
+number generators.
+
+First, in |neuron_with_cpp_mechanisms| parameters passed to the functions need to be
+of the correct type as it was already mentioned in :ref:`function-decls-with-incorrect-types`.
+The most usual consequence of that is that NEURON random API functions that were taking as
+an argument a ``void*`` now need to called with a ``Rand*``. An example of the changes needed
+to fix this issue is given in `182129 <https://github.com/ModelDBRepository/182129/pull/1>`_.
+
+Another related change is with ``scop_rand()`` function that is usually used for defining a
+``URAND`` ``FUNCTION`` in mod files to return a number based on a uniform distribution from 0 to 1.
+This function now takes no argument anymore. An example of this change can also be found in
+`182129 <https://github.com/ModelDBRepository/182129/pull/1>`_.
+
+Finally, the preferred random number generator is ``Random123``. You can find more information
+about that in :meth:`Random.Random123` and :ref:`Randomness in NEURON models`. An example of the
+usage of ``Random123`` can be seen in `netstim.mod <https://github.com/neuronsimulator/nrn/blob/master/src/nrnoc/netstim.mod>`_
+and its `corresponding test <https://github.com/neuronsimulator/nrn/blob/master/test/coreneuron/test_psolve.py#L60>`_.`
+Another important aspect of ``Random123`` is that it's supported in CoreNEURON as well. For more
+information about this see :ref:`Random Number Generators: Random123 vs MCellRan4`.

docs/hoc/programming/math/matrix.rst

@@ -53,20 +53,17 @@ Matrix
         By default, a new Matrix is of type MFULL (= 1) and allocates storage for 
         all nrow*ncol elements. Scaffolding is in place for matrices of storage 
         type MSPARSE (=2) and MBAND (=3) but not many methods have been interfaced 
-        to the meschach library at this time. If a method is called on a matrix type 
+        to the eigen library at this time. If a method is called on a matrix type 
         whose method has not been implemented, an error message will be printed. 
         It is intended that implemented methods will be transparent to the user, eg 
         m*x=b (``x = m.solv(b)`` ) will solve the linear system 
         regardless of the type of m and 
         v1 = m*v2 (``v1 = m.mulv(v2)`` ) will perform the vector multiplication. 
 
-        Matrix is implemented using the 
-        `meschach c library by David E. Stewart <http://www.math.uiowa.edu/~dstewart/meschach/meschach.html>`_
-        (discovered at http://www.netlib.org/c/index.html\ ) which contains a large collection 
-        of routines for sparse, banded, and full matrices. Many of the useful 
-        routines  have not 
-        been interfaced with the hoc interpreter but can be easily added on request 
-        or you can add it yourself 
+        Matrix is implemented using the `eigen3 library <https://eigen.tuxfamily.org>`_ 
+        which contains a large collection of routines for sparse, banded, and full matrices. 
+        Many of the useful routines  have not been interfaced with the hoc 
+        interpreter but can be easily added on request or you can add it yourself 
         by analogy with the code in ``nrn/src/ivoc/(matrix.c ocmatrix.[ch])`` 
         At this time the MFULL matrix type is complete enough to do useful work 
         and MSPARSE can be used to multiply a matrix by a vector and solve