Improved wrappers and OpenMPI handling in Python MPI tests

pyproject.toml

@@ -4,6 +4,7 @@ markers = [
     "skipif_missing_hdf5: skip test if NEST was built without HDF5 support",
     "skipif_missing_mpi: skip test if NEST was built without MPI support",
     "skipif_missing_threads: skip test if NEST was built without multithreading support",
+    "skipif_incompatible_mpi: skip if NEST with built with MPI that does not work with subprocess",
     "simulation: the simulation class to use. Always pass a 2nd dummy argument"
 ]
 

testsuite/do_tests.sh

@@ -508,7 +508,7 @@ if test "${PYTHON}"; then
 	XUNIT_FILE="${REPORTDIR}/${XUNIT_NAME}_sli2py_mpi.xml"
 	env
 	set +e
-	"${PYTHON}" -m pytest --noconftest --verbose --timeout $TIME_LIMIT --junit-xml="${XUNIT_FILE}" --numprocesses=1 \
+	"${PYTHON}" -m pytest --verbose --timeout $TIME_LIMIT --junit-xml="${XUNIT_FILE}" --numprocesses=1 \
 		    "${PYNEST_TEST_DIR}/sli2py_mpi" 2>&1 | tee -a "${TEST_LOGFILE}"
 	set -e
     fi

testsuite/pytests/conftest.py

@@ -34,6 +34,7 @@ def test_gsl():
 import dataclasses
 import os
 import pathlib
+import subprocess
 import sys
 
 import nest
@@ -156,6 +157,25 @@ def have_plotting():
         return False
 
 
+@pytest.fixture(scope="session")
+def subprocess_compatible_mpi():
+    """Until at least OpenMPI 4.1.6, the following fails due to a bug in OpenMPI, from 5.0.7 is definitely safe."""
+
+    res = subprocess.run(["mpirun", "-np", "1", "echo"])
+    return res.returncode == 0
+
+
+@pytest.fixture(autouse=True)
+def skipif_incompatible_mpi(request, subprocess_compatible_mpi):
+    """
+    Globally applied fixture that skips tests marked to be skipped when MPI is
+    not compatible with subprocess.
+    """
+
+    if not subprocess_compatible_mpi and request.node.get_closest_marker("skipif_incompatible_mpi"):
+        pytest.skip("skipped because MPI is incompatible with subprocess")
+
+
 @pytest.fixture(autouse=True)
 def simulation_class(request):
     return getattr(request, "param", testsimulation.Simulation)

testsuite/pytests/sli2py_mpi/mpi_test_wrapper.py

@@ -29,29 +29,33 @@
 
 - The process is managed by subclasses of the `MPITestWrapper` base class
 - Each test file must contain **exactly one** test function
+    - The test function must be protected with the `@pytest.mark.skipif_incompatible_mpi`
+      marker to protect against buggy OpenMPI versions (at least up to 4.1.6; 5.0.7 and
+      later are definitely good).
     - The test function must be decorated with a subclass of `MPITestWrapper`
     - The wrapper will write a modified version of the test file as `runner.py`
       to a temporary directory and mpirun it from there; results are collected
       in the temporary directory
     - The test function can be decorated with other pytest decorators. These
       are evaluated in the wrapping process
     - No decorators are written to the `runner.py` file.
-    - Test files **must not import nest** outside the test function
+    - Test files must import all required modules (especially `nest`) inside the
+      test function.
+    - The docstring for the test function shall in its last line specify what data
+      the test function outputs for comparison by the test.
     - In `runner.py`, the following constants are defined:
          - `SPIKE_LABEL`
          - `MULTI_LABEL`
          - `OTHER_LABEL`
       They must be used as `label` for spike recorders and multimeters, respectively,
       or for other files for output data (CSV files). They are format strings expecting
       the number of processes with which NEST is run as argument.
-- `conftest.py` must not be loaded, otherwise mpirun will return a non-zero exit code;
-  use `pytest --noconftest`
 - Set `debug=True` on the decorator to see debug output and keep the
   temporary directory that has been created (latter works only in
   Python 3.12 and later)
 - Evaluation criteria are determined by the `MPITestWrapper` subclass
 
-This is still work in progress.
+
 """
 
 import ast

testsuite/pytests/sli2py_mpi/test_all_to_all.py

@@ -19,24 +19,22 @@
 # You should have received a copy of the GNU General Public License
 # along with NEST.  If not, see <http://www.gnu.org/licenses/>.
 
-import numpy as np
-import pandas
 import pytest
 from mpi_test_wrapper import MPITestAssertEqual
 
 
 # Parametrization over the number of nodes here only to show hat it works
+@pytest.mark.skipif_incompatible_mpi
 @pytest.mark.parametrize("N", [4, 7])
 @MPITestAssertEqual([1, 4], debug=False)
 def test_all_to_all(N):
     """
     Confirm that all-to-all connections created correctly for more targets than local nodes.
+
+    The test is performed on connection data written to OTHER_LABEL.
     """
 
     import nest
-    import pandas as pd
-
-    nest.ResetKernel()
 
     nrns = nest.Create("parrot_neuron", n=N)
     nest.Connect(nrns, nrns, "all_to_all")

testsuite/pytests/sli2py_mpi/test_gap_junctions_mpi.py

@@ -19,25 +19,24 @@
 # You should have received a copy of the GNU General Public License
 # along with NEST.  If not, see <http://www.gnu.org/licenses/>.
 
-
+import pytest
 from mpi_test_wrapper import MPITestAssertEqual
 
 
+@pytest.mark.skipif_incompatible_mpi
+@pytest.mark.skipif_missing_gsl
 @MPITestAssertEqual([1, 2, 4], debug=False)
 def test_gap_junctions_mpi():
     """
     Test gap junction functionality in parallel.
 
     This is an overall test of the hh_psc_alpha_gap model connected by gap_junction.
     The test checks if the gap junction functionality works in parallel.
+
+    The test is performed on the spike data recorded to SPIKE_LABEL during the simulation.
     """
 
     import nest
-    import pandas as pd
-
-    # We can only test here if GSL is available
-    if not nest.ll_api.sli_func("statusdict/have_gsl ::"):
-        return
 
     total_vps = 4
     h = 0.1

testsuite/pytests/sli2py_mpi/test_issue_1957.py

@@ -19,14 +19,17 @@
 # You should have received a copy of the GNU General Public License
 # along with NEST.  If not, see <http://www.gnu.org/licenses/>.
 
-
+import pytest
 from mpi_test_wrapper import MPITestAssertEqual
 
 
+@pytest.mark.skipif_incompatible_mpi
 @MPITestAssertEqual([2, 4])
 def test_issue_1957():
     """
     Confirm that GetConnections works in parallel without hanging if not all ranks have connections.
+
+    The test is performed on connection data written to OTHER_LABEL.
     """
 
     import nest

testsuite/pytests/sli2py_mpi/test_issue_2119.py

@@ -32,11 +32,14 @@
 ]
 
 
+@pytest.mark.skipif_incompatible_mpi
 @pytest.mark.parametrize(["kind", "specs"], random_params)
 @MPITestAssertEqual([1, 2, 4])
 def test_issue_2119(kind, specs):
     """
     Confirm that randomized node parameters work correctly under MPI and OpenMP.
+
+    The test is performed on GID-V_m data written to OTHER_LABEL.
     """
 
     import nest

testsuite/pytests/sli2py_mpi/test_issue_281.py

@@ -20,17 +20,20 @@
 # along with NEST.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import pytest
 from mpi_test_wrapper import MPITestAssertCompletes
 
 
+@pytest.mark.skipif_incompatible_mpi
 @MPITestAssertCompletes([1, 2, 4])
 def test_issue_281():
     """
     Confirm that ConnectLayers works MPI-parallel for fixed fan-out.
+
+    This test only confirms completion, no data is tested.
     """
 
     import nest
-    import pandas as pd
 
     layer = nest.Create("parrot_neuron", positions=nest.spatial.grid(shape=[3, 3]))
     nest.Connect(

testsuite/pytests/sli2py_mpi/test_issue_600.py

@@ -20,21 +20,21 @@
 # along with NEST.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import pytest
 from mpi_test_wrapper import MPITestAssertEqual
 
 
+@pytest.mark.skipif_incompatible_mpi
+@pytest.mark.skipif_missing_gsl
 @MPITestAssertEqual([1, 2, 4], debug=False)
 def test_issue_600():
     """
     Confirm that waveform relaxation works with MPI.
+
+    The test compares data written by spike recorder to SPIKE_LABEL.
     """
 
     import nest
-    import pandas as pd
-
-    # We can only test here if GSL is available
-    if not nest.ll_api.sli_func("statusdict/have_gsl ::"):
-        return
 
     total_vps = 4
     h = 0.1

testsuite/pytests/sli2py_mpi/test_mini_brunel_ps.py

@@ -19,20 +19,21 @@
 # You should have received a copy of the GNU General Public License
 # along with NEST.  If not, see <http://www.gnu.org/licenses/>.
 
-
+import pytest
 from mpi_test_wrapper import MPITestAssertEqual
 
 
+@pytest.mark.skipif_incompatible_mpi
 @MPITestAssertEqual([1, 2, 4])
 def test_mini_brunel_ps():
     """
     Confirm that downscaled Brunel net with precise neurons is invariant under number of MPI ranks.
+
+    The test compares data written by spike_recorder to SPIKE_LABEL.
     """
 
     import nest
 
-    nest.ResetKernel()
-
     nest.set(total_num_virtual_procs=4, overwrite_files=True)
 
     # Model parameters

testsuite/pytests/sli2py_mpi/test_self_get_conns_with_empty_ranks.py

@@ -19,17 +19,18 @@
 # You should have received a copy of the GNU General Public License
 # along with NEST.  If not, see <http://www.gnu.org/licenses/>.
 
-import numpy as np
-import pandas
 import pytest
 from mpi_test_wrapper import MPITestAssertEqual
 
 
 # Parametrization over the number of nodes here only to show hat it works
+@pytest.mark.skipif_incompatible_mpi
 @MPITestAssertEqual([1, 2, 4], debug=False)
-def test_self_get_conns_with_empty_ranks():
+def test_get_conns_with_empty_ranks():
     """
-    Selftest: Confirm that connections can be gathered correctly even if some ranks have no neurons.
+    Confirm that connections can be gathered correctly even if some ranks have no neurons.
+
+    The test compares connection data written to OTHER_LABEL.
     """
 
     import nest