Skip to content

Commit 009ac0c

Browse files
heplesserheplesser
authored
Merge pull request #3524 from heplesser/pynest-ng
PyNEST-NG: A new Python interface for NEST directly connecting Python to C++. This removes the SLI interpreter.
2 parents fcbb3b3 + b2d9844 commit 009ac0c

1,220 files changed

Lines changed: 16957 additions & 109046 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build_nest_wheel.yml‎

Lines changed: 318 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,318 @@
1+
name: Build NEST simulator wheels
2+
3+
on:
4+
push:
5+
branches: [master, pynest-ng, hep_pynest-ng]
6+
pull_request:
7+
release:
8+
types: [published]
9+
workflow_dispatch:
10+
11+
jobs:
12+
build_wheels:
13+
name: Build wheels on ${{ matrix.os }}
14+
runs-on: ${{ matrix.os }}
15+
strategy:
16+
fail-fast: true
17+
matrix:
18+
# TODO: Conditional matrix: fewer builds for PRs, full builds for releases/pushes
19+
# os: ${{ github.event_name == 'pull_request' && fromJSON('["ubuntu-24.04"]') || fromJSON('["macos-15-intel", "macos-15", "ubuntu-24.04"]') }}
20+
os: ["macos-15-intel", "macos-15", "ubuntu-24.04"]
21+
steps:
22+
- uses: actions/checkout@v4
23+
24+
- name: Cache pip packages
25+
uses: actions/cache@v4
26+
with:
27+
path: ~/.cache/pip
28+
key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }}
29+
restore-keys: |
30+
${{ runner.os }}-pip-
31+
32+
- name: Build wheels
33+
uses: pypa/cibuildwheel@v2.22.0
34+
with:
35+
package-dir: .
36+
output-dir: wheelhouse
37+
config-file: "{package}/pyproject.toml"
38+
39+
- uses: actions/upload-artifact@v4
40+
with:
41+
name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
42+
path: ./wheelhouse/*.whl
43+
44+
build_sdist:
45+
name: Build source distribution
46+
runs-on: ubuntu-latest
47+
steps:
48+
- uses: actions/checkout@v4
49+
50+
- name: Cache pip packages
51+
uses: actions/cache@v4
52+
with:
53+
path: ~/.cache/pip
54+
key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }}
55+
restore-keys: |
56+
${{ runner.os }}-pip-
57+
58+
- name: Build sdist
59+
run: pipx run build --sdist
60+
61+
- uses: actions/upload-artifact@v4
62+
with:
63+
name: cibw-sdist
64+
path: dist/*.tar.gz
65+
66+
upload_pypi:
67+
name: Upload wheels to pypi
68+
needs: [test_local_wheels, build_sdist]
69+
runs-on: ubuntu-latest
70+
environment: testpypi
71+
permissions:
72+
id-token: write
73+
# Only run on releases or manual workflow dispatch from non-fork branches
74+
if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.repository.fork == false) || (github.event_name == 'push' && github.event.repository.fork == false)
75+
steps:
76+
- uses: actions/download-artifact@v4
77+
with:
78+
# unpacks all CIBW artifacts into dist/
79+
pattern: cibw-*
80+
path: dist
81+
merge-multiple: true
82+
83+
- name: Publish to TestPyPI
84+
uses: pypa/gh-action-pypi-publish@release/v1
85+
with:
86+
attestations: false
87+
verbose: true
88+
# only for test pypi
89+
repository-url: https://test.pypi.org/legacy/
90+
91+
test_local_wheels:
92+
name: Test local wheels on ${{ matrix.os }}
93+
needs: [build_wheels]
94+
runs-on: ${{ matrix.os }}
95+
strategy:
96+
fail-fast: false # Continue testing all platforms even if one fails
97+
matrix:
98+
# Test only on key platforms for speed
99+
include:
100+
- os: ubuntu-latest
101+
name-suffix: "ubuntu"
102+
artifact-os: "ubuntu-24.04"
103+
- os: macos-15
104+
name-suffix: "macos-arm"
105+
artifact-os: "macos-15"
106+
steps:
107+
- name: Download wheels
108+
uses: actions/download-artifact@v4
109+
with:
110+
pattern: cibw-wheels-${{ matrix.artifact-os }}-*
111+
path: wheelhouse
112+
merge-multiple: true
113+
114+
- name: Set up Python
115+
uses: actions/setup-python@v5
116+
with:
117+
python-version: '3.12'
118+
119+
- name: Install local wheel and test
120+
run: |
121+
# Create and activate virtual environment
122+
python -m venv test_venv
123+
. test_venv/bin/activate
124+
python -m pip install --upgrade pip
125+
126+
# Get system info for wheel selection
127+
PYTHON_VERSION=$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")
128+
ARCH=$(uname -m)
129+
OS=$(uname -s)
130+
echo "Testing on Python $PYTHON_VERSION, OS: $OS, Architecture: $ARCH"
131+
132+
# Find compatible wheel with fallback chain
133+
wheel_file=$(
134+
find wheelhouse -name "*.whl" | {
135+
if [ "$OS" = "Darwin" ]; then
136+
# macOS: prefer architecture-specific, fallback to any macOS
137+
grep "cp${PYTHON_VERSION}.*macosx.*${ARCH}" | head -1 || \
138+
grep "cp${PYTHON_VERSION}.*macosx" | head -1
139+
else
140+
# Linux: prefer manylinux, avoid musllinux on glibc systems
141+
grep "cp${PYTHON_VERSION}.*manylinux" | head -1 || \
142+
grep -v musllinux | grep "cp${PYTHON_VERSION}" | head -1
143+
fi
144+
} || find wheelhouse -name "nest_simulator-*.whl" | head -1
145+
)
146+
147+
# Validate that the wheel file exists before installing
148+
if [ -z "$wheel_file" ] || [ ! -f "$wheel_file" ]; then
149+
echo "::error::No compatible wheel file found for this platform (Python $PYTHON_VERSION, OS: $OS, ARCH: $ARCH)."
150+
echo "Available wheels:"
151+
find wheelhouse -name "*.whl" || echo "None found"
152+
exit 1
153+
fi
154+
echo "Selected wheel: $(basename "$wheel_file")"
155+
156+
# Install wheel and dependencies
157+
python -m pip install "$wheel_file" numpy matplotlib cython
158+
159+
# Test import
160+
python -c "import nest; print('Local wheel test successful:', nest.help())"
161+
162+
test_uploaded_package:
163+
name: Test uploaded package on ${{ matrix.os }}
164+
needs: [upload_pypi]
165+
runs-on: ${{ matrix.os }}
166+
# TODO: Only run comprehensive tests on releases or manual dispatch
167+
# if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
168+
strategy:
169+
fail-fast: false
170+
matrix:
171+
include:
172+
- os: ubuntu-latest
173+
name-suffix: "ubuntu"
174+
- os: ubuntu-latest
175+
container: debian:latest
176+
name-suffix: "debian"
177+
# - os: ubuntu-latest
178+
# container: alpine:latest
179+
# name-suffix: "alpine"
180+
# - os: ubuntu-latest
181+
# container: rockylinux:8
182+
# name-suffix: "rocky8"
183+
- os: ubuntu-latest
184+
container: fedora:latest
185+
name-suffix: "fedora"
186+
- os: macos-15-intel
187+
name-suffix: "macos-intel"
188+
- os: macos-15
189+
name-suffix: "macos-arm"
190+
container: ${{ matrix.container }}
191+
steps:
192+
- name: Set up Python
193+
uses: actions/setup-python@v5
194+
with:
195+
python-version: '3.12'
196+
197+
- name: Cache pip packages
198+
uses: actions/cache@v4
199+
with:
200+
path: ~/.cache/pip
201+
key: ${{ runner.os }}-${{ matrix.name-suffix || 'default' }}-pip-test-${{ hashFiles('**/pyproject.toml') }}
202+
restore-keys: |
203+
${{ runner.os }}-${{ matrix.name-suffix || 'default' }}-pbuildsip-test-
204+
${{ runner.os }}-pip-
205+
206+
- name: Install system dependencies (Linux)
207+
if: runner.os == 'Linux'
208+
run: |
209+
# Check if we're in a container (root user) or native runner (needs sudo)
210+
SUDO_CMD=""
211+
if [ "$EUID" -ne 0 ] && command -v sudo >/dev/null 2>&1; then
212+
SUDO_CMD="sudo"
213+
fi
214+
215+
if command -v apt-get >/dev/null 2>&1; then
216+
$SUDO_CMD apt-get update
217+
$SUDO_CMD apt-get install -y python3 python3-pip openmpi-bin libopenmpi-dev
218+
elif command -v apk >/dev/null 2>&1; then
219+
$SUDO_CMD apk update
220+
$SUDO_CMD apk add python3 py3-pip openmpi openmpi-dev build-base gcompat
221+
$SUDO_CMD apk add gcc musl-dev python3-dev libffi-dev openssl-dev make
222+
elif command -v yum >/dev/null 2>&1; then
223+
$SUDO_CMD yum update -y
224+
$SUDO_CMD yum install -y python3 python3-pip openmpi-devel gcc-c++ make
225+
elif command -v dnf >/dev/null 2>&1; then
226+
$SUDO_CMD dnf update -y
227+
$SUDO_CMD dnf install -y python3 python3-pip openmpi-devel gcc-c++ make
228+
fi
229+
230+
- name: Install system dependencies (macOS)
231+
if: runner.os == 'macOS'
232+
run: |
233+
brew install open-mpi
234+
235+
- name: Check Alpine compatibility
236+
if: matrix.container == 'alpine:latest'
237+
run: |
238+
echo "Checking for musllinux wheel compatibility on Alpine Linux..."
239+
# Alpine uses musl libc, not glibc - we need musllinux wheels
240+
# Since we're installing from TestPyPI, we can't check local wheels
241+
# Instead, we'll let pip handle compatibility and catch glibc errors
242+
243+
- name: Install and test package and multithreading
244+
run: |
245+
# Create and activate virtual environment
246+
python -m venv test_venv
247+
. test_venv/bin/activate
248+
python -m pip install --upgrade pip
249+
250+
# For Alpine (musl), try to install and handle glibc incompatibility gracefully
251+
if [ "${{ matrix.container }}" = "alpine:latest" ]; then
252+
echo "Installing on Alpine Linux (musl libc)..."
253+
TMP_ERR=$(mktemp)
254+
# TODO: use pip install --no-binary :all: nest_simulator after pypi upload
255+
if ! python -m pip install --index-url https://test.pypi.org/simple/ \
256+
--extra-index-url https://pypi.org/simple nest-simulator 2> "$TMP_ERR"; then
257+
echo "::warning::Package installation failed on Alpine - likely due to glibc/musl incompatibility"
258+
echo "pip install error output:"
259+
cat "$TMP_ERR"
260+
echo "This is expected if only manylinux (glibc) wheels are available."
261+
echo "Skipping Alpine test as no musllinux wheels are available."
262+
exit 0
263+
fi
264+
else
265+
# TODO: use pip install nest_simulator after pypi upload
266+
python -m pip install --index-url https://test.pypi.org/simple/ \
267+
--extra-index-url https://pypi.org/simple nest-simulator
268+
fi
269+
python -c "import nest; print('NEST import successful:', nest.nest.help())"
270+
python -c "
271+
import nest
272+
import time
273+
import os
274+
275+
print('=== OpenMP Multithreading Test ===')
276+
277+
# Set available threads
278+
max_threads = min(os.cpu_count() or 1, 4) # Limit to 4 for CI
279+
nest.local_num_threads = max_threads
280+
281+
print(f'Available CPU cores: {os.cpu_count()}')
282+
print(f'NEST threads set to: {nest.local_num_threads}')
283+
print(f'OpenMP max threads: {os.environ.get(\"OMP_NUM_THREADS\", \"not set\")}')
284+
285+
# Simple network simulation to test threading
286+
nest.ResetKernel()
287+
288+
# Create neurons
289+
n_neurons = 1000
290+
neurons = nest.Create('iaf_psc_alpha', n_neurons)
291+
292+
# Create noise generator
293+
noise = nest.Create('poisson_generator', params={'rate': 1000.0})
294+
295+
# Connect with some probability
296+
nest.Connect(noise, neurons, {'rule': 'pairwise_bernoulli', 'p': 0.1})
297+
nest.Connect(neurons, neurons, {'rule': 'pairwise_bernoulli', 'p': 0.01})
298+
299+
# Record spikes
300+
spikedetector = nest.Create('spike_recorder')
301+
nest.Connect(neurons[:100], spikedetector) # Record from subset
302+
303+
print(f'Created network with {n_neurons} neurons using {nest.local_num_threads} thread(s)')
304+
305+
# Run simulation and measure time
306+
start_time = time.time()
307+
nest.Simulate(100.0) # 100ms simulation
308+
end_time = time.time()
309+
310+
# Get results and print summary
311+
events = spikedetector.events
312+
n_spikes = len(events['times'])
313+
sim_time = end_time - start_time
314+
315+
print(f'Simulation completed in {sim_time:.3f}s')
316+
print(f'Recorded {n_spikes} spikes')
317+
print(f'OpenMP multithreading test successful with {nest.local_num_threads} thread(s)!')
318+
"

‎.github/workflows/nestbuildmatrix.yml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -581,8 +581,6 @@ jobs:
581581
# available use flags (all default to "OFF"):
582582
# openmp, mpi, python, gsl, ltdl, boost, sionlib, libneurosim, optimize, warning, userdoc, music, readline
583583
use:
584-
- "optimize, warning"
585-
- "boost, optimize, warning"
586584
- "openmp, python, gsl, ltdl, boost, optimize, warning"
587585
- "mpi, python, gsl, ltdl, boost, optimize, warning"
588586
- "openmp, mpi, python, gsl, ltdl, boost, hdf5, sionlib, libneurosim, optimize, warning, music, detailed-timers, threaded-timers, mpi-sync-timer"
@@ -778,7 +776,6 @@ jobs:
778776
-Dwith-python=${{ contains(matrix.use, 'python') && 'ON' || 'OFF' }} \
779777
-Dwith-gsl=${{ contains(matrix.use, 'gsl') && 'ON' || 'OFF' }} \
780778
-Dwith-ltdl=${{ contains(matrix.use, 'ltdl') && 'ON' || 'OFF' }} \
781-
-Dwith-readline=${{ contains(matrix.use, 'readline') && 'ON' || 'OFF' }} \
782779
-Dwith-hdf5=${{ contains(matrix.use, 'hdf5') && 'ON' || 'OFF' }} \
783780
-Dwith-sionlib=${{ contains(matrix.use, 'sionlib') && '$HOME/.cache/sionlib.install' || 'OFF' }} \
784781
-Dwith-libneurosim=${{ contains(matrix.use, 'libneurosim') && '$HOME/.cache/libneurosim.install' || 'OFF' }} \
@@ -910,7 +907,6 @@ jobs:
910907
-Dwith-python=${{ contains(matrix.use, 'python') && 'ON' || 'OFF' }} \
911908
-Dwith-gsl=${{ contains(matrix.use, 'gsl') && 'ON' || 'OFF' }} \
912909
-Dwith-ltdl=${{ contains(matrix.use, 'ltdl') && 'ON' || 'OFF' }} \
913-
-Dwith-readline=${{ contains(matrix.use, 'readline') && 'ON' || 'OFF' }} \
914910
-Dwith-hdf5=${{ contains(matrix.use, 'hdf5') && 'ON' || 'OFF' }} \
915911
-Dwith-sionlib=${{ contains(matrix.use, 'sionlib') && '$HOME/.cache/sionlib.install' || 'OFF' }} \
916912
-Dwith-libneurosim=${{ contains(matrix.use, 'libneurosim') && '$HOME/.cache/libneurosim.install' || 'OFF' }} \

‎.pre-commit-config.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ repos:
3939
"testsuite/run_test.sh",
4040
"testsuite/do_tests.sh",
4141
"testsuite/junit_xml.sh",
42-
"examples/run_examples.sh",
43-
"examples/list_examples.sh",
42+
"pynest/examples/run_examples.sh",
43+
"pynest/examples/list_examples.sh",
4444
]
4545

4646
- repo: https://github.com/pre-commit/mirrors-clang-format

‎.pylintrc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
options = unneeded-not, line-too-long, unnecessary-semicolon, trailing-whitespace, missing-final-newline, bad-indentation, multiple-statements, bare-except
77
ignore = CVS .git conda env __pycache__ .pytest_cache .mypy_cache
88

9-
disable = no-member, redefined-outer-name, invalid-name, consider-using-f-string, wrong-import-order, missing-function-docstring, missing-method-docstring, missing-class-docstring, attribute-defined-outside-init, no-else-return, cell-var-from-loop, import-error, pointless-string-statement, unused-import, redefined-builtin, superfluous-parens, unused-variable, too-many-locals, consider-using-from-import, consider-using-enumerate, no-name-in-module, too-many-arguments, too-many-instance-attributes, import-outside-toplevel, too-few-public-methods, cyclic-import, missing-module-docstring, unidiomatic-typecheck, dangerous-default-value, unused-argument, use-dict-literal, exec-used, no-self-use, too-many-statements, ungrouped-imports, consider-using-sys-exit, too-many-statements, redundant-u-string-prefix, protected-access, consider-using-dict-comprehension, no-else-raise, too-many-nested-blocks, use-a-generator, reimported, undefined-variable, too-many-branches, raise-missing-from, trailing-comma-tuple, unspecified-encoding, consider-using-with, f-string-without-interpolation, broad-except, unnecessary-pass, global-statement, too-many-lines, consider-merging-isinstance, redefined-argument-from-local, global-variable-undefined, use-implicit-booleaness-not-len, inconsistent-return-statements, consider-using-in, inconsistent-return-statements, keyword-arg-before-vararg, consider-using-dict-items, import-self, fixme, c-extension-no-member, too-many-public-methods, consider-iterating-dictionary, consider-using-max-builtin, super-with-arguments, expression-not-assigned, unnecessary-comprehension, no-self-argument, chained-comparison, undefined-loop-variable, empty-docstring, use-maxsplit-arg, pointless-statement, wrong-import-position, redundant-unittest-assert, eval-used, not-callable, invalid-unary-operand-type, consider-using-generator, R0801, unnecessary-dunder-call, logging-fstring-interpolation, consider-using-get, useless-object-inheritance, unrecognized-option, unknown-option-value, useless-option-value
9+
disable = no-member, redefined-outer-name, invalid-name, consider-using-f-string, wrong-import-order, missing-function-docstring, missing-method-docstring, missing-class-docstring, attribute-defined-outside-init, no-else-return, cell-var-from-loop, import-error, pointless-string-statement, unused-import, redefined-builtin, superfluous-parens, unused-variable, too-many-locals, consider-using-from-import, consider-using-enumerate, no-name-in-module, too-many-arguments, too-many-instance-attributes, too-many-return-statements, import-outside-toplevel, too-few-public-methods, cyclic-import, missing-module-docstring, unidiomatic-typecheck, dangerous-default-value, unused-argument, use-dict-literal, exec-used, no-self-use, too-many-statements, ungrouped-imports, consider-using-sys-exit, too-many-statements, redundant-u-string-prefix, protected-access, consider-using-dict-comprehension, no-else-raise, too-many-nested-blocks, use-a-generator, reimported, undefined-variable, too-many-branches, raise-missing-from, trailing-comma-tuple, unspecified-encoding, consider-using-with, f-string-without-interpolation, broad-except, unnecessary-pass, global-statement, too-many-lines, consider-merging-isinstance, redefined-argument-from-local, global-variable-undefined, use-implicit-booleaness-not-len, inconsistent-return-statements, consider-using-in, inconsistent-return-statements, keyword-arg-before-vararg, consider-using-dict-items, import-self, fixme, c-extension-no-member, too-many-public-methods, consider-iterating-dictionary, consider-using-max-builtin, super-with-arguments, expression-not-assigned, unnecessary-comprehension, no-self-argument, chained-comparison, undefined-loop-variable, empty-docstring, use-maxsplit-arg, pointless-statement, wrong-import-position, redundant-unittest-assert, eval-used, not-callable, invalid-unary-operand-type, consider-using-generator, R0801, unnecessary-dunder-call, logging-fstring-interpolation, consider-using-get, useless-object-inheritance, unrecognized-option, unknown-option-value, useless-option-value
1010

1111
const-naming-style = snake_case
1212
method-naming-style = PascalCase

0 commit comments

Comments
 (0)