diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 85a21b930..4a80f5e55 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -130,6 +130,8 @@ repos: - pytest_codspeed - Sphinx >= 5.3.0 - sphinxcontrib-spelling + - types-psutil + - psutil args: - --python-version=3.13 - --txt-report=.tox/.tmp/.mypy/python-3.13 @@ -146,6 +148,8 @@ repos: - pytest_codspeed - Sphinx >= 5.3.0 - sphinxcontrib-spelling + - types-psutil + - psutil args: - --python-version=3.11 - --txt-report=.tox/.tmp/.mypy/python-3.11 diff --git a/CHANGES/1233.bugfix.rst b/CHANGES/1233.bugfix.rst new file mode 100644 index 000000000..e5a659027 --- /dev/null +++ b/CHANGES/1233.bugfix.rst @@ -0,0 +1,2 @@ +Fix ``MutliDict`` & ``CIMultiDict`` memory leak when deleting values or clearing them +-- by :user:`Vizonex` diff --git a/CHANGES/1233.contrib.rst b/CHANGES/1233.contrib.rst new file mode 100644 index 000000000..ff5f12226 --- /dev/null +++ b/CHANGES/1233.contrib.rst @@ -0,0 +1,2 @@ +Added memory leak test for popping or deleting attributes from a multidict to prevent future issues or bogus claims. +-- by :user:`Vizonex` diff --git a/multidict/_multilib/hashtable.h b/multidict/_multilib/hashtable.h index 16b82ded1..82019f9eb 100644 --- a/multidict/_multilib/hashtable.h +++ b/multidict/_multilib/hashtable.h @@ -1877,7 +1877,7 @@ md_traverse(MultiDictObject *md, visitproc visit, void *arg) static inline int md_clear(MultiDictObject *md) { - if (md->used == 0) { + if (md->keys == NULL || md->keys == &empty_htkeys) { return 0; } md->version = NEXT_VERSION(md->state); diff --git a/pytest.ini b/pytest.ini index f5c094b96..d2e0ed8b6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -41,7 +41,7 @@ doctest_optionflags = ALLOW_UNICODE ELLIPSIS # Marks tests with an empty parameterset as xfail(run=False) empty_parameter_set_mark = xfail -faulthandler_timeout = 60 +faulthandler_timeout = 90 filterwarnings = error diff --git a/requirements/pytest.txt b/requirements/pytest.txt index fd61e5d7e..3671cab9a 100644 --- a/requirements/pytest.txt +++ b/requirements/pytest.txt @@ -2,3 +2,4 @@ objgraph==3.6.2 pytest==8.4.0 pytest-codspeed==3.2.0 pytest-cov==6.1.0 +psutil==7.0.0 \ No newline at end of file diff --git a/tests/isolated/multidict_pop.py b/tests/isolated/multidict_pop.py new file mode 100644 index 000000000..daced359e --- /dev/null +++ b/tests/isolated/multidict_pop.py @@ -0,0 +1,80 @@ +# Test for memory leaks surrounding deletion of values or +# bad cleanups. +# SEE: https://github.com/aio-libs/multidict/issues/1232 +# We want to make sure that bad predictions or bougus claims +# of memory leaks can be prevented in the future. + +import gc +import psutil +import os +from multidict import MultiDict + + +def trim_ram() -> None: + """Forces python garbage collection.""" + gc.collect() + + +process = psutil.Process(os.getpid()) + + +def get_memory_usage() -> int: + memory_info = process.memory_info() + return memory_info.rss / (1024 * 1024) # type: ignore[no-any-return] + + +keys = [f"X-Any-{i}" for i in range(100)] +headers = {key: key * 2 for key in keys} + + +def check_for_leak() -> None: + trim_ram() + usage = get_memory_usage() + assert usage < 50, f"Memory leaked at: {usage} MB" + + +def _test_pop() -> None: + for _ in range(10): + for _ in range(100): + result = MultiDict(headers) + for k in keys: + result.pop(k) + check_for_leak() + + +def _test_popall() -> None: + for _ in range(10): + for _ in range(100): + result = MultiDict(headers) + for k in keys: + result.popall(k) + check_for_leak() + + +def _test_popone() -> None: + for _ in range(10): + for _ in range(100): + result = MultiDict(headers) + for k in keys: + result.popone(k) + check_for_leak() + + +def _test_del() -> None: + for _ in range(10): + for _ in range(100): + result = MultiDict(headers) + for k in keys: + del result[k] + check_for_leak() + + +def _run_isolated_case() -> None: + _test_pop() + _test_popall() + _test_popone() + _test_del() + + +if __name__ == "__main__": + _run_isolated_case() diff --git a/tests/test_leaks.py b/tests/test_leaks.py index ded7cf065..56126d4bc 100644 --- a/tests/test_leaks.py +++ b/tests/test_leaks.py @@ -15,6 +15,7 @@ "multidict_extend_multidict.py", "multidict_extend_tuple.py", "multidict_update_multidict.py", + "multidict_pop.py", ), ) @pytest.mark.leaks