Fix Memory leaks and add tests to prevent memory leaks during md_clear from passing

.pre-commit-config.yaml

@@ -130,6 +130,8 @@ repos:
     - pytest_codspeed
     - Sphinx >= 5.3.0
     - sphinxcontrib-spelling
+    - types-psutil
+    - psutil
     args:
     - --python-version=3.13
     - --txt-report=.tox/.tmp/.mypy/python-3.13
@@ -146,6 +148,8 @@ repos:
     - pytest_codspeed
     - Sphinx >= 5.3.0
     - sphinxcontrib-spelling
+    - types-psutil
+    - psutil
     args:
     - --python-version=3.11
     - --txt-report=.tox/.tmp/.mypy/python-3.11

CHANGES/1233.bugfix.rst

@@ -0,0 +1,2 @@
+Fix ``MutliDict`` & ``CIMultiDict`` memory leak when deleting values or clearing them 
+-- by :user:`Vizonex`

CHANGES/1233.contrib.rst

@@ -0,0 +1,2 @@
+Added memory leak test for popping or deleting attributes from a multidict to prevent future issues or bogus claims.
+-- by :user:`Vizonex`

multidict/_multidict.c

@@ -674,7 +674,7 @@ multidict_popone(MultiDictObject *self, PyObject *const *args,
     if (md_pop_one(self, key, &ret_val) < 0) {
         return NULL;
     }
-
+    
     ASSERT_CONSISTENT(self, false);
     if (ret_val == NULL) {
         if (_default != NULL) {

multidict/_multilib/hashtable.h

@@ -1877,21 +1877,20 @@ md_traverse(MultiDictObject *md, visitproc visit, void *arg)
 static inline int
 md_clear(MultiDictObject *md)
 {
-    if (md->used == 0) {
+    if ((md->keys == &empty_htkeys) || (md->state == NULL)) {
         return 0;
     }
     md->version = NEXT_VERSION(md->state);
-
+    
     entry_t *entries = htkeys_entries(md->keys);
     for (Py_ssize_t pos = 0; pos < md->keys->nentries; pos++) {
         entry_t *entry = entries + pos;
-        if (entry->identity != NULL) {
-            Py_CLEAR(entry->identity);
-            Py_CLEAR(entry->key);
-            Py_CLEAR(entry->value);
-        }
+        // Py_CLEAR has null checks of it's own making it easier to free.
+        Py_CLEAR(entry->identity);
+        Py_CLEAR(entry->key);
+        Py_CLEAR(entry->value);
     }
-
+    
     md->used = 0;
     if (md->keys != &empty_htkeys) {
         htkeys_free(md->keys);
@@ -1910,7 +1909,7 @@ _md_check_consistency(MultiDictObject *md, bool update)
 
 #define CHECK(expr) assert(expr)
     //    do { if (!(expr)) { assert(0 && Py_STRINGIFY(expr)); } } while (0)
-
+    
     htkeys_t *keys = md->keys;
     CHECK(keys != NULL);
     Py_ssize_t calc_usable = USABLE_FRACTION(htkeys_nslots(keys));
@@ -1929,7 +1928,7 @@ _md_check_consistency(MultiDictObject *md, bool update)
         Py_ssize_t ix = htkeys_get_index(keys, i);
         CHECK(DKIX_DUMMY <= ix && ix <= calc_usable);
     }
-
+    
     entry_t *entries = htkeys_entries(keys);
     for (Py_ssize_t i = 0; i < calc_usable; i++) {
         entry_t *entry = &entries[i];

pytest.ini

@@ -41,7 +41,7 @@ doctest_optionflags = ALLOW_UNICODE ELLIPSIS
 # Marks tests with an empty parameterset as xfail(run=False)
 empty_parameter_set_mark = xfail
 
-faulthandler_timeout = 60
+faulthandler_timeout = 90
 
 filterwarnings =
   error

requirements/pytest.txt

@@ -2,3 +2,4 @@ objgraph==3.6.2
 pytest==8.4.0
 pytest-codspeed==3.2.0
 pytest-cov==6.1.0
+psutil==7.0.0

tests/isolated/multidict_pop.py

@@ -0,0 +1,80 @@
+# Test for memory leaks surrounding deletion of values or
+# bad cleanups.
+# SEE: https://github.com/aio-libs/multidict/issues/1232
+# We want to make sure that bad predictions or bougus claims
+# of memory leaks can be prevented in the future.
+
+import gc
+import psutil
+import os
+from multidict import MultiDict
+
+
+def trim_ram() -> None:
+    """Forces python garbage collection."""
+    gc.collect()
+
+
+process = psutil.Process(os.getpid())
+
+
+def get_memory_usage() -> int:
+    memory_info = process.memory_info()
+    return memory_info.rss / (1024 * 1024)  # type: ignore[no-any-return]
+
+
+keys = [f"X-Any-{i}" for i in range(100)]
+headers = {key: key * 2 for key in keys}
+
+
+def check_for_leak() -> None:
+    trim_ram()
+    usage = get_memory_usage()
+    assert usage < 50, f"Memory leaked at: {usage} MB"
+
+
+def _test_pop() -> None:
+    for _ in range(10):
+        for _ in range(100):
+            result = MultiDict(headers)
+            for k in keys:
+                result.pop(k)
+        check_for_leak()
+
+
+def _test_popall() -> None:
+    for _ in range(10):
+        for _ in range(100):
+            result = MultiDict(headers)
+            for k in keys:
+                result.popall(k)
+        check_for_leak()
+
+
+def _test_popone() -> None:
+    for _ in range(10):
+        for _ in range(100):
+            result = MultiDict(headers)
+            for k in keys:
+                result.popone(k)
+        check_for_leak()
+
+
+def _test_del() -> None:
+    for _ in range(10):
+        for _ in range(100):
+            result = MultiDict(headers)
+            for k in keys:
+                del result[k]
+        check_for_leak()
+
+
+def _run_isolated_case() -> None:
+    _test_pop()
+    _test_popall()
+    _test_popone()
+    _test_del()
+
+
+if __name__ == "__main__":
+    _run_isolated_case()

tests/test_leaks.py

@@ -15,6 +15,7 @@
         "multidict_extend_multidict.py",
         "multidict_extend_tuple.py",
         "multidict_update_multidict.py",
+        "multidict_pop.py",
     ),
 )
 @pytest.mark.leaks