ensure warnings cannot become errors in assert_ (#4864)

* ensure warnings cannot become errors in assert_

* also assert_duckarray_allclose

* add whats new

* ensure warnings are raised

* Update doc/whats-new.rst

* use a decorator

Author: mathause

doc/whats-new.rst

@@ -130,6 +130,8 @@ Internal Changes
   in ipython (:issue:`4741`, :pull:`4742`). By `Richard Kleijn <https://github.com/rhkleijn>`_.
 - Added the ``set_close`` method to ``Dataset`` and ``DataArray`` for beckends to specify how to voluntary release
   all resources. (:pull:`#4809`), By `Alessandro Amici <https://github.com/alexamici>`_.
+- Ensure warnings cannot be turned into exceptions in :py:func:`testing.assert_equal` and
+  the other ``assert_*`` functions (:pull:`4864`). By `Mathias Hauser <https://github.com/mathause>`_.
 
 .. _whats-new.0.16.2:
 
@@ -146,7 +148,7 @@ Deprecations
 
 - :py:attr:`~core.accessor_dt.DatetimeAccessor.weekofyear` and :py:attr:`~core.accessor_dt.DatetimeAccessor.week`
   have been deprecated. Use ``DataArray.dt.isocalendar().week``
-  instead (:pull:`4534`). By `Mathias Hauser <https://github.com/mathause>`_,
+  instead (:pull:`4534`). By `Mathias Hauser <https://github.com/mathause>`_.
   `Maximilian Roos <https://github.com/max-sixty>`_, and `Spencer Clark <https://github.com/spencerkclark>`_.
 - :py:attr:`DataArray.rolling` and :py:attr:`Dataset.rolling` no longer support passing ``keep_attrs``
   via its constructor. Pass ``keep_attrs`` via the applied function, i.e. use

xarray/testing.py

@@ -1,5 +1,6 @@
 """Testing functions exposed to the user API"""
 import functools
+import warnings
 from typing import Hashable, Set, Union
 
 import numpy as np
@@ -21,6 +22,19 @@
 )
 
 
+def ensure_warnings(func):
+    # sometimes tests elevate warnings to errors
+    # -> make sure that does not happen in the assert_* functions
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+
+            return func(*args, **kwargs)
+
+    return wrapper
+
+
 def _decode_string_data(data):
     if data.dtype.kind == "S":
         return np.core.defchararray.decode(data, "utf-8", "replace")
@@ -38,6 +52,7 @@ def _data_allclose_or_equiv(arr1, arr2, rtol=1e-05, atol=1e-08, decode_bytes=Tru
         return duck_array_ops.allclose_or_equiv(arr1, arr2, rtol=rtol, atol=atol)
 
 
+@ensure_warnings
 def assert_equal(a, b):
     """Like :py:func:`numpy.testing.assert_array_equal`, but for xarray
     objects.
@@ -69,6 +84,7 @@ def assert_equal(a, b):
         raise TypeError("{} not supported by assertion comparison".format(type(a)))
 
 
+@ensure_warnings
 def assert_identical(a, b):
     """Like :py:func:`xarray.testing.assert_equal`, but also matches the
     objects' names and attributes.
@@ -99,6 +115,7 @@ def assert_identical(a, b):
         raise TypeError("{} not supported by assertion comparison".format(type(a)))
 
 
+@ensure_warnings
 def assert_allclose(a, b, rtol=1e-05, atol=1e-08, decode_bytes=True):
     """Like :py:func:`numpy.testing.assert_allclose`, but for xarray objects.
 
@@ -182,6 +199,7 @@ def _format_message(x, y, err_msg, verbose):
     return "\n".join(parts)
 
 
+@ensure_warnings
 def assert_duckarray_allclose(
     actual, desired, rtol=1e-07, atol=0, err_msg="", verbose=True
 ):
@@ -192,6 +210,7 @@ def assert_duckarray_allclose(
     assert allclose, _format_message(actual, desired, err_msg=err_msg, verbose=verbose)
 
 
+@ensure_warnings
 def assert_duckarray_equal(x, y, err_msg="", verbose=True):
     """ Like `np.testing.assert_array_equal`, but for duckarrays """
     __tracebackhide__ = True

xarray/tests/test_testing.py

@@ -1,3 +1,5 @@
+import warnings
+
 import numpy as np
 import pytest
 
@@ -127,3 +129,41 @@ def test_assert_duckarray_equal(duckarray, obj1, obj2):
     b = duckarray(obj2)
 
     xr.testing.assert_duckarray_equal(a, b)
+
+
+@pytest.mark.parametrize(
+    "func",
+    [
+        "assert_equal",
+        "assert_identical",
+        "assert_allclose",
+        "assert_duckarray_equal",
+        "assert_duckarray_allclose",
+    ],
+)
+def test_ensure_warnings_not_elevated(func):
+    # make sure warnings are not elevated to errors in the assertion functions
+    # e.g. by @pytest.mark.filterwarnings("error")
+    # see https://github.com/pydata/xarray/pull/4760#issuecomment-774101639
+
+    # define a custom Variable class that raises a warning in assert_*
+    class WarningVariable(xr.Variable):
+        @property  # type: ignore
+        def dims(self):
+            warnings.warn("warning in test")
+            return super().dims
+
+        def __array__(self):
+            warnings.warn("warning in test")
+            return super().__array__()
+
+    a = WarningVariable("x", [1])
+    b = WarningVariable("x", [2])
+
+    with warnings.catch_warnings(record=True) as w:
+        # elevate warnings to errors
+        warnings.filterwarnings("error")
+        with pytest.raises(AssertionError):
+            getattr(xr.testing, func)(a, b)
+
+        assert len(w) > 0