Skip to content

Permit fast percentile with mdtol=0 #4755

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rcomer merged 9 commits into from
May 23, 2022
Merged

Permit fast percentile with mdtol=0 #4755

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
edd551c
Permit fast percentile with mdtol=0
May 19, 2022
7865c2c
What's new
May 19, 2022
376f8df
Check arrays like they're masked, and make the expected arrays masked
May 20, 2022
f409f44
Bigger arrays to avoid np quirk, bug fix, check masking in 2d_multi
May 20, 2022
1a0a051
Suppress numpy masked percentile warning
May 20, 2022
3471908
Use a larger test array that numpy handles how it handles most of the…
May 23, 2022
2a42c18
Broadcast the mask in the non-lazy case with added dim
May 23, 2022
f0a6a7b
Bug fix on ndim test
May 23, 2022
7af47a1
Review fixes
May 23, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion lib/iris/analysis/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
from collections.abc import Iterable
import functools
from functools import wraps
import warnings

import dask.array as da
import numpy as np
Expand Down Expand Up @@ -592,6 +593,12 @@ def aggregate(self, data, axis, **kwargs):
):
fraction_not_missing = data.count(axis=axis) / data.shape[axis]
mask_update = 1 - mdtol > fraction_not_missing
if np.array(result).ndim > np.array(mask_update).ndim:
# call_func created trailing dimension.
mask_update = np.broadcast_to(
mask_update.reshape(mask_update.shape + (1,)),
np.array(result).shape,
)
if ma.isMaskedArray(result):
result.mask = result.mask | mask_update
else:
Expand Down Expand Up @@ -1297,7 +1304,12 @@ def _calc_percentile(data, percent, fast_percentile_method=False, **kwargs):
)
if ma.is_masked(data):
raise TypeError(msg)
result = np.percentile(data, percent, axis=-1)
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore",
"Warning: 'partition' will ignore the 'mask' of the MaskedArray.",
)
result = np.percentile(data, percent, axis=-1)
result = result.T
else:
quantiles = percent / 100.0
Expand Down
31 changes: 19 additions & 12 deletions lib/iris/tests/unit/analysis/test_PERCENTILE.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def check_percentile_calc(
if self.lazy:
data = as_lazy_data(data)

expected = np.array(expected)
expected = ma.array(expected)

actual = self.agg_method(
data,
Expand All @@ -52,9 +52,9 @@ def check_percentile_calc(
self.assertFalse(is_lazy)

if approx:
self.assertArrayAlmostEqual(actual, expected)
self.assertMaskedArrayAlmostEqual(actual, expected)
else:
self.assertArrayEqual(actual, expected)
self.assertMaskedArrayEqual(actual, expected)

def test_1d_single(self):
data = np.arange(11)
Expand Down Expand Up @@ -131,7 +131,7 @@ def test_masked_2d_single(self):
def test_masked_2d_multi(self):
shape = (3, 10)
data = ma.arange(np.prod(shape)).reshape(shape)
data[1] = ma.masked
data[1, ::2] = ma.masked
percent = np.array([10, 50, 70, 80])
axis = 0
mdtol = 0.1
Expand All @@ -140,10 +140,11 @@ def test_masked_2d_multi(self):
# linear interpolation.
expected = percent / 100 * 20
# Other columns are first column plus column number.
expected = (
expected = ma.array(
np.broadcast_to(expected, (shape[-1], percent.size))
+ np.arange(shape[-1])[:, np.newaxis]
)
expected[::2] = ma.masked

self.check_percentile_calc(
data, axis, percent, expected, mdtol=mdtol, approx=True
Expand Down Expand Up @@ -205,7 +206,9 @@ def setUp(self):
self.agg_method = PERCENTILE.aggregate

def test_masked(self):
shape = (2, 11)
# Using (3,11) because np.percentile returns a masked array anyway with
# (2, 11)
shape = (3, 11)
data = ma.arange(np.prod(shape)).reshape(shape)
data[0, ::2] = ma.masked
emsg = (
Expand All @@ -218,13 +221,15 @@ def test_masked(self):
)

def test_masked_mdtol_0(self):
shape = (2, 11)
# Using (3,11) because np.percentile returns a masked array anyway with
# (2, 11)
shape = (3, 11)
axis = 0
percent = 50
data = ma.arange(np.prod(shape)).reshape(shape)
data[0, ::2] = ma.masked
expected = np.arange(shape[-1]) + 5.5
expected[0] = ma.masked
expected = ma.arange(shape[-1]) + 11
expected[::2] = ma.masked
self.check_percentile_calc(data, axis, percent, expected, mdtol=0)

@mock.patch("numpy.percentile")
Expand Down Expand Up @@ -292,6 +297,8 @@ def setUp(self):
self.agg_method = PERCENTILE.lazy_aggregate

def test_masked(self):
# Using (3,11) because np.percentile returns a masked array anyway with
# (2, 11)
shape = (2, 11)
data = ma.arange(np.prod(shape)).reshape(shape)
data[0, ::2] = ma.masked
Expand All @@ -307,14 +314,14 @@ def test_masked(self):
as_concrete_data(actual)

def test_masked_mdtol_0(self):
shape = (2, 11)
shape = (3, 11)
axis = 0
percent = 50
data = ma.arange(np.prod(shape)).reshape(shape)
data[0, ::2] = ma.masked
data = as_lazy_data(data)
expected = np.arange(shape[-1]) + 5.5
expected[0] = ma.masked
expected = ma.arange(shape[-1]) + 11
expected[::2] = ma.masked
self.check_percentile_calc(data, axis, percent, expected, mdtol=0)

@mock.patch("numpy.percentile", return_value=np.array([2, 4]))
Expand Down