From b55cc0531d4344e43b2c79de76a901210db144b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Danie=CC=88l=20de=20Kok?= Date: Fri, 27 May 2022 09:20:18 +0200 Subject: [PATCH 1/3] test_ops: do not lower precision in conversion to Torch tensor float64 test values close to zero were rounded by conversion to a float32 Torch tensor, resuling in mismatches between Thinc and Torch gradients. This change prevents the loss in precision. --- thinc/tests/backends/test_ops.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/thinc/tests/backends/test_ops.py b/thinc/tests/backends/test_ops.py index 5d9f17524..06b743968 100644 --- a/thinc/tests/backends/test_ops.py +++ b/thinc/tests/backends/test_ops.py @@ -1259,16 +1259,13 @@ def test_ngrams(): def test_compare_activations_to_torch(ops, dtype, x, torch_func): import torch - def cast_torch(scalar: float): - return torch.tensor([scalar], requires_grad=True) - func_name, pytorch_func = torch_func forward = getattr(ops, func_name) backward = getattr(ops, "backprop_" + func_name) # The tolerance of isclose is set to 1e-06 instead of # the default 1e-08 due to the GELU x_thinc = ops.asarray([x], dtype=dtype) - x_torch = cast_torch(x) + x_torch = xp2torch(x_thinc, requires_grad=True) y = pytorch_func(x_torch) y_thinc = forward(x_thinc) y.backward() From 400cc2156fa1890334f45e0f2db59fe577b9199c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Danie=CC=88l=20de=20Kok?= Date: Fri, 27 May 2022 10:30:42 +0200 Subject: [PATCH 2/3] test_ops: compare arrays on same device in Torch comparison --- thinc/tests/backends/test_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thinc/tests/backends/test_ops.py b/thinc/tests/backends/test_ops.py index 06b743968..3d5531d89 100644 --- a/thinc/tests/backends/test_ops.py +++ b/thinc/tests/backends/test_ops.py @@ -1271,7 +1271,7 @@ def test_compare_activations_to_torch(ops, dtype, x, torch_func): y.backward() assert x_thinc.dtype == y_thinc.dtype assert ops.xp.isclose(y_thinc, forward(x_thinc, inplace=True), atol=1e-06) - assert ops.xp.isclose(y_thinc, y.detach().numpy(), atol=1e-06) + assert ops.xp.isclose(y_thinc, y.detach(), atol=1e-06) x_thinc = ops.asarray([x], dtype=dtype) dY_thinc = ops.asarray([1.0], dtype=dtype) dY_thinc_inplace = dY_thinc.copy() From 88b17bb70d4103b75c53d63e46e45d5c15c26285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Danie=CC=88l=20de=20Kok?= Date: Fri, 27 May 2022 10:31:07 +0200 Subject: [PATCH 3/3] test_maxout: compare arrays with same precision --- thinc/tests/backends/test_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thinc/tests/backends/test_ops.py b/thinc/tests/backends/test_ops.py index 3d5531d89..c8cb6a9df 100644 --- a/thinc/tests/backends/test_ops.py +++ b/thinc/tests/backends/test_ops.py @@ -216,7 +216,7 @@ def test_seq2col_window_one_small(ops): @given(X=strategies.arrays_BOP()) def test_maxout(ops, dtype, X): X = ops.asarray(X, dtype=dtype) - expected_best = X.max(axis=-1) + expected_best = X.max(axis=-1).astype(dtype) predicted_best, which = ops.maxout(X) assert predicted_best.dtype == dtype ops.xp.testing.assert_allclose(