From 7afae6cf64714348b27d7a91c26fd7252c6208e8 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Mon, 8 Aug 2022 11:11:53 +0200 Subject: [PATCH 1/3] Introduce error margin by variable in tests --- openfisca_core/tools/test_runner.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/openfisca_core/tools/test_runner.py b/openfisca_core/tools/test_runner.py index c351965a62..3acacf1f3e 100644 --- a/openfisca_core/tools/test_runner.py +++ b/openfisca_core/tools/test_runner.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import collections import warnings import sys import os @@ -214,14 +215,28 @@ def check_variable(self, variable_name: str, expected_value, period, entity_inde actual_value = self.simulation.calculate(variable_name, period) + absolute_error_margin = self.test.get('absolute_error_margin') + variable_absolute_error_margin = ( + absolute_error_margin.get(variable_name, absolute_error_margin.get("default")) + if isinstance(absolute_error_margin, collections.Mapping) + else absolute_error_margin + ) + + relative_error_margin = self.test.get('relative_error_margin') + variable_relative_error_margin = ( + relative_error_margin.get(variable_name, relative_error_margin.get("default")) + if isinstance(relative_error_margin, collections.Mapping) + else relative_error_margin + ) + if entity_index is not None: actual_value = actual_value[entity_index] return assert_near( actual_value, expected_value, - absolute_error_margin = self.test.get('absolute_error_margin'), + absolute_error_margin = variable_absolute_error_margin, message = f"{variable_name}@{period}: ", - relative_error_margin = self.test.get('relative_error_margin'), + relative_error_margin = variable_relative_error_margin, ) def should_ignore_variable(self, variable_name: str): From d2dfa884c060839827930cb93cf2104a2352ab34 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Wed, 10 Aug 2022 12:58:26 +0200 Subject: [PATCH 2/3] Add tests --- tests/core/test_yaml.py | 2 ++ .../failing_test_variable_absolute_error_margin.yaml | 9 +++++++++ .../failing_test_variable_relative_error_margin.yaml | 9 +++++++++ .../yaml_tests/test_absolute_error_margin.yaml | 11 +++++++++++ .../yaml_tests/test_relative_error_margin.yaml | 11 +++++++++++ 5 files changed, 42 insertions(+) create mode 100644 tests/fixtures/yaml_tests/failing_test_variable_absolute_error_margin.yaml create mode 100644 tests/fixtures/yaml_tests/failing_test_variable_relative_error_margin.yaml diff --git a/tests/core/test_yaml.py b/tests/core/test_yaml.py index f63e37ff39..36ac8949c8 100644 --- a/tests/core/test_yaml.py +++ b/tests/core/test_yaml.py @@ -37,6 +37,7 @@ def test_relative_error_margin_success(tax_benefit_system): def test_relative_error_margin_fail(tax_benefit_system): assert run_yaml_test(tax_benefit_system, 'failing_test_relative_error_margin.yaml') == EXIT_TESTSFAILED + assert run_yaml_test(tax_benefit_system, 'failing_test_variable_relative_error_margin.yaml') == EXIT_TESTSFAILED def test_absolute_error_margin_success(tax_benefit_system): @@ -45,6 +46,7 @@ def test_absolute_error_margin_success(tax_benefit_system): def test_absolute_error_margin_fail(tax_benefit_system): assert run_yaml_test(tax_benefit_system, 'failing_test_absolute_error_margin.yaml') == EXIT_TESTSFAILED + assert run_yaml_test(tax_benefit_system, 'failing_test_variable_absolute_error_margin.yaml') == EXIT_TESTSFAILED def test_run_tests_from_directory(tax_benefit_system): diff --git a/tests/fixtures/yaml_tests/failing_test_variable_absolute_error_margin.yaml b/tests/fixtures/yaml_tests/failing_test_variable_absolute_error_margin.yaml new file mode 100644 index 0000000000..af708e8e94 --- /dev/null +++ b/tests/fixtures/yaml_tests/failing_test_variable_absolute_error_margin.yaml @@ -0,0 +1,9 @@ +- name: "Failing test: result out of variable specific absolute error margin" + period: 2015-01 + absolute_error_margin: + default: 100 + income_tax: 50 + input: + salary: 2000 + output: + income_tax: 351 # 300 diff --git a/tests/fixtures/yaml_tests/failing_test_variable_relative_error_margin.yaml b/tests/fixtures/yaml_tests/failing_test_variable_relative_error_margin.yaml new file mode 100644 index 0000000000..1f1559a829 --- /dev/null +++ b/tests/fixtures/yaml_tests/failing_test_variable_relative_error_margin.yaml @@ -0,0 +1,9 @@ +- name: "Failing test: result out of variable specific relative error margin" + period: 2015-01 + relative_error_margin: + default: 1 + income_tax: 0.05 + input: + salary: 2000 + output: + income_tax: 316 # 300 diff --git a/tests/fixtures/yaml_tests/test_absolute_error_margin.yaml b/tests/fixtures/yaml_tests/test_absolute_error_margin.yaml index be7de2d5cb..65dbb308e3 100644 --- a/tests/fixtures/yaml_tests/test_absolute_error_margin.yaml +++ b/tests/fixtures/yaml_tests/test_absolute_error_margin.yaml @@ -5,3 +5,14 @@ salary: 2000 output: income_tax: 350 # 300 + + +- name: "Result within absolute error margin" + period: 2015-01 + absolute_error_margin: + default: 100 + income_tax: 50 + input: + salary: 2000 + output: + income_tax: 350 # 300 diff --git a/tests/fixtures/yaml_tests/test_relative_error_margin.yaml b/tests/fixtures/yaml_tests/test_relative_error_margin.yaml index 7845d6f361..d39a9e4143 100644 --- a/tests/fixtures/yaml_tests/test_relative_error_margin.yaml +++ b/tests/fixtures/yaml_tests/test_relative_error_margin.yaml @@ -5,3 +5,14 @@ salary: 2000 output: income_tax: 290 # 300 + + +- name: "Result within variable relative error margin" + period: 2015-01 + relative_error_margin: + default: .001 + income_tax: 0.05 + input: + salary: 2000 + output: + income_tax: 290 # 300 From ed98b3619f672c28c31cbaadca849bb1eee00770 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Fri, 12 Aug 2022 20:04:21 +0200 Subject: [PATCH 3/3] Bump version --- CHANGELOG.md | 6 +++ openfisca_core/tools/test_runner.py | 38 ++++++++++--------- setup.py | 2 +- tests/core/test_yaml.py | 2 - .../failing_test_absolute_error_margin.yaml | 11 ++++++ .../failing_test_relative_error_margin.yaml | 11 ++++++ ...g_test_variable_absolute_error_margin.yaml | 9 ----- ...g_test_variable_relative_error_margin.yaml | 9 ----- 8 files changed, 50 insertions(+), 38 deletions(-) delete mode 100644 tests/fixtures/yaml_tests/failing_test_variable_absolute_error_margin.yaml delete mode 100644 tests/fixtures/yaml_tests/failing_test_variable_relative_error_margin.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index db44866fc5..a74ddad455 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 35.11.0 [#1149](https://github.com/openfisca/openfisca-core/pull/1149) + +#### New Features + +- Introduce variable dependent error margins in YAML tests + ### 35.10.1 [#1143](https://github.com/openfisca/openfisca-core/pull/1143) #### Bug fix diff --git a/openfisca_core/tools/test_runner.py b/openfisca_core/tools/test_runner.py index 3acacf1f3e..0820750806 100644 --- a/openfisca_core/tools/test_runner.py +++ b/openfisca_core/tools/test_runner.py @@ -1,6 +1,5 @@ -# -*- coding: utf-8 -*- +from __future__ import annotations -import collections import warnings import sys import os @@ -208,35 +207,40 @@ def check_output(self): def check_variable(self, variable_name: str, expected_value, period, entity_index = None): if self.should_ignore_variable(variable_name): return - if isinstance(expected_value, dict): + + if isinstance(expected_value, Dict): for requested_period, expected_value_at_period in expected_value.items(): self.check_variable(variable_name, expected_value_at_period, requested_period, entity_index) - return + + return None actual_value = self.simulation.calculate(variable_name, period) - absolute_error_margin = self.test.get('absolute_error_margin') - variable_absolute_error_margin = ( - absolute_error_margin.get(variable_name, absolute_error_margin.get("default")) - if isinstance(absolute_error_margin, collections.Mapping) - else absolute_error_margin - ) + absolute_error_margin = self.test.get("absolute_error_margin") + + if isinstance(absolute_error_margin, Dict): + absolute_error_margin = absolute_error_margin.get( + variable_name, + absolute_error_margin.get("default"), + ) relative_error_margin = self.test.get('relative_error_margin') - variable_relative_error_margin = ( - relative_error_margin.get(variable_name, relative_error_margin.get("default")) - if isinstance(relative_error_margin, collections.Mapping) - else relative_error_margin - ) + + if isinstance(relative_error_margin, Dict): + relative_error_margin = relative_error_margin.get( + variable_name, + relative_error_margin.get("default"), + ) if entity_index is not None: actual_value = actual_value[entity_index] + return assert_near( actual_value, expected_value, - absolute_error_margin = variable_absolute_error_margin, + absolute_error_margin, message = f"{variable_name}@{period}: ", - relative_error_margin = variable_relative_error_margin, + relative_error_margin = relative_error_margin, ) def should_ignore_variable(self, variable_name: str): diff --git a/setup.py b/setup.py index cc1216e69b..ad2c48f5c7 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ setup( name = 'OpenFisca-Core', - version = '35.10.1', + version = '35.11.0', author = 'OpenFisca Team', author_email = 'contact@openfisca.org', classifiers = [ diff --git a/tests/core/test_yaml.py b/tests/core/test_yaml.py index 36ac8949c8..f63e37ff39 100644 --- a/tests/core/test_yaml.py +++ b/tests/core/test_yaml.py @@ -37,7 +37,6 @@ def test_relative_error_margin_success(tax_benefit_system): def test_relative_error_margin_fail(tax_benefit_system): assert run_yaml_test(tax_benefit_system, 'failing_test_relative_error_margin.yaml') == EXIT_TESTSFAILED - assert run_yaml_test(tax_benefit_system, 'failing_test_variable_relative_error_margin.yaml') == EXIT_TESTSFAILED def test_absolute_error_margin_success(tax_benefit_system): @@ -46,7 +45,6 @@ def test_absolute_error_margin_success(tax_benefit_system): def test_absolute_error_margin_fail(tax_benefit_system): assert run_yaml_test(tax_benefit_system, 'failing_test_absolute_error_margin.yaml') == EXIT_TESTSFAILED - assert run_yaml_test(tax_benefit_system, 'failing_test_variable_absolute_error_margin.yaml') == EXIT_TESTSFAILED def test_run_tests_from_directory(tax_benefit_system): diff --git a/tests/fixtures/yaml_tests/failing_test_absolute_error_margin.yaml b/tests/fixtures/yaml_tests/failing_test_absolute_error_margin.yaml index a51ae6894e..4928b06711 100644 --- a/tests/fixtures/yaml_tests/failing_test_absolute_error_margin.yaml +++ b/tests/fixtures/yaml_tests/failing_test_absolute_error_margin.yaml @@ -5,3 +5,14 @@ salary: 2000 output: income_tax: 351 # 300 + + +- name: "Failing test: result out of variable specific absolute error margin" + period: 2015-01 + absolute_error_margin: + default: 100 + income_tax: 50 + input: + salary: 2000 + output: + income_tax: 351 # 300 diff --git a/tests/fixtures/yaml_tests/failing_test_relative_error_margin.yaml b/tests/fixtures/yaml_tests/failing_test_relative_error_margin.yaml index 9258946c3d..c0788cfa96 100644 --- a/tests/fixtures/yaml_tests/failing_test_relative_error_margin.yaml +++ b/tests/fixtures/yaml_tests/failing_test_relative_error_margin.yaml @@ -5,3 +5,14 @@ salary: 2000 output: income_tax: 316 # 300 + + +- name: "Failing test: result out of variable specific relative error margin" + period: 2015-01 + relative_error_margin: + default: 1 + income_tax: 0.05 + input: + salary: 2000 + output: + income_tax: 316 # 300 diff --git a/tests/fixtures/yaml_tests/failing_test_variable_absolute_error_margin.yaml b/tests/fixtures/yaml_tests/failing_test_variable_absolute_error_margin.yaml deleted file mode 100644 index af708e8e94..0000000000 --- a/tests/fixtures/yaml_tests/failing_test_variable_absolute_error_margin.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- name: "Failing test: result out of variable specific absolute error margin" - period: 2015-01 - absolute_error_margin: - default: 100 - income_tax: 50 - input: - salary: 2000 - output: - income_tax: 351 # 300 diff --git a/tests/fixtures/yaml_tests/failing_test_variable_relative_error_margin.yaml b/tests/fixtures/yaml_tests/failing_test_variable_relative_error_margin.yaml deleted file mode 100644 index 1f1559a829..0000000000 --- a/tests/fixtures/yaml_tests/failing_test_variable_relative_error_margin.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- name: "Failing test: result out of variable specific relative error margin" - period: 2015-01 - relative_error_margin: - default: 1 - income_tax: 0.05 - input: - salary: 2000 - output: - income_tax: 316 # 300