Introduce error margin by variable in tests

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 35.11.0 [#1149](https://github.com/openfisca/openfisca-core/pull/1149)
+
+#### New Features
+
+- Introduce variable dependent error margins in YAML tests
+
 ### 35.10.1 [#1143](https://github.com/openfisca/openfisca-core/pull/1143)
 
 #### Bug fix

openfisca_core/tools/test_runner.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+from __future__ import annotations
 
 import warnings
 import sys
@@ -207,21 +207,40 @@ def check_output(self):
     def check_variable(self, variable_name: str, expected_value, period, entity_index = None):
         if self.should_ignore_variable(variable_name):
             return
-        if isinstance(expected_value, dict):
+
+        if isinstance(expected_value, Dict):
             for requested_period, expected_value_at_period in expected_value.items():
                 self.check_variable(variable_name, expected_value_at_period, requested_period, entity_index)
-            return
+
+            return None
 
         actual_value = self.simulation.calculate(variable_name, period)
 
+        absolute_error_margin = self.test.get("absolute_error_margin")
+
+        if isinstance(absolute_error_margin, Dict):
+            absolute_error_margin = absolute_error_margin.get(
+                variable_name,
+                absolute_error_margin.get("default"),
+                )
+
+        relative_error_margin = self.test.get('relative_error_margin')
+
+        if isinstance(relative_error_margin, Dict):
+            relative_error_margin = relative_error_margin.get(
+                variable_name,
+                relative_error_margin.get("default"),
+                )
+
         if entity_index is not None:
             actual_value = actual_value[entity_index]
+
         return assert_near(
             actual_value,
             expected_value,
-            absolute_error_margin = self.test.get('absolute_error_margin'),
+            absolute_error_margin,
             message = f"{variable_name}@{period}: ",
-            relative_error_margin = self.test.get('relative_error_margin'),
+            relative_error_margin = relative_error_margin,
             )
 
     def should_ignore_variable(self, variable_name: str):

setup.py

@@ -48,7 +48,7 @@
 
 setup(
     name = 'OpenFisca-Core',
-    version = '35.10.1',
+    version = '35.11.0',
     author = 'OpenFisca Team',
     author_email = 'contact@openfisca.org',
     classifiers = [

tests/fixtures/yaml_tests/failing_test_absolute_error_margin.yaml

@@ -5,3 +5,14 @@
     salary: 2000
   output:
     income_tax: 351 # 300
+
+
+- name: "Failing test: result out of variable specific absolute error margin"
+  period: 2015-01
+  absolute_error_margin:
+    default: 100
+    income_tax: 50
+  input:
+    salary: 2000
+  output:
+    income_tax: 351 # 300

tests/fixtures/yaml_tests/failing_test_relative_error_margin.yaml

@@ -5,3 +5,14 @@
     salary: 2000
   output:
     income_tax: 316 # 300
+
+
+- name: "Failing test: result out of variable specific relative error margin"
+  period: 2015-01
+  relative_error_margin:
+    default: 1
+    income_tax: 0.05
+  input:
+    salary: 2000
+  output:
+    income_tax: 316 # 300

tests/fixtures/yaml_tests/test_absolute_error_margin.yaml

@@ -5,3 +5,14 @@
     salary: 2000
   output:
     income_tax: 350 # 300
+
+
+- name: "Result within absolute error margin"
+  period: 2015-01
+  absolute_error_margin:
+    default: 100
+    income_tax: 50
+  input:
+    salary: 2000
+  output:
+    income_tax: 350 # 300

tests/fixtures/yaml_tests/test_relative_error_margin.yaml

@@ -5,3 +5,14 @@
     salary: 2000
   output:
     income_tax: 290 # 300
+
+
+- name: "Result within variable relative error margin"
+  period: 2015-01
+  relative_error_margin:
+    default: .001
+    income_tax: 0.05
+  input:
+    salary: 2000
+  output:
+    income_tax: 290 # 300