diff --git a/README.md b/README.md index 04ff117..56628d6 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,8 @@ Then `results_robot_output.xml` will be created under `path/to/`. ### [Read the developer guide on how to write your own handler](DEVGUIDE.md) +You might also want to look at [specification for handler results](handler_result_specification.md) + ### Configuring your handler to Oxygen Oxygen knows about different handlers based on the [`config.yml`](https://github.com/eficode/robotframework-oxygen/blob/master/config.yml) file. This configuration file can be interacted with through Oxygen's command line. diff --git a/handler_result_specification.md b/handler_result_specification.md new file mode 100644 index 0000000..6f6eccb --- /dev/null +++ b/handler_result_specification.md @@ -0,0 +1,132 @@ +# Oxygen handler result specification +``` +{ + "$defs": { + "OxygenKeywordDict": { + "properties": { + "pass": { + "title": "Pass", + "type": "boolean" + }, + "name": { + "title": "Name", + "type": "string" + }, + "elapsed": { + "title": "Elapsed", + "type": "number" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "messages": { + "items": { + "type": "string" + }, + "title": "Messages", + "type": "array" + }, + "teardown": { + "$ref": "#/$defs/OxygenKeywordDict" + }, + "keywords": { + "items": { + "$ref": "#/$defs/OxygenKeywordDict" + }, + "title": "Keywords", + "type": "array" + } + }, + "required": [ + "pass", + "name" + ], + "title": "OxygenKeywordDict", + "type": "object" + }, + "OxygenSuiteDict": { + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "setup": { + "$ref": "#/$defs/OxygenKeywordDict" + }, + "teardown": { + "$ref": "#/$defs/OxygenKeywordDict" + }, + "suites": { + "items": { + "$ref": "#/$defs/OxygenSuiteDict" + }, + "title": "Suites", + "type": "array" + }, + "tests": { + "items": { + "$ref": "#/$defs/OxygenTestCaseDict" + }, + "title": "Tests", + "type": "array" + } + }, + "required": [ + "name" + ], + "title": "OxygenSuiteDict", + "type": "object" + }, + "OxygenTestCaseDict": { + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "keywords": { + "items": { + "$ref": "#/$defs/OxygenKeywordDict" + }, + "title": "Keywords", + "type": "array" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "setup": { + "$ref": "#/$defs/OxygenKeywordDict" + }, + "teardown": { + "$ref": "#/$defs/OxygenKeywordDict" + } + }, + "required": [ + "name", + "keywords" + ], + "title": "OxygenTestCaseDict", + "type": "object" + } + }, + "allOf": [ + { + "$ref": "#/$defs/OxygenSuiteDict" + } + ] +} +``` \ No newline at end of file diff --git a/parser_specification.md b/parser_specification.md deleted file mode 100644 index e3ba765..0000000 --- a/parser_specification.md +++ /dev/null @@ -1,50 +0,0 @@ -TestSuite: - -{ - 'name': , - 'tags': [ - , - ], - 'setup': , - 'teardown': , - 'suites': [ - , - ], - 'tests': [ - '', - ], -} - - -TestCase: - -{ - 'name': , - 'tags': [ - , - ], - 'setup': , - 'teardown': , - 'keywords': [ - , - ], -} - - -Keyword: - -{ - 'name': , - 'pass': , - 'elapsed': , // milliseconds - 'tags': [ - , - ], - 'messages': [ - , - ], - 'teardown': , - 'keywords': [ - , - ], -} diff --git a/requirements.txt b/requirements.txt index da6a5c6..f8f313b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ robotframework>=3.0.4 junitparser==2.0 PyYAML>=3.13 +pydantic>=2.4.2 ### Dev mock>=2.0.0 diff --git a/setup.py b/setup.py index ebca99c..efb4263 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,8 @@ install_requires=[ 'robotframework>=3.0.4', 'junitparser==2.0', - 'PyYAML>=3.13' + 'PyYAML>=3.13', + 'pydantic>=2.4.2' ], packages=find_packages(SRC), package_dir={'': 'src'}, diff --git a/src/oxygen/base_handler.py b/src/oxygen/base_handler.py index a0eb4e0..281a0e0 100644 --- a/src/oxygen/base_handler.py +++ b/src/oxygen/base_handler.py @@ -2,10 +2,10 @@ from inspect import signature, Parameter -from oxygen.errors import MismatchArgumentException +from .errors import MismatchArgumentException from .robot_interface import (RobotInterface, get_keywords_from, set_special_keyword) - +from .utils import validate_with_deprecation_warning class BaseHandler(object): DEFAULT_CLI = {tuple(['result_file']): {}} @@ -132,6 +132,8 @@ def _build_results(self, keyword, setup_keyword, teardown_keyword): f'parse_results expects at least {accepted_params_min} ' 'arguments but got 1') + self._validate(test_results) + _, result_suite = self._interface.result.build_suite( 100000, test_results) @@ -149,6 +151,9 @@ def _build_results(self, keyword, setup_keyword, teardown_keyword): self._inject_suite_report(test, result_suite) + def _validate(self, oxygen_result_dict): + validate_with_deprecation_warning(oxygen_result_dict, self) + def _inject_suite_report(self, test, result_suite): '''Add the given suite to the parent suite of the test case. diff --git a/src/oxygen/errors.py b/src/oxygen/errors.py index 4134f13..933ac85 100644 --- a/src/oxygen/errors.py +++ b/src/oxygen/errors.py @@ -32,3 +32,7 @@ class MismatchArgumentException(Exception): class InvalidConfigurationException(Exception): pass + + +class InvalidOxygenResultException(Exception): + pass diff --git a/src/oxygen/gatling.py b/src/oxygen/gatling.py index fa8b107..13fb55b 100644 --- a/src/oxygen/gatling.py +++ b/src/oxygen/gatling.py @@ -65,10 +65,7 @@ def _transform_tests(self, result_file): keyword = { 'name': ' | '.join(columns), 'pass': True, - 'tags': [], 'messages': [], - 'teardown': [], - 'keywords': [], } if status == 'KO': @@ -77,9 +74,6 @@ def _transform_tests(self, result_file): test_case = { 'name': step_name, - 'tags': [], - 'setup': [], - 'teardown': [], 'keywords': [keyword] } @@ -88,9 +82,6 @@ def _transform_tests(self, result_file): test_suite = { 'name': 'Gatling Scenario', 'tags': self._tags, - 'setup': [], - 'teardown': [], - 'suites': [], 'tests': test_cases, } diff --git a/src/oxygen/junit.py b/src/oxygen/junit.py index d66e772..b4fa5f2 100644 --- a/src/oxygen/junit.py +++ b/src/oxygen/junit.py @@ -38,10 +38,7 @@ def _transform_tests(self, node): suite_dict = { 'name': 'JUnit Execution', 'tags': self._tags, - 'setup': [], - 'teardown': [], 'suites': [], - 'tests': [], } if isinstance(node, JUnitXmlTestSuite): @@ -63,8 +60,6 @@ def _transform_test_suite(self, test_suite): suite_dict = { 'name': test_suite.name, 'tags': [], - 'setup': [], - 'teardown': [], 'suites': [], 'tests': [], } @@ -91,9 +86,7 @@ def _transform_test_case(self, test_case): test_dict = { 'name': '{} (Execution)'.format(test_case.name), 'pass': True, - 'tags': [], 'messages': [], - 'teardown': [], 'keywords': [], } @@ -126,8 +119,6 @@ def _transform_test_case(self, test_case): test_case_dict = { 'name': test_case.name, 'tags': [], - 'setup': [], - 'teardown': [], 'keywords': [ test_dict, ], diff --git a/src/oxygen/oxygen.py b/src/oxygen/oxygen.py index 517d72b..3d30c05 100644 --- a/src/oxygen/oxygen.py +++ b/src/oxygen/oxygen.py @@ -18,6 +18,7 @@ InvalidConfigurationException, ResultFileNotFoundException) from .robot_interface import RobotInterface +from .utils import validate_with_deprecation_warning from .version import VERSION @@ -301,6 +302,7 @@ def convert_to_robot_result(self, args): output_filename = self.get_output_filename(args.get('result_file')) parsed_results = args['func']( **{k: v for (k, v) in args.items() if not callable(v)}) + validate_with_deprecation_warning(parsed_results, args['func']) robot_suite = RobotInterface().running.build_suite(parsed_results) robot_suite.run(output=output_filename, log=None, diff --git a/src/oxygen/oxygen_handler_result.py b/src/oxygen/oxygen_handler_result.py new file mode 100644 index 0000000..1702e32 --- /dev/null +++ b/src/oxygen/oxygen_handler_result.py @@ -0,0 +1,71 @@ +''' IMPORTANT + +OxygenKeywordDict is defined like this since key `pass` is reserved +word in Python, and thus raises SyntaxError if defined like a class. +However, in the functional style you cannot refer to the TypedDict itself +recursively, like you can with with class style. Oh bother. + +See more: + - https://docs.python.org/3/library/typing.html?highlight=typeddict#typing.TypedDict + - https://stackoverflow.com/a/72460065 +''' + +import functools + +from typing import List +# TODO FIXME: Python 3.10 requires these to be imported from here +# Python 3.10 EOL is in 2026 +from typing_extensions import TypedDict, Required + +from pydantic import TypeAdapter, ValidationError + +from .errors import InvalidOxygenResultException + + +_Pass = TypedDict('_Pass', { 'pass': Required[bool], 'name': Required[str] }) +# define required fields in this one above +class OxygenKeywordDict(_Pass, total=False): + elapsed: float # milliseconds + tags: List[str] + messages: List[str] + teardown: 'OxygenKeywordDict' # in RF, keywords do not have setup kw; just put it as first kw in `keywords` + keywords: List['OxygenKeywordDict'] + + +class OxygenTestCaseDict(TypedDict, total=False): + name: Required[str] + keywords: Required[List[OxygenKeywordDict]] + tags: List[str] + setup: OxygenKeywordDict + teardown: OxygenKeywordDict + + +class OxygenSuiteDict(TypedDict, total=False): + name: Required[str] + tags: List[str] + setup: OxygenKeywordDict + teardown: OxygenKeywordDict + suites: List['OxygenSuiteDict'] + tests: List[OxygenTestCaseDict] + + +def _change_validationerror_to_oxygenexception(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except ValidationError as e: + raise InvalidOxygenResultException(e) + return wrapper + +@_change_validationerror_to_oxygenexception +def validate_oxygen_suite(oxygen_result_dict): + return TypeAdapter(OxygenSuiteDict).validate_python(oxygen_result_dict) + +@_change_validationerror_to_oxygenexception +def validate_oxygen_test_case(oxygen_test_case_dict): + return TypeAdapter(OxygenTestCaseDict).validate_python(oxygen_test_case_dict) + +@_change_validationerror_to_oxygenexception +def validate_oxygen_keyword(oxygen_kw_dict): + return TypeAdapter(OxygenKeywordDict).validate_python(oxygen_kw_dict) diff --git a/src/oxygen/robot3_interface.py b/src/oxygen/robot3_interface.py index e1a72f5..023abf9 100644 --- a/src/oxygen/robot3_interface.py +++ b/src/oxygen/robot3_interface.py @@ -343,7 +343,7 @@ def create_wrapper_keyword(self, class RobotRunningInterface(object): def build_suite(self, parsed_results): robot_root_suite = RobotRunningSuite(parsed_results['name']) - for parsed_suite in parsed_results['suites']: + for parsed_suite in parsed_results.get('suites', []): robot_suite = robot_root_suite.suites.create(parsed_suite['name']) for subsuite in parsed_suite.get('suites', []): robot_subsuite = self.build_suite(subsuite) @@ -355,9 +355,9 @@ def build_suite(self, parsed_results): def build_tests(self, oxygen_suite, robot_suite): for parsed_test in oxygen_suite.get('tests', []): name = parsed_test['name'] - tags = parsed_test['tags'] + tags = parsed_test.get('tags', []) kw = parsed_test['keywords'][0] - msg = '\n'.join(kw['messages']) + msg = '\n'.join(kw.get('messages', [])) test_robot_counterpart = robot_suite.tests.create(name, tags=tags) if kw['pass']: args = [msg if msg else 'Test passed :D'] diff --git a/src/oxygen/robot4_interface.py b/src/oxygen/robot4_interface.py index e4f4779..c405d5b 100644 --- a/src/oxygen/robot4_interface.py +++ b/src/oxygen/robot4_interface.py @@ -343,7 +343,7 @@ def create_wrapper_keyword(self, class RobotRunningInterface(object): def build_suite(self, parsed_results): robot_root_suite = RobotRunningSuite(parsed_results['name']) - for parsed_suite in parsed_results['suites']: + for parsed_suite in parsed_results.get('suites', []): robot_suite = robot_root_suite.suites.create(parsed_suite['name']) for subsuite in parsed_suite.get('suites', []): robot_subsuite = self.build_suite(subsuite) @@ -355,9 +355,9 @@ def build_suite(self, parsed_results): def build_tests(self, oxygen_suite, robot_suite): for parsed_test in oxygen_suite.get('tests', []): name = parsed_test['name'] - tags = parsed_test['tags'] + tags = parsed_test.get('tags', []) kw = parsed_test['keywords'][0] - msg = '\n'.join(kw['messages']) + msg = '\n'.join(kw.get('messages', [])) test_robot_counterpart = robot_suite.tests.create(name, tags=tags) if kw['pass']: args = [msg if msg else 'Test passed :D'] diff --git a/src/oxygen/utils.py b/src/oxygen/utils.py index 768df03..1c81c41 100644 --- a/src/oxygen/utils.py +++ b/src/oxygen/utils.py @@ -4,10 +4,11 @@ from pathlib import Path -from .errors import (SubprocessException, +from .errors import (InvalidOxygenResultException, ResultFileIsNotAFileException, - ResultFileNotFoundException) - + ResultFileNotFoundException, + SubprocessException) +from .oxygen_handler_result import validate_oxygen_suite def run_command_line(command, check_return_code=True, **env): new_env = os.environ.copy() @@ -27,7 +28,6 @@ def run_command_line(command, check_return_code=True, **env): f'code {proc.returncode}:\n"{proc.stdout.decode("utf-8")}"') return proc.stdout - def validate_path(filepath): try: path = Path(filepath) @@ -39,3 +39,17 @@ def validate_path(filepath): raise ResultFileIsNotAFileException(f'File "{path}" is not a file, ' 'but a directory') return path + +def validate_with_deprecation_warning(oxygen_result_dict, handler): + try: + validate_oxygen_suite(oxygen_result_dict) + except InvalidOxygenResultException as e: + import warnings + # this is not done with triple quotes intentionally + # to get sensible formatting to output + msg = (f'\n{handler.__module__} is producing invalid results:\n' + f'{e}\n\n' + 'In Oxygen 1.0, handlers will need to produce valid ' + 'results.\nSee: ' + 'https://github.com/eficode/robotframework-oxygen/blob/master/parser_specification.md') + warnings.warn(msg) diff --git a/tasks.py b/tasks.py index d2d54f6..0df05d5 100644 --- a/tasks.py +++ b/tasks.py @@ -37,7 +37,7 @@ def install(context, package=None): 'multiple times to select several targets.' }) def utest(context, test=None): - run(f'pytest {" ".join(test) if test else UNIT_TESTS} -q --disable-warnings', + run(f'pytest {" -k".join(test) if test else UNIT_TESTS} -q --disable-warnings', env={'PYTHONPATH': str(SRCPATH)}, pty=(not system() == 'Windows')) @@ -67,6 +67,25 @@ def test(context): atest(context) @task +def update_oxygen_schema(context): + import sys + import json + from pydantic import TypeAdapter + + sys.path.insert(0, str(SRCPATH)) + from oxygen.oxygen_handler_result import OxygenSuiteDict + + schema = TypeAdapter(OxygenSuiteDict).json_schema() + out = json.dumps(schema, indent=2) + with open(CURDIR / 'handler_result_specification.md', 'r+') as f: + header = f.readline() + f.seek(0) + f.write(header) + f.write(f'```\n{out}\n```') + f.truncate() + print('Updated schema') + +@task(pre=[update_oxygen_schema]) def doc(context): doc_path = CURDIR / 'docs' if not doc_path.exists(): @@ -82,3 +101,4 @@ def doc(context): @task(pre=[clean]) def build(context): run(f'python -m build --wheel') + diff --git a/tests/utest/gatling/test_basic_functionality.py b/tests/utest/gatling/test_basic_functionality.py index a93f7ba..3f36fb5 100644 --- a/tests/utest/gatling/test_basic_functionality.py +++ b/tests/utest/gatling/test_basic_functionality.py @@ -7,6 +7,7 @@ from oxygen.base_handler import BaseHandler from oxygen.gatling import GatlingHandler from oxygen.errors import GatlingHandlerException +from oxygen.oxygen_handler_result import validate_oxygen_suite from ..helpers import (example_robot_output, GATLING_EXPECTED_OUTPUT, get_config, @@ -88,3 +89,4 @@ def test_gatling_parsing(self): example_file = RESOURCES_PATH / 'gatling-example-simulation.log' retval = self.handler._transform_tests(example_file) compare(retval, GATLING_EXPECTED_OUTPUT) + self.assertTrue(validate_oxygen_suite(retval)) diff --git a/tests/utest/helpers.py b/tests/utest/helpers.py index 0174a41..0646a41 100644 --- a/tests/utest/helpers.py +++ b/tests/utest/helpers.py @@ -8,6 +8,8 @@ from robot.api import ExecutionResult from yaml import FullLoader, load +from oxygen.oxygen_handler_result import OxygenKeywordDict, OxygenTestCaseDict + TEST_CONFIG = ''' oxygen.junit: handler: JUnitHandler @@ -55,1167 +57,662 @@ def example_robot_output(): output = RESOURCES_PATH / 'example_robot_output.xml' return ExecutionResult(output) +MINIMAL_KEYWORD_DICT = { 'name': 'someKeyword', 'pass': True } +MINIMAL_TC_DICT = { 'name': 'Minimal TC', 'keywords': [MINIMAL_KEYWORD_DICT] } +MINIMAL_SUITE_DICT = {'name': 'Minimal Suite', + 'suites': [{ + 'name': 'Minimal Subsuite', + 'tests': [ MINIMAL_TC_DICT ]}]} + + +class _ListSubclass(list): + '''Used in test cases''' + pass + + +class _KwSubclass(OxygenKeywordDict): + '''Used in test cases''' + pass + + +class _TCSubclass(OxygenTestCaseDict): + '''Used in test cases''' + pass + + GATLING_EXPECTED_OUTPUT = {'name': 'Gatling Scenario', - 'setup': [], - 'suites': [], 'tags': ['GATLING'], - 'teardown': [], - 'tests': [{'keywords': [{'keywords': [], + 'tests': [{'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 4 | | Home | ' '1533120479221 | 1533120479313 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 1 | | Home | ' '1533120479124 | 1533120479298 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 1 | | Home Redirect 1 | ' '1533120479321 | 1533120479368 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home Redirect 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home Redirect 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 4 | | Home Redirect 1 | ' '1533120479321 | 1533120479367 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home Redirect 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home Redirect 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 2 | | Home | ' '1533120480235 | 1533120480421 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 2 | | Home Redirect 1 | ' '1533120480422 | 1533120480466 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home Redirect 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home Redirect 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 1 | | Search | ' '1533120480410 | 1533120480744 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Search', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Search'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 4 | | Search | ' '1533120480407 | 1533120480768 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Search', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Search'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 3 | | Home | ' '1533120481234 | 1533120481322 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 3 | | Home Redirect 1 | ' '1533120481323 | 1533120481368 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home Redirect 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home Redirect 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 2 | | Search | ' '1533120481486 | 1533120481533 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Search', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Search'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 1 | | Select | ' '1533120481736 | 1533120481783 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Select', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Select'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 4 | | Select | ' '1533120481756 | 1533120481800 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Select', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Select'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 5 | | Home | ' '1533120482095 | 1533120482187 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 5 | | Home Redirect 1 | ' '1533120482188 | 1533120482235 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home Redirect 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home Redirect 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 3 | | Search | ' '1533120482385 | 1533120482431 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Search', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Search'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 2 | | Select | ' '1533120482536 | 1533120482582 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Select', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Select'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 1 | | Page 0 | ' '1533120482801 | 1533120482848 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 0', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 0'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 4 | | Page 0 | ' '1533120482804 | 1533120482848 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 0', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 0'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 6 | | Home | ' '1533120483096 | 1533120483190 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 6 | | Home Redirect 1 | ' '1533120483191 | 1533120483238 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home Redirect 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home Redirect 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 5 | | Search | ' '1533120483256 | 1533120483303 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Search', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Search'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 3 | | Select | ' '1533120483436 | 1533120483482 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Select', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Select'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 2 | | Page 0 | ' '1533120483585 | 1533120483631 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 0', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 0'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 1 | | Page 1 | ' '1533120483835 | 1533120483881 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 4 | | Page 1 | ' '1533120483845 | 1533120483889 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 8 | | Home | ' '1533120484086 | 1533120484182 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 7 | | Home | ' '1533120484095 | 1533120484221 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 8 | | Home Redirect 1 | ' '1533120484183 | 1533120484228 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home Redirect 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home Redirect 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 7 | | Home Redirect 1 | ' '1533120484222 | 1533120484269 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home Redirect 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home Redirect 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 6 | | Search | ' '1533120484255 | 1533120484303 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Search', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Search'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 5 | | Select | ' '1533120484305 | 1533120484352 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Select', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Select'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 3 | | Page 0 | ' '1533120484476 | 1533120484523 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 0', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 0'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 2 | | Page 1 | ' '1533120484635 | 1533120484679 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 1 | | Page 2 | ' '1533120484887 | 1533120484933 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 2', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 2'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 4 | | Page 2 | ' '1533120484896 | 1533120484940 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 2', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 2'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 9 | | Home | ' '1533120485084 | 1533120485170 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 9 | | Home Redirect 1 | ' '1533120485171 | 1533120485225 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home Redirect 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home Redirect 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 8 | | Search | ' '1533120485246 | 1533120485291 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Search', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Search'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 7 | | Search | ' '1533120485286 | 1533120485331 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Search', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Search'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 6 | | Select | ' '1533120485306 | 1533120485357 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Select', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Select'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 5 | | Page 0 | ' '1533120485345 | 1533120485408 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 0', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 0'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 3 | | Page 1 | ' '1533120485536 | 1533120485583 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 2 | | Page 2 | ' '1533120485676 | 1533120485720 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 2', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 2'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 4 | | Page 3 | ' '1533120485936 | 1533120485981 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 3', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 3'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 1 | | Page 3 | ' '1533120485936 | 1533120485982 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 3', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 3'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 10 | | Home | ' '1533120486085 | 1533120486166 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 10 | | Home Redirect 1 | ' '1533120486167 | 1533120486212 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home Redirect 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home Redirect 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 9 | | Search | ' '1533120486247 | 1533120486291 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Search', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Search'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 8 | | Select | ' '1533120486295 | 1533120486340 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Select', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Select'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 7 | | Select | ' '1533120486335 | 1533120486380 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Select', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Select'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 6 | | Page 0 | ' '1533120486355 | 1533120486403 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 0', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 0'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 5 | | Page 1 | ' '1533120486414 | 1533120486460 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 3 | | Page 2 | ' '1533120486586 | 1533120486631 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 2', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 2'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 2 | | Page 3 | ' '1533120486716 | 1533120486761 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 3', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 3'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 4 | | Form | ' '1533120486987 | 1533120487033 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Form', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Form'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 11 | | Home | ' '1533120487094 | 1533120487191 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 11 | | Home Redirect 1 | ' '1533120487192 | 1533120487237 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home Redirect 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home Redirect 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 10 | | Search | ' '1533120487236 | 1533120487280 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Search', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Search'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 9 | | Select | ' '1533120487284 | 1533120487332 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Select', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Select'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 8 | | Page 0 | ' '1533120487334 | 1533120487378 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 0', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 0'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 7 | | Page 0 | ' '1533120487375 | 1533120487420 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 0', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 0'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 6 | | Page 1 | ' '1533120487416 | 1533120487464 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 5 | | Page 2 | ' '1533120487455 | 1533120487502 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 2', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 2'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 3 | | Page 3 | ' '1533120487615 | 1533120487662 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 3', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 3'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 4 | | Post | ' '1533120488053 | 1533120488103 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Post', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Post'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 4 | | Post Redirect 1 | ' '1533120488113 | 1533120488157 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Post Redirect 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Post Redirect 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 12 | | Home | ' '1533120488096 | 1533120488181 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 12 | | Home Redirect 1 | ' '1533120488182 | 1533120488249 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Home Redirect 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Home Redirect 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 11 | | Search | ' '1533120488256 | 1533120488300 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Search', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Search'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 10 | | Select | ' '1533120488276 | 1533120488319 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Select', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Select'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 9 | | Page 0 | ' '1533120488334 | 1533120488390 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 0', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 0'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 8 | | Page 1 | ' '1533120488384 | 1533120488429 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 7 | | Page 1 | ' '1533120488425 | 1533120488471 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 6 | | Page 2 | ' '1533120488465 | 1533120488513 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 2', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 2'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 5 | | Page 3 | ' '1533120488504 | 1533120488549 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 3', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 3'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 12 | | Search | ' '1533120489266 | 1533120489311 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Search', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Search'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 11 | | Select | ' '1533120489306 | 1533120489348 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Select', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Select'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 10 | | Page 0 | ' '1533120489316 | 1533120489358 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 0', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 0'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 9 | | Page 1 | ' '1533120489395 | 1533120489441 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 8 | | Page 2 | ' '1533120489425 | 1533120489470 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 2', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 2'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 7 | | Page 2 | ' '1533120489463 | 1533120489509 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 2', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 2'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 6 | | Page 3 | ' '1533120489504 | 1533120489577 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 3', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 3'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 12 | | Select | ' '1533120490304 | 1533120490353 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Select', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Select'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 11 | | Page 0 | ' '1533120490344 | 1533120490386 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 0', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 0'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 10 | | Page 1 | ' '1533120490365 | 1533120490407 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 9 | | Page 2 | ' '1533120490436 | 1533120490481 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 2', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 2'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 8 | | Page 3 | ' '1533120490475 | 1533120490525 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 3', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 3'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 7 | | Page 3 | ' '1533120490515 | 1533120490560 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 3', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 3'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 12 | | Page 0 | ' '1533120491366 | 1533120491413 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 0', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 0'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 11 | | Page 1 | ' '1533120491386 | 1533120491430 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 10 | | Page 2 | ' '1533120491405 | 1533120491447 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 2', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 2'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 9 | | Page 3 | ' '1533120491486 | 1533120491531 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 3', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 3'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 8 | | Form | ' '1533120491515 | 1533120491560 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Form', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Form'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 12 | | Page 1 | ' '1533120492404 | 1533120492449 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 11 | | Page 2 | ' '1533120492436 | 1533120492479 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 2', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 2'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 10 | | Page 3 | ' '1533120492446 | 1533120492488 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 3', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 3'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 8 | | Post | ' '1533120492564 | 1533120492610 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Post', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Post'}, + {'keywords': [{ 'messages': ['status.find.is(201), but actually ' 'found 200'], 'name': 'REQUEST | Admins | 8 | | Post Redirect 1 | ' '1533120492611 | 1533120492655 | KO | ' 'status.find.is(201), but actually found 200', - 'pass': False, - 'tags': [], - 'teardown': []}], - 'name': 'Post Redirect 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': False}], + 'name': 'Post Redirect 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 8 | | Form | ' '1533120492674 | 1533120492719 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Form', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Form'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 12 | | Page 2 | ' '1533120493454 | 1533120493499 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 2', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 2'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 11 | | Page 3 | ' '1533120493476 | 1533120493518 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 3', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Page 3'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Admins | 8 | | Post | ' '1533120493706 | 1533120493750 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Post', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': True}], + 'name': 'Post'}, + {'keywords': [{ 'messages': ['status.find.is(201), but actually ' 'found 200'], 'name': 'REQUEST | Admins | 8 | | Post Redirect 1 | ' '1533120493751 | 1533120493797 | KO | ' 'status.find.is(201), but actually found 200', - 'pass': False, - 'tags': [], - 'teardown': []}], - 'name': 'Post Redirect 1', - 'setup': [], - 'tags': [], - 'teardown': []}, - {'keywords': [{'keywords': [], + 'pass': False}], + 'name': 'Post Redirect 1'}, + {'keywords': [{ 'messages': [], 'name': 'REQUEST | Users | 12 | | Page 3 | ' '1533120494505 | 1533120494552 | OK', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'Page 3', - 'setup': [], - 'tags': [], - 'teardown': []}]} + 'pass': True}], + 'name': 'Page 3'}]} diff --git a/tests/utest/junit/test_basic_functionality.py b/tests/utest/junit/test_basic_functionality.py index 71034ce..a845bf9 100644 --- a/tests/utest/junit/test_basic_functionality.py +++ b/tests/utest/junit/test_basic_functionality.py @@ -6,8 +6,9 @@ from testfixtures import compare from oxygen.base_handler import BaseHandler -from oxygen.junit import JUnitHandler from oxygen.errors import JUnitHandlerException, ResultFileIsNotAFileException +from oxygen.junit import JUnitHandler +from oxygen.oxygen_handler_result import validate_oxygen_suite from ..helpers import example_robot_output, get_config, RESOURCES_PATH class JUnitBasicTests(TestCase): @@ -96,26 +97,17 @@ def test_check_for_keyword(self, mock_report): def test_transform_tests_with_single_test_suite(self): expected_output = { 'name': 'JUnit Execution', - 'setup': [], 'suites': [{'name': 'com.example.demo.DemoApplicationTests', - 'setup': [], - 'suites': [], - 'tags': [], - 'teardown': [], - 'tests': [{'keywords': [{'elapsed': 454.0, - 'keywords': [], - 'messages': [], - 'name': 'contextLoads() (Execution)', - 'pass': True, - 'tags': [], - 'teardown': []}], - 'name': 'contextLoads()', - 'setup': [], - 'tags': [], - 'teardown': []}]}], + 'suites': [], + 'tags': [], + 'tests': [{'keywords': [{'elapsed': 454.0, + 'keywords': [], + 'messages': [], + 'name': 'contextLoads() (Execution)', + 'pass': True}], + 'name': 'contextLoads()', + 'tags': []}]}], 'tags': ['JUNIT', 'EXTRA_JUNIT_CASE'], - 'teardown': [], - 'tests': [] } xml = JUnitXml.fromfile(str(RESOURCES_PATH / 'junit-single-testsuite.xml')) retval = self.handler._transform_tests(xml) @@ -125,97 +117,71 @@ def test_transform_tests_with_multiple_suites(self): expected_output = { 'name': 'JUnit Execution', 'tags': ['JUNIT', 'EXTRA_JUNIT_CASE'], - 'setup': [], - 'teardown': [], 'suites': [{ 'name': 'suite1', 'tags': [], - 'setup': [], - 'teardown': [], 'suites': [{ 'name': 'suite2', 'tags': [], - 'setup': [], - 'teardown': [], 'suites': [], 'tests': [{ 'name': 'casea', 'tags': ['oxygen-junit-unknown-execution-time'], - 'setup': [], - 'teardown': [], 'keywords': [{'name': 'casea (Execution)', 'pass': True, - 'tags': [], 'messages': [], - 'teardown': [], 'keywords': [], 'elapsed': 0.0}] }, { 'name': 'caseb', 'tags': ['oxygen-junit-unknown-execution-time'], - 'setup': [], - 'teardown': [], 'keywords': [{ 'name': 'caseb (Execution)', 'pass': True, - 'tags': [], 'messages': [], - 'teardown': [], - 'keywords': [], - 'elapsed': 0.0 + 'elapsed': 0.0, + 'keywords': [] }] }] }], 'tests': [{ 'name': 'case1', 'tags': ['oxygen-junit-unknown-execution-time'], - 'setup': [], - 'teardown': [], 'keywords': [{ 'name': 'case1 (Execution)', 'pass': True, - 'tags': [], 'messages': [], - 'teardown': [], 'keywords': [], 'elapsed': 0.0 }] }, { 'name': 'case2', 'tags': ['oxygen-junit-unknown-execution-time'], - 'setup': [], - 'teardown': [], 'keywords': [{ 'name': 'case2 (Execution)', 'pass': False, - 'tags': [], 'messages': [ 'ERROR: Example error message (the_error_type)' ], - 'teardown': [], 'keywords': [], 'elapsed': 0.0 }] }, { 'name': 'case3', 'tags': ['oxygen-junit-unknown-execution-time'], - 'setup': [], - 'teardown': [], 'keywords': [{ 'name': 'case3 (Execution)', 'pass': False, - 'tags': [], 'messages': [ 'FAIL: Example failure message (the_failure_type)' ], - 'teardown': [], 'keywords': [], 'elapsed': 0.0 }] }] }], - 'tests': [] } xml = JUnitXml.fromfile(RESOURCES_PATH / 'junit.xml') retval = self.handler._transform_tests(xml) compare(retval, expected_output) + self.assertTrue(validate_oxygen_suite(retval)) diff --git a/tests/utest/oxygen_handler_result/__init__.py b/tests/utest/oxygen_handler_result/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/utest/oxygen_handler_result/shared_tests.py b/tests/utest/oxygen_handler_result/shared_tests.py new file mode 100644 index 0000000..f4c1e8e --- /dev/null +++ b/tests/utest/oxygen_handler_result/shared_tests.py @@ -0,0 +1,45 @@ +from ..helpers import (MINIMAL_KEYWORD_DICT, + _ListSubclass, + _KwSubclass) + +class SharedTestsForName(object): + def shared_test_for_name(self): + class StrSubclass(str): + pass + valid_inherited = StrSubclass('someKeyword') + this_is_not_None = StrSubclass(None) + + self.valid_inputs_for('name', + '', + 'someKeyword', + b'someKeyword', + valid_inherited, + this_is_not_None) + + self.invalid_inputs_for('name', None) + + +class SharedTestsForTags(object): + def shared_test_for_tags(self): + self.valid_inputs_for('tags', + [], + ['some-tag', 'another-tag'], + _ListSubclass()) + + invalid_inherited = _ListSubclass() + invalid_inherited.append(123) + + self.invalid_inputs_for('tags', [123], None, {'foo': 'bar'}, object()) + + +class SharedTestsForKeywordField(object): + def shared_test_for_keyword_field(self, attribute): + valid_inherited = _KwSubclass(**MINIMAL_KEYWORD_DICT) + + self.valid_inputs_for(attribute, + MINIMAL_KEYWORD_DICT, + valid_inherited, + {**MINIMAL_KEYWORD_DICT, + 'something_random': 'will-be-ignored'}) + + self.invalid_inputs_for(attribute, None, {}) diff --git a/tests/utest/oxygen_handler_result/test_OxygenKeywordDict.py b/tests/utest/oxygen_handler_result/test_OxygenKeywordDict.py new file mode 100644 index 0000000..668410f --- /dev/null +++ b/tests/utest/oxygen_handler_result/test_OxygenKeywordDict.py @@ -0,0 +1,149 @@ +from unittest import TestCase + +from oxygen.errors import InvalidOxygenResultException +from oxygen.oxygen_handler_result import (validate_oxygen_keyword, + OxygenKeywordDict) + +from ..helpers import (MINIMAL_KEYWORD_DICT, + _ListSubclass, + _KwSubclass) +from .shared_tests import (SharedTestsForKeywordField, + SharedTestsForName, + SharedTestsForTags) + + +class TestOxygenKeywordDict(TestCase, + SharedTestsForName, + SharedTestsForTags, + SharedTestsForKeywordField): + def setUp(self): + self.minimal = MINIMAL_KEYWORD_DICT + + def test_validate_oxygen_keyword_validates_correctly(self): + with self.assertRaises(InvalidOxygenResultException): + validate_oxygen_keyword({}) + + def test_validate_oxygen_keyword_with_minimal_valid(self): + minimal1 = { 'name': 'somename', 'pass': True } + minimal2 = { 'name': 'somename', 'pass': False } + + self.assertEqual(validate_oxygen_keyword(minimal1), minimal1) + self.assertEqual(validate_oxygen_keyword(minimal2), minimal2) + + def valid_inputs_for(self, attribute, *valid_inputs): + for valid_input in valid_inputs: + self.assertTrue(validate_oxygen_keyword({**self.minimal, + attribute: valid_input})) + + def invalid_inputs_for(self, attribute, *invalid_inputs): + for invalid_input in invalid_inputs: + with self.assertRaises(InvalidOxygenResultException): + validate_oxygen_keyword({**self.minimal, + attribute: invalid_input}) + + def test_validate_oxygen_keyword_validates_name(self): + self.shared_test_for_name() + + def test_validate_oxygen_keyword_validates_pass(self): + ''' + Due note that boolean cannot be subclassed in Python: + https://mail.python.org/pipermail/python-dev/2002-March/020822.html + ''' + self.valid_inputs_for('pass', True, False, 0, 1, 0.0, 1.0) + self.invalid_inputs_for('pass', [], {}, None, object(), -999, -99.9) + + def test_validate_oxygen_keyword_validates_tags(self): + self.shared_test_for_tags() + + def test_validate_oxygen_keyword_validates_elapsed(self): + class FloatSubclass(float): + pass + + self.valid_inputs_for('elapsed', + 123.4, + -123.0, + '123.4', + '-999.999', + 123, + FloatSubclass()) + + self.invalid_inputs_for('elapsed', '', None, object()) + + def test_validate_oxygen_keyword_validates_messages(self): + valid_inherited = _ListSubclass() + valid_inherited.append('message') + + self.valid_inputs_for('messages', + [], + ['message'], + _ListSubclass(), + valid_inherited) + + invalid_inherited = _ListSubclass() + invalid_inherited.append('message') + invalid_inherited.append(123) + + self.invalid_inputs_for('messages', + 'some,messages', + None, + invalid_inherited) + + def test_validate_oxygen_keyword_validates_teardown(self): + self.shared_test_for_keyword_field('teardown') + + def test_validate_oxygen_keyword_validates_keywords(self): + valid_inherited = _ListSubclass() + valid_inherited.append(_KwSubclass(**self.minimal)) + + self.valid_inputs_for('keywords', + [], + [self.minimal, {**self.minimal, + 'something_random': 'will-be-ignored'}], + _ListSubclass(), # empty inherited list + valid_inherited) + + invalid_inherited = _ListSubclass() + invalid_inherited.append(_KwSubclass(**self.minimal)) + invalid_inherited.append(123) + self.invalid_inputs_for('keywords', None, invalid_inherited) + + def test_validate_oxygen_keyword_with_maximal_valid(self): + expected = { + 'name': 'keyword', + 'pass': True, + 'tags': ['some-tag'], + 'messages': ['some message'], + 'teardown': { + 'name': 'teardownKeyword', + 'pass': True, + 'tags': ['teardown-kw'], + 'messages': ['Teardown passed'], + 'keywords': [] + }, + 'keywords': [{ + 'name': 'subKeyword', + 'pass': False, + # tags missing intentionally + 'messages': ['This particular kw failed'], + 'teardown': { + 'name': 'anotherTeardownKw', + 'pass': True, + 'tags': ['teardown-kw'], + 'messages': ['message from anotherTeardownKw'], + # teardown missing intentionally + 'keywords': [] + }, + 'keywords': [{ + 'name': 'subsubKeyword', + 'pass': True, + }] + },{ + 'name': 'anotherSubKeyword', + 'pass': True, + 'tags': [], + 'messages': [], + 'keywords': [] + }] + } + + self.assertEqual(validate_oxygen_keyword(expected), expected) diff --git a/tests/utest/oxygen_handler_result/test_OxygenSuiteDict.py b/tests/utest/oxygen_handler_result/test_OxygenSuiteDict.py new file mode 100644 index 0000000..8237cd7 --- /dev/null +++ b/tests/utest/oxygen_handler_result/test_OxygenSuiteDict.py @@ -0,0 +1,81 @@ +from unittest import TestCase + +from oxygen.errors import InvalidOxygenResultException +from oxygen.oxygen_handler_result import OxygenSuiteDict, validate_oxygen_suite + +from ..helpers import (MINIMAL_TC_DICT, + MINIMAL_SUITE_DICT, + _ListSubclass, + _TCSubclass) +from .shared_tests import (SharedTestsForName, + SharedTestsForKeywordField, + SharedTestsForTags) + +class TestOxygenSuiteDict(TestCase, + SharedTestsForName, + SharedTestsForKeywordField, + SharedTestsForTags): + def setUp(self): + self.minimal = MINIMAL_SUITE_DICT + + def test_validate_oxygen_suite_validates_correctly(self): + with self.assertRaises(InvalidOxygenResultException): + validate_oxygen_suite({}) + + def test_validate_oxygen_suite_with_minimal_valid(self): + expected = { + 'name': 'My Suite' + } + + self.assertEqual(validate_oxygen_suite(expected), expected) + self.assertEqual(validate_oxygen_suite(self.minimal), self.minimal) + + def valid_inputs_for(self, attribute, *valid_inputs): + for valid_input in valid_inputs: + self.assertTrue(validate_oxygen_suite({**self.minimal, + attribute: valid_input})) + + def invalid_inputs_for(self, attribute, *invalid_inputs): + for invalid_input in invalid_inputs: + with self.assertRaises(InvalidOxygenResultException): + validate_oxygen_suite({**self.minimal, attribute: invalid_input}) + + def test_validate_oxygen_suite_validates_name(self): + self.shared_test_for_name() + + def test_validate_oxygen_suite_validates_tags(self): + self.shared_test_for_tags() + + def test_validate_oxygen_suite_validates_setup(self): + self.shared_test_for_keyword_field('setup') + + def test_validate_oxygen_suite_validates_teardown(self): + self.shared_test_for_keyword_field('teardown') + + def test_validate_oxygen_suite_validates_suites(self): + class OxygenSuiteDictSubclass(OxygenSuiteDict): + pass + valid_inherited = _ListSubclass() + valid_inherited.append(OxygenSuiteDictSubclass(**self.minimal)) + + self.valid_inputs_for('suites', + [], + [self.minimal], + valid_inherited, + [ OxygenSuiteDictSubclass(**self.minimal) ]) + + self.invalid_inputs_for('suites', None, [ {} ]) + + def test_validate_oxygen_suite_validates_tests(self): + valid_inherited = _ListSubclass() + valid_inherited.append(_TCSubclass(**MINIMAL_TC_DICT)) + valid_inherited.append(MINIMAL_TC_DICT) + + self.valid_inputs_for('tests', + [], + [ MINIMAL_TC_DICT ], + valid_inherited, + [ _TCSubclass(**MINIMAL_TC_DICT) ]) + + + self.invalid_inputs_for('tests', None, [ {} ]) diff --git a/tests/utest/oxygen_handler_result/test_OxygenTestCaseDict.py b/tests/utest/oxygen_handler_result/test_OxygenTestCaseDict.py new file mode 100644 index 0000000..1ff0728 --- /dev/null +++ b/tests/utest/oxygen_handler_result/test_OxygenTestCaseDict.py @@ -0,0 +1,66 @@ +from unittest import TestCase + +from oxygen.errors import InvalidOxygenResultException +from oxygen.oxygen_handler_result import validate_oxygen_test_case + +from ..helpers import _ListSubclass, MINIMAL_KEYWORD_DICT, MINIMAL_TC_DICT +from .shared_tests import (SharedTestsForKeywordField, + SharedTestsForName, + SharedTestsForTags) + +class TestOxygenTestCaseDict(TestCase, + SharedTestsForName, + SharedTestsForTags, + SharedTestsForKeywordField): + def setUp(self): + self.minimal = MINIMAL_TC_DICT + + def test_validate_oxygen_tc_validates_correctly(self): + with self.assertRaises(InvalidOxygenResultException): + validate_oxygen_test_case({}) + + def test_validate_oxygen_tc_with_minimal_valid(self): + expected = { + 'name': 'My TC', + 'keywords': [] + } + self.assertEqual(validate_oxygen_test_case(expected), expected) + self.assertEqual(validate_oxygen_test_case(self.minimal), self. minimal) + + def valid_inputs_for(self, attribute, *valid_inputs): + for valid_input in valid_inputs: + self.assertTrue(validate_oxygen_test_case({**self.minimal, + attribute: valid_input})) + + def invalid_inputs_for(self, attribute, *invalid_inputs): + for invalid_input in invalid_inputs: + with self.assertRaises(InvalidOxygenResultException): + validate_oxygen_test_case({**self.minimal, + attribute: invalid_input}) + + def test_validate_oxygen_tc_validates_name(self): + self.shared_test_for_name() + + def test_validate_oxygen_tc_validates_keywords(self): + valid_inherited = _ListSubclass() + valid_inherited.append(MINIMAL_KEYWORD_DICT) + + self.valid_inputs_for('keywords', + [], + [ MINIMAL_KEYWORD_DICT ], + _ListSubclass(), + valid_inherited) + + invalid_inherited = _ListSubclass() + invalid_inherited.append( {} ) + + self.invalid_inputs_for('keywords', None, invalid_inherited) + + def test_validate_oxygen_tc_validates_tags(self): + self.shared_test_for_tags() + + def test_validate_oxygen_tc_validates_setup(self): + self.shared_test_for_keyword_field('setup') + + def test_validate_oxygen_tc_validates_teardown(self): + self.shared_test_for_keyword_field('teardown') diff --git a/tests/utest/oxygen_handler_result/test_deprecation_warning.py b/tests/utest/oxygen_handler_result/test_deprecation_warning.py new file mode 100644 index 0000000..85bde73 --- /dev/null +++ b/tests/utest/oxygen_handler_result/test_deprecation_warning.py @@ -0,0 +1,52 @@ +''' +Currently, the introduction of pydantic[1] to validate result dictionaries that +handlers return is planned to just raise a deprecation warning. + +After 1.0 -- ie. a backwards-incompatible release -- we should turn deprecation +warning to actually start failing. See issue [2]. + +[1| https://github.com/eficode/robotframework-oxygen/issues/43 +[2] https://github.com/eficode/robotframework-oxygen/issues/45 +''' +from unittest import TestCase +from unittest.mock import patch +from oxygen.base_handler import BaseHandler +from oxygen.oxygen import OxygenCLI + +from ..helpers import get_config, MINIMAL_SUITE_DICT + +class TestDeprecationWarningWhenValidating(TestCase): + def setUp(self): + self.cli = OxygenCLI() + + def _validate_warning_msg(self, warning, module_name): + warning_message = str(warning.warning) + for expected in (module_name, + 'validation error for typed-dict', + 'In Oxygen 1.0, handlers will need to produce valid results.'): + self.assertIn(expected, warning_message) + + def test_warning_about_invalid_result(self): + handler = BaseHandler(get_config()['oxygen.junit']) + + with self.assertWarns(UserWarning) as warning: + handler._validate({}) + + self._validate_warning_msg(warning, 'oxygen.base_handler') + + @patch('oxygen.oxygen.RobotInterface') + def test_warning_about_invalid_result_in_CLI(self, mock_iface): + with self.assertWarns(UserWarning) as warning: + self.cli.convert_to_robot_result({ + 'result_file': 'doesentmatter', + 'func': lambda **_: {**MINIMAL_SUITE_DICT, 'setup': []} + }) + + mock_iface.assert_any_call() + # this one has weird name because we fake `func` with lambda + self._validate_warning_msg(warning, 'test_deprecation_warning') + + def test_deprecation_was_removed(self): + '''Remove this test once deprecation warning has been removed''' + if self.cli.__version__.startswith('1'): + self.fail('Deprecation warning should have been removed in 1.0') diff --git a/tests/utest/zap/test_basic_functionality.py b/tests/utest/zap/test_basic_functionality.py index cb4b22c..7048cfb 100644 --- a/tests/utest/zap/test_basic_functionality.py +++ b/tests/utest/zap/test_basic_functionality.py @@ -4,8 +4,10 @@ from testfixtures import compare -from oxygen.zap import ZAProxyHandler from oxygen.errors import ZAProxyHandlerException +from oxygen.oxygen_handler_result import validate_oxygen_suite +from oxygen.zap import ZAProxyHandler + from ..helpers import (example_robot_output, get_config, RESOURCES_PATH, @@ -312,3 +314,4 @@ def test_check_for_keyword(self, mock_report): def test_zap_parsing(self): retval = self.handler.parse_results(RESOURCES_PATH / 'zap' / 'zap.xml') compare(retval, ZAP_EXPECTED_OUTPUT) + self.assertTrue(validate_oxygen_suite(retval))