Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 48 additions & 9 deletions libs/giskard-checks/src/giskard/checks/core/scenario.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,12 @@ class Scenario[InputType, OutputType, TraceType: Trace](BaseModel): # pyright:
Optional custom trace type to use. If not provided, the trace type will be
inferred from the components. Useful when using custom trace subclasses
with additional computed fields or methods.
annotations : dict[str, Any]
Key-value pairs merged into the trace at run start.
Can be accessed as `trace.annotations` during scenario execution.
target : ProviderType[[InputType], OutputType] | ProviderType[[InputType, TraceType], OutputType] | NotProvided
Default SUT for interactions whose outputs are ``NOT_PROVIDED`` when no
per-call ``target`` is passed to ``run`` (see ``with_target``).
"""

name: str = Field(
Expand Down Expand Up @@ -153,17 +159,19 @@ def interact(

Parameters
----------
inputs : InputType | Callable | Generator | InputGenerator
The input specification for the interaction.
outputs : OutputType | Callable
The output specification for the interaction.
inputs : InputGenerator | Generator | Callable
Comment thread
kevinmessiaen marked this conversation as resolved.
Outdated
Input specification: static value, ``InputGenerator``, generator, or
callable producing inputs (same options as ``Interact``).
outputs : OutputType | Callable | NotProvided, optional
Output specification, or ``NOT_PROVIDED`` to use the scenario-level
or ``run()``-level target. Defaults to ``NOT_PROVIDED``.
metadata : dict[str, object] | None
Optional metadata to attach to the interaction.

Returns
-------
Scenario
Self for method chaining.
Self
This scenario for method chaining.
"""
interaction = Interact(
inputs=inputs,
Expand Down Expand Up @@ -230,6 +238,17 @@ def with_annotations(self, annotations: dict[str, Any]) -> Self:

Annotations provide shared, read-only context available on the Trace
as `trace.annotations` during scenario execution.

Parameters
----------
annotations : dict[str, Any]
Key-value pairs merged into the trace at run start.
Can be accessed as `trace.annotations` during scenario execution.

Returns
-------
Self
This scenario for method chaining.
"""
self.annotations = annotations
return self
Expand All @@ -241,7 +260,18 @@ def with_target(
| ProviderType[[InputType, TraceType], OutputType]
),
) -> Self:
"""Set scenario-level target for the scenario."""
"""Set the default SUT for interactions with ``NOT_PROVIDED`` outputs.

Parameters
----------
target : ProviderType[[InputType], OutputType] | ProviderType[[InputType, TraceType], OutputType]
Callable that produces outputs given inputs (and optionally the trace).

Returns
-------
Self
This scenario for method chaining.
"""
self.target = target
return self

Expand All @@ -260,10 +290,19 @@ async def run(
- Interaction specs update the shared trace
- Checks validate the current trace and stop execution on failure

Parameters
----------
target : ProviderType[[InputType], OutputType] | ProviderType[[InputType, TraceType], OutputType] | NotProvided, optional
SUT used to replace ``NOT_PROVIDED`` outputs on ``Interact`` specs.
Defaults to ``NOT_PROVIDED``; overrides the scenario's ``target`` when set.
return_exception : bool, default False
If True, exceptions raised by checks become ``CheckResult.error``
entries instead of propagating.

Returns
-------
ScenarioResult
Results from executing the scenario.
ScenarioResult[TraceType]
Aggregated step results, timing, and final trace.
"""
# Lazy import to avoid circular dependency
from ..scenarios.runner import get_runner
Expand Down
14 changes: 10 additions & 4 deletions libs/giskard-checks/src/giskard/checks/scenarios/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ class ScenarioRunner:
```python
runner = ScenarioRunner()
result = await runner.run(scenario)
result = await runner.run(scenario, target=my_sut, return_exception=True)
```
"""

Expand All @@ -100,13 +101,18 @@ async def run[InputType, OutputType, TraceType: Trace[Any, Any]](
----------
scenario : Scenario
The scenario to execute.
return_exception : bool
If True, return results even when exceptions occur instead of raising.
target : ProviderType[[InputType], OutputType] | ProviderType[[InputType, TraceType], OutputType] | NotProvided, optional
SUT used to replace ``NOT_PROVIDED`` outputs on ``Interact`` specs.
Defaults to ``NOT_PROVIDED``; unresolved targets fall back to
``scenario.target`` when set via ``Scenario.with_target``.
return_exception : bool, default False
If True, exceptions raised by checks become ``CheckResult.error``
entries instead of propagating.

Returns
-------
ScenarioResult
Results from executing the scenario.
ScenarioResult[TraceType]
Aggregated step results, timing, and final trace.
"""

start_time = time.perf_counter()
Expand Down