fix(py): Inherit project name from parent for OpenAI agent wrapper (#2037)

CC @catherine-langchain

Author: jacoblee93

python/langsmith/__init__.py

@@ -21,7 +21,7 @@
 
 # Avoid calling into importlib on every call to __version__
 
-__version__ = "0.4.30"
+__version__ = "0.4.31"
 version = __version__  # for backwards compatibility
 
 

python/langsmith/wrappers/_openai_agents.py

@@ -96,6 +96,7 @@ class RunData(TypedDict):
         start_time: datetime
         dotted_order: str
         parent_run_id: Optional[str]
+        project_name: Optional[str]
 
     class OpenAIAgentsTracingProcessor(tracing.TracingProcessor):  # type: ignore[no-redef]
         """Tracing processor for the `OpenAI Agents SDK <https://openai.github.io/openai-agents-python/>`_.
@@ -187,10 +188,12 @@ def on_trace_start(self, trace: tracing.Trace) -> None:
             start_time = datetime.now(timezone.utc)
 
             # Use LangSmith parent run tree if available, else create new trace
+            project_name = self._project_name
             if current_run_tree is not None:
                 trace_id = str(current_run_tree.trace_id)
                 parent_run_id = str(current_run_tree.id)
                 parent_dotted_order = current_run_tree.dotted_order
+                project_name = self._project_name or current_run_tree.session_name
             else:
                 trace_id = trace_run_id
                 parent_run_id = None
@@ -207,6 +210,7 @@ def on_trace_start(self, trace: tracing.Trace) -> None:
                 start_time=start_time,
                 dotted_order=dotted_order,
                 parent_run_id=parent_run_id,
+                project_name=project_name,
             )
 
             run_extra = {"metadata": self._metadata or {}}
@@ -228,7 +232,7 @@ def on_trace_start(self, trace: tracing.Trace) -> None:
                     revision_id=None,
                     extra=run_extra,
                     tags=self._tags,
-                    project_name=self._project_name,
+                    project_name=project_name,
                 )
 
                 self.client.create_run(**run_data)
@@ -250,7 +254,7 @@ def on_trace_end(self, trace: tracing.Trace) -> None:
                         inputs=self._first_response_inputs.pop(trace.trace_id, {}),
                         outputs=self._last_response_outputs.pop(trace.trace_id, {}),
                         extra={"metadata": metadata},
-                        project_name=self._project_name,
+                        project_name=run["project_name"],
                     )
                 except Exception as e:
                     logger.exception(f"Error updating trace run: {e}")
@@ -287,6 +291,7 @@ def on_span_start(self, span: tracing.Span) -> None:
                 start_time=span_start_time,
                 dotted_order=dotted_order,
                 parent_run_id=parent_run["id"],
+                project_name=parent_run["project_name"],
             )
 
             run_name = agent_utils.get_run_name(span)
@@ -302,7 +307,7 @@ def on_span_start(self, span: tracing.Span) -> None:
                     parent_run_id=parent_run["id"],
                     dotted_order=dotted_order,
                     inputs=extracted.get("inputs", {}),
-                    project_name=self._project_name,
+                    project_name=parent_run["project_name"],
                 )
                 if span.started_at:
                     run_data["start_time"] = datetime.fromisoformat(span.started_at)
@@ -330,7 +335,7 @@ def on_span_end(self, span: tracing.Span) -> None:
                     outputs=outputs,
                     inputs=inputs,
                     extra=extracted,
-                    project_name=self._project_name,
+                    project_name=run["project_name"],
                 )
                 if span.ended_at:
                     run_data["end_time"] = datetime.fromisoformat(span.ended_at)

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "langsmith"
-version = "0.4.30"
+version = "0.4.31"
 description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."
 authors = [
     {name = "LangChain", email = "[email protected]"},

python/tests/integration_tests/wrappers/test_openai_agents.py

@@ -1,12 +1,14 @@
 import asyncio
 import json
 from unittest import mock
+from uuid import uuid4
 
 import pytest
 from agents import Agent, Runner, set_trace_processors
 
 import langsmith
 from langsmith.wrappers import OpenAIAgentsTracingProcessor
+from tests.integration_tests.test_client import safe_delete_dataset
 
 
 def _collect_trace_requests(mock_session: mock.MagicMock):
@@ -78,3 +80,85 @@ async def test_openai_agents_tracing_processor():
         event for event in all_events if event.get("name") == "Agent workflow"
     ]
     assert len(agent_runs) > 0, "No agent workflow runs found in trace"
+
+
+@pytest.mark.xfail(reason="Flaky test - may fail intermittently")
+async def test_openai_agents_with_evaluate():
+    client = langsmith.Client()
+
+    processor = OpenAIAgentsTracingProcessor(client=client)
+    set_trace_processors([processor])
+
+    agent = Agent(
+        name="Captain Obvious",
+        instructions="You are Captain Obvious, the world's"
+        + " most literal technical support agent.",
+    )
+
+    question = (
+        "Why is my code failing when I try to divide by zero?"
+        " I keep getting this error message."
+    )
+
+    # Example: Cat image validation
+    question = "What type of cat is shown in this image?"
+    provided_answer = "This is a tuxedo cat with black and white fur pattern."
+    image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/b/b7/George%2C_a_perfect_example_of_a_tuxedo_cat.jpg/1250px-George%2C_a_perfect_example_of_a_tuxedo_cat.jpg"
+
+    dataset_name = "OpenAI Agent Testing" + str(uuid4().hex[:4])
+
+    if client.has_dataset(dataset_name=dataset_name):
+        safe_delete_dataset(client, dataset_name=dataset_name)
+    try:
+        dataset = client.create_dataset(dataset_name=dataset_name)
+        client.create_examples(
+            inputs=[
+                {
+                    "question": question,
+                    "answer": provided_answer,
+                    "image_url": image_url,
+                }
+            ],
+            outputs=[{"is_correct": True}],
+            dataset_id=dataset.id,
+        )
+
+        async def run_agent(inputs: dict):
+            """Run agent and track the final response."""
+            await Runner.run(agent, question)
+            return {"result": "foo"}
+
+        async def alignment(outputs: dict, reference_outputs: dict) -> bool:
+            """Check if the agent chose the correct route."""
+            return True
+
+        experiment = await client.aevaluate(
+            run_agent,
+            data=dataset_name,
+            evaluators=[alignment],
+            experiment_prefix="agent-gpt-5-mini",
+            max_concurrency=2,
+            blocking=True,
+        )
+        await asyncio.sleep(5)
+        experiment_results = client.get_experiment_results(
+            name=experiment.experiment_name
+        )
+        assert experiment_results["stats"].run_count == 1
+        assert (
+            experiment_results["stats"].feedback_stats.get("alignment", {}).get("n")
+            == 1
+        )
+        assert (
+            experiment_results["stats"].feedback_stats.get("alignment", {}).get("avg")
+            == 1
+        )
+        examples = list(experiment_results["examples_with_runs"])
+        assert len(examples) == 1
+        run = client.read_run(examples[0].runs[0].id, load_child_runs=True)
+        assert len(run.child_runs) == 1
+        assert run.child_runs[0].name == "Agent workflow"
+        assert len(run.child_runs[0].child_runs) == 1
+        assert run.child_runs[0].child_runs[0].name == "Captain Obvious"
+    finally:
+        safe_delete_dataset(client, dataset_name=dataset_name)