feat(replay): add feedback to LLM context

src/sentry/replays/endpoints/project_replay_summarize_breadcrumbs.py

@@ -34,7 +34,7 @@
 logger = logging.getLogger(__name__)
 
 
-class ErrorEvent(TypedDict):
+class GroupEvent(TypedDict):
     id: str
     title: str
     message: str
@@ -97,23 +97,24 @@ def get(self, request: Request, project: Project, replay_id: str) -> Response:
             error_events = []
         else:
             error_events = fetch_error_details(project_id=project.id, error_ids=error_ids)
-
         return self.paginate(
             request=request,
             paginator_cls=GenericOffsetPaginator,
             data_fn=functools.partial(fetch_segments_metadata, project.id, replay_id),
-            on_results=functools.partial(analyze_recording_segments, error_events, replay_id),
+            on_results=functools.partial(
+                analyze_recording_segments, error_events, replay_id, project.id
+            ),
         )
 
 
-def fetch_error_details(project_id: int, error_ids: list[str]) -> list[ErrorEvent]:
-    """Fetch error details given error IDs and return a list of ErrorEvent objects."""
+def fetch_error_details(project_id: int, error_ids: list[str]) -> list[GroupEvent]:
+    """Fetch error details given error IDs and return a list of GroupEvent objects."""
     try:
         node_ids = [Event.generate_node_id(project_id, event_id=id) for id in error_ids]
         events = nodestore.backend.get_multi(node_ids)
 
         return [
-            ErrorEvent(
+            GroupEvent(
                 category="error",
                 id=event_id,
                 title=data.get("title", ""),
@@ -128,7 +129,35 @@ def fetch_error_details(project_id: int, error_ids: list[str]) -> list[ErrorEven
         return []
 
 
-def generate_error_log_message(error: ErrorEvent) -> str:
+def fetch_feedback_details(feedback_id: str | None, project_id: int | None = None):
+    """
+    Fetch user feedback associated with a specific feedback event ID.
+    """
+    if project_id is None or feedback_id is None:
+        return None
+
+    try:
+        node_id = Event.generate_node_id(project_id, event_id=feedback_id)
+        event = nodestore.backend.get(node_id)
+
+        return (
+            GroupEvent(
+                category="feedback",
+                id=feedback_id,
+                title="User Feedback",
+                timestamp=event.get("timestamp", 0.0) * 1000,  # feedback timestamp is in seconds
+                message=event.get("contexts", {}).get("feedback", {}).get("message", ""),
+            )
+            if event
+            else None
+        )
+
+    except Exception as e:
+        sentry_sdk.capture_exception(e)
+        return None
+
+
+def generate_error_log_message(error: GroupEvent) -> str:
     title = error["title"]
     message = error["message"]
     timestamp = error["timestamp"]
@@ -137,15 +166,19 @@ def generate_error_log_message(error: ErrorEvent) -> str:
 
 
 def get_request_data(
-    iterator: Iterator[tuple[int, memoryview]], error_events: list[ErrorEvent]
+    iterator: Iterator[tuple[int, memoryview]],
+    error_events: list[GroupEvent],
+    project_id: int | None = None,
 ) -> list[str]:
     # Sort error events by timestamp
     error_events.sort(key=lambda x: x["timestamp"])
-    return list(gen_request_data(iterator, error_events))
+    return list(gen_request_data(iterator, error_events, project_id))
 
 
 def gen_request_data(
-    iterator: Iterator[tuple[int, memoryview]], error_events: list[ErrorEvent]
+    iterator: Iterator[tuple[int, memoryview]],
+    error_events: list[GroupEvent],
+    project_id: int | None = None,
 ) -> Generator[str]:
     """Generate log messages from events and errors in chronological order."""
     error_idx = 0
@@ -163,7 +196,7 @@ def gen_request_data(
                 error_idx += 1
 
             # Yield the current event's log message
-            if message := as_log_message(event):
+            if message := as_log_message(event, project_id):
                 yield message
 
     # Yield any remaining error messages
@@ -175,12 +208,15 @@ def gen_request_data(
 
 @sentry_sdk.trace
 def analyze_recording_segments(
-    error_events: list[ErrorEvent],
+    error_events: list[GroupEvent],
     replay_id: str,
+    project_id: int,
     segments: list[RecordingSegmentStorageMeta],
 ) -> dict[str, Any]:
     # Combine breadcrumbs and error details
-    request_data = json.dumps({"logs": get_request_data(iter_segment_data(segments), error_events)})
+    request_data = json.dumps(
+        {"logs": get_request_data(iter_segment_data(segments), error_events, project_id)}
+    )
 
     # Log when the input string is too large. This is potential for timeout.
     if len(request_data) > 100000:
@@ -194,7 +230,7 @@ def analyze_recording_segments(
     return json.loads(make_seer_request(request_data).decode("utf-8"))
 
 
-def as_log_message(event: dict[str, Any]) -> str | None:
+def as_log_message(event: dict[str, Any], project_id: int | None = None) -> str | None:
     """Return an event as a log message.
 
     Useful in AI contexts where the event's structure is an impediment to the AI's understanding
@@ -206,6 +242,14 @@ def as_log_message(event: dict[str, Any]) -> str | None:
     timestamp = event.get("timestamp", 0.0)
 
     match event_type:
+        case EventType.FEEDBACK:
+            feedback_id = event["data"]["payload"].get("data", {}).get("feedback_id", None)
+            feedback = fetch_feedback_details(feedback_id, project_id)
+            if feedback:
+                message = feedback["message"]
+                return f"User submitted feedback: '{message}' at {timestamp}"
+            else:
+                return None
         case EventType.CLICK:
             return f"User clicked on {event["data"]["payload"]["message"]} at {timestamp}"
         case EventType.DEAD_CLICK:

src/sentry/replays/usecases/ingest/event_parser.py

@@ -76,6 +76,7 @@ class EventType(Enum):
     UNKNOWN = 13
     CANVAS = 14
     OPTIONS = 15
+    FEEDBACK = 16
 
 
 def which(event: dict[str, Any]) -> EventType:
@@ -135,6 +136,8 @@ def which(event: dict[str, Any]) -> EventType:
                     return EventType.HYDRATION_ERROR
                 elif category == "replay.mutations":
                     return EventType.MUTATIONS
+                elif category == "sentry.feedback":
+                    return EventType.FEEDBACK
                 else:
                     return EventType.UNKNOWN
             elif event["data"]["tag"] == "performanceSpan":

tests/sentry/replays/test_project_replay_summarize_breadcrumbs.py

@@ -1,6 +1,6 @@
 import uuid
 import zlib
-from datetime import datetime, timezone
+from datetime import UTC, datetime, timezone
 from unittest.mock import patch
 
 import requests
@@ -11,7 +11,7 @@
 from sentry import nodestore
 from sentry.eventstore.models import Event
 from sentry.replays.endpoints.project_replay_summarize_breadcrumbs import (
-    ErrorEvent,
+    GroupEvent,
     as_log_message,
     get_request_data,
 )
@@ -56,6 +56,42 @@ def save_recording_segment(
         )
         FilestoreBlob().set(metadata, zlib.compress(data) if compressed else data)
 
+    def mock_create_feedback_occurrence(self, project_id: int, replay_id: str | None = None):
+        dt = datetime.now(UTC)
+
+        event = {
+            "project_id": project_id,
+            "event_id": "56b08cf7852c42cbb95e4a6998c66ad6",
+            "timestamp": dt.timestamp(),
+            "received": dt.isoformat(),
+            "first_seen": dt.isoformat(),
+            "user": {
+                "ip_address": "72.164.175.154",
+                "email": "[email protected]",
+                "id": 880461,
+                "isStaff": False,
+                "name": "Josh Ferge",
+            },
+            "contexts": {
+                "feedback": {
+                    "contact_email": "[email protected]",
+                    "name": "Josh Ferge",
+                    "message": "Great website!",
+                    "replay_id": replay_id,
+                    "url": "https://sentry.sentry.io/feedback/?statsPeriod=14d",
+                },
+            },
+        }
+
+        self.store_event(
+            data={
+                "event_id": event["event_id"],
+                "timestamp": event["timestamp"],
+                "contexts": event["contexts"],
+            },
+            project_id=self.project.id,
+        )
+
     @patch("sentry.replays.endpoints.project_replay_summarize_breadcrumbs.make_seer_request")
     def test_get(self, make_seer_request):
         return_value = json.dumps({"hello": "world"}).encode()
@@ -215,7 +251,7 @@ def test_get_with_error(self, make_seer_request):
         assert response.content == return_value
 
     @patch("sentry.replays.endpoints.project_replay_summarize_breadcrumbs.make_seer_request")
-    def test_get_with_error_context_disabled(self, make_seer_request):
+    def test_get_with_error_context_disabled_and_enabled(self, make_seer_request):
         """Test handling of breadcrumbs with error context disabled"""
         return_value = json.dumps({"error": "An error happened"}).encode()
         make_seer_request.return_value = return_value
@@ -261,6 +297,7 @@ def test_get_with_error_context_disabled(self, make_seer_request):
         ]
         self.save_recording_segment(0, json.dumps(data).encode())
 
+        # with error context disabled
         with self.feature(
             {
                 "organizations:session-replay": True,
@@ -280,6 +317,85 @@ def test_get_with_error_context_disabled(self, make_seer_request):
         assert response.get("Content-Type") == "application/json"
         assert response.content == return_value
 
+        # with error context enabled
+        with self.feature(
+            {
+                "organizations:session-replay": True,
+                "organizations:replay-ai-summaries": True,
+                "organizations:gen-ai-features": True,
+            }
+        ):
+            response = self.client.get(self.url, {"enable_error_context": "true"})
+
+        call_args = json.loads(make_seer_request.call_args[0][0])
+        assert "logs" in call_args
+        assert any("ZeroDivisionError" in log for log in call_args["logs"])
+        assert any("division by zero" in log for log in call_args["logs"])
+
+        assert response.status_code == 200
+        assert response.get("Content-Type") == "application/json"
+        assert response.content == return_value
+
+    @patch("sentry.replays.endpoints.project_replay_summarize_breadcrumbs.make_seer_request")
+    def test_get_with_feedback(self, make_seer_request):
+        """Test handling of breadcrumbs with user feedback"""
+        return_value = json.dumps({"feedback": "Feedback was submitted"}).encode()
+        make_seer_request.return_value = return_value
+
+        self.mock_create_feedback_occurrence(self.project.id, replay_id=self.replay_id)
+
+        now = datetime.now(timezone.utc)
+
+        self.store_replays(
+            mock_replay(
+                now,
+                self.project.id,
+                self.replay_id,
+            )
+        )
+
+        data = [
+            {
+                "type": 5,
+                "timestamp": float(now.timestamp()),
+                "data": {
+                    "tag": "breadcrumb",
+                    "payload": {"category": "console", "message": "hello"},
+                },
+            },
+            {
+                "type": 5,
+                "timestamp": float(now.timestamp()),
+                "data": {
+                    "tag": "breadcrumb",
+                    "payload": {
+                        "category": "sentry.feedback",
+                        "data": {"feedback_id": "56b08cf7852c42cbb95e4a6998c66ad6"},
+                    },
+                },
+            },
+        ]
+        self.save_recording_segment(0, json.dumps(data).encode())
+
+        with self.feature(
+            {
+                "organizations:session-replay": True,
+                "organizations:replay-ai-summaries": True,
+                "organizations:gen-ai-features": True,
+            }
+        ):
+            response = self.client.get(self.url)
+
+        make_seer_request.assert_called_once()
+        call_args = json.loads(make_seer_request.call_args[0][0])
+        assert "logs" in call_args
+        assert any("Great website!" in log for log in call_args["logs"])
+        assert any("User submitted feedback" in log for log in call_args["logs"])
+
+        assert response.status_code == 200
+        assert response.get("Content-Type") == "application/json"
+        assert response.content == return_value
+
 
 def test_get_request_data():
     def _faker():
@@ -307,14 +423,14 @@ def _faker():
         )
 
     error_events = [
-        ErrorEvent(
+        GroupEvent(
             category="error",
             id="123",
             title="ZeroDivisionError",
             timestamp=3.0,
             message="division by zero",
         ),
-        ErrorEvent(
+        GroupEvent(
             category="error",
             id="234",
             title="BadError",

tests/sentry/replays/unit/test_event_parser.py

@@ -590,6 +590,13 @@ def test_which():
     }
     assert which(event) == EventType.CONSOLE
 
+    event = {
+        "type": 5,
+        "timestamp": 0.0,
+        "data": {"tag": "breadcrumb", "payload": {"category": "sentry.feedback"}},
+    }
+    assert which(event) == EventType.FEEDBACK
+
     event = {
         "type": 5,
         "timestamp": 0.0,