feat(retry): add comprehensive tracking of all failed attempts and exceptions

instructor/core/exceptions.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any
+from typing import Any, NamedTuple
 
 
 class InstructorError(Exception):
@@ -9,6 +9,14 @@ class InstructorError(Exception):
     pass
 
 
+class FailedAttempt(NamedTuple):
+    """Represents a single failed retry attempt."""
+
+    attempt_number: int
+    exception: Exception
+    completion: Any | None = None
+
+
 class IncompleteOutputException(InstructorError):
     """Exception raised when the output from LLM is incomplete due to max tokens limit reached."""
 
@@ -34,13 +42,15 @@ def __init__(
         n_attempts: int,
         total_usage: int,
         create_kwargs: dict[str, Any] | None = None,
+        failed_attempts: list[FailedAttempt] | None = None,
         **kwargs: dict[str, Any],
     ):
         self.last_completion = last_completion
         self.messages = messages
         self.n_attempts = n_attempts
         self.total_usage = total_usage
         self.create_kwargs = create_kwargs
+        self.failed_attempts = failed_attempts or []
         super().__init__(*args, **kwargs)
 
 

instructor/core/retry.py

@@ -6,7 +6,7 @@
 from json import JSONDecodeError
 from typing import Any, Callable, TypeVar
 
-from .exceptions import InstructorRetryException, AsyncValidationError
+from .exceptions import InstructorRetryException, AsyncValidationError, FailedAttempt
 from .hooks import Hooks
 from ..mode import Mode
 from ..processing.response import (
@@ -175,6 +175,9 @@ def retry_sync(
     # Pre-extract stream flag to avoid repeated lookup
     stream = kwargs.get("stream", False)
 
+    # Track all failed attempts
+    failed_attempts: list[FailedAttempt] = []
+
     try:
         response = None
         for attempt in max_retries:
@@ -200,6 +203,15 @@ def retry_sync(
                     logger.debug(f"Parse error: {e}")
                     hooks.emit_parse_error(e)
 
+                    # Track this failed attempt
+                    failed_attempts.append(
+                        FailedAttempt(
+                            attempt_number=attempt.retry_state.attempt_number,
+                            exception=e,
+                            completion=response,
+                        )
+                    )
+
                     # Check if this is the last attempt
                     if isinstance(max_retries, Retrying) and hasattr(
                         max_retries, "stop"
@@ -231,6 +243,15 @@ def retry_sync(
                     logger.debug(f"Completion error: {e}")
                     hooks.emit_completion_error(e)
 
+                    # Track this failed attempt
+                    failed_attempts.append(
+                        FailedAttempt(
+                            attempt_number=attempt.retry_state.attempt_number,
+                            exception=e,
+                            completion=response,
+                        )
+                    )
+
                     # Check if this is the last attempt for completion errors
                     if isinstance(max_retries, Retrying) and hasattr(
                         max_retries, "stop"
@@ -261,6 +282,7 @@ def retry_sync(
             ),  # Use the optimized function instead of nested lookups
             create_kwargs=kwargs,
             total_usage=total_usage,
+            failed_attempts=failed_attempts,
         ) from e
 
 
@@ -304,6 +326,9 @@ async def retry_async(
     # Pre-extract stream flag to avoid repeated lookup
     stream = kwargs.get("stream", False)
 
+    # Track all failed attempts
+    failed_attempts: list[FailedAttempt] = []
+
     try:
         response = None
         async for attempt in max_retries:
@@ -333,6 +358,15 @@ async def retry_async(
                     logger.debug(f"Parse error: {e}")
                     hooks.emit_parse_error(e)
 
+                    # Track this failed attempt
+                    failed_attempts.append(
+                        FailedAttempt(
+                            attempt_number=attempt.retry_state.attempt_number,
+                            exception=e,
+                            completion=response,
+                        )
+                    )
+
                     # Check if this is the last attempt
                     if isinstance(max_retries, AsyncRetrying) and hasattr(
                         max_retries, "stop"
@@ -364,6 +398,15 @@ async def retry_async(
                     logger.debug(f"Completion error: {e}")
                     hooks.emit_completion_error(e)
 
+                    # Track this failed attempt
+                    failed_attempts.append(
+                        FailedAttempt(
+                            attempt_number=attempt.retry_state.attempt_number,
+                            exception=e,
+                            completion=response,
+                        )
+                    )
+
                     # Check if this is the last attempt for completion errors
                     if isinstance(max_retries, AsyncRetrying) and hasattr(
                         max_retries, "stop"
@@ -394,4 +437,5 @@ async def retry_async(
             ),  # Use the optimized function instead of nested lookups
             create_kwargs=kwargs,
             total_usage=total_usage,
+            failed_attempts=failed_attempts,
         ) from e