feat: enable streaming usage metrics for OpenAI-compatible providers

Inject `stream_options={"include_usage": True}` when streaming and
OpenTelemetry telemetry is active. Telemetry always overrides any caller
preference to ensure complete and consistent observability metrics.

Changes:
- Add `get_stream_options_for_telemetry()` utility in openai_compat.py
- Integrate telemetry-driven stream_options injection in OpenAIMixin
  (benefits OpenAI, Bedrock, Runpod, vLLM, TGI, and 12+ other providers)
- Integrate telemetry-driven stream_options injection in LiteLLMOpenAIMixin
  (benefits WatsonX and other LiteLLM-based providers)
- Add `_litellm_extra_request_params()` hook for provider-specific params
- Remove duplicated stream_options logic from Bedrock, Runpod, WatsonX
- Comprehensive unit tests for injection behavior

Fixes #3981

Author: skamenan7

src/llama_stack/providers/remote/inference/bedrock/bedrock.py

@@ -81,14 +81,7 @@ async def openai_chat_completion(
         self,
         params: OpenAIChatCompletionRequestWithExtraBody,
     ) -> OpenAIChatCompletion | AsyncIterator[OpenAIChatCompletionChunk]:
-        """Override to enable streaming usage metrics and handle authentication errors."""
-        # Enable streaming usage metrics when telemetry is active
-        if params.stream:
-            if params.stream_options is None:
-                params.stream_options = {"include_usage": True}
-            elif "include_usage" not in params.stream_options:
-                params.stream_options = {**params.stream_options, "include_usage": True}
-
+        """Override to handle authentication errors and null responses."""
         try:
             logger.debug(f"Calling Bedrock OpenAI API with model={params.model}, stream={params.stream}")
             result = await super().openai_chat_completion(params=params)

src/llama_stack/providers/remote/inference/runpod/runpod.py

@@ -4,14 +4,7 @@
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
 
-from collections.abc import AsyncIterator
-
 from llama_stack.providers.utils.inference.openai_mixin import OpenAIMixin
-from llama_stack_api import (
-    OpenAIChatCompletion,
-    OpenAIChatCompletionChunk,
-    OpenAIChatCompletionRequestWithExtraBody,
-)
 
 from .config import RunpodImplConfig
 
@@ -29,15 +22,3 @@ class RunpodInferenceAdapter(OpenAIMixin):
     def get_base_url(self) -> str:
         """Get base URL for OpenAI client."""
         return str(self.config.base_url)
-
-    async def openai_chat_completion(
-        self,
-        params: OpenAIChatCompletionRequestWithExtraBody,
-    ) -> OpenAIChatCompletion | AsyncIterator[OpenAIChatCompletionChunk]:
-        """Override to add RunPod-specific stream_options requirement."""
-        params = params.model_copy()
-
-        if params.stream and not params.stream_options:
-            params.stream_options = {"include_usage": True}
-
-        return await super().openai_chat_completion(params)

src/llama_stack/providers/remote/inference/watsonx/watsonx.py

@@ -13,16 +13,13 @@
 from llama_stack.log import get_logger
 from llama_stack.providers.remote.inference.watsonx.config import WatsonXConfig
 from llama_stack.providers.utils.inference.litellm_openai_mixin import LiteLLMOpenAIMixin
-from llama_stack.providers.utils.inference.openai_compat import prepare_openai_completion_params
-from llama_stack.providers.utils.inference.stream_utils import wrap_async_stream
 from llama_stack_api import (
     Model,
     ModelType,
     OpenAIChatCompletion,
     OpenAIChatCompletionChunk,
     OpenAIChatCompletionRequestWithExtraBody,
     OpenAIChatCompletionUsage,
-    OpenAICompletion,
     OpenAICompletionRequestWithExtraBody,
     OpenAIEmbeddingsRequestWithExtraBody,
     OpenAIEmbeddingsResponse,
@@ -48,57 +45,25 @@ def __init__(self, config: WatsonXConfig):
             openai_compat_api_base=self.get_base_url(),
         )
 
+    def _litellm_extra_request_params(
+        self,
+        params: OpenAIChatCompletionRequestWithExtraBody | OpenAICompletionRequestWithExtraBody,
+    ) -> dict[str, Any]:
+        # These are watsonx-specific parameters used by LiteLLM.
+        return {"timeout": self.config.timeout, "project_id": self.config.project_id}
+
     async def openai_chat_completion(
         self,
         params: OpenAIChatCompletionRequestWithExtraBody,
     ) -> OpenAIChatCompletion | AsyncIterator[OpenAIChatCompletionChunk]:
         """
-        Override parent method to add timeout and inject usage object when missing.
+        Override parent method to inject usage object when missing.
+
         This works around a LiteLLM defect where usage block is sometimes dropped.
+        Note: request parameter construction (including telemetry-driven stream_options injection)
+        is handled by LiteLLMOpenAIMixin via _litellm_extra_request_params().
         """
-
-        # Add usage tracking for streaming when telemetry is active
-        stream_options = params.stream_options
-        if params.stream:
-            if stream_options is None:
-                stream_options = {"include_usage": True}
-            elif "include_usage" not in stream_options:
-                stream_options = {**stream_options, "include_usage": True}
-
-        model_obj = await self.model_store.get_model(params.model)
-
-        request_params = await prepare_openai_completion_params(
-            model=self.get_litellm_model_name(model_obj.provider_resource_id),
-            messages=params.messages,
-            frequency_penalty=params.frequency_penalty,
-            function_call=params.function_call,
-            functions=params.functions,
-            logit_bias=params.logit_bias,
-            logprobs=params.logprobs,
-            max_completion_tokens=params.max_completion_tokens,
-            max_tokens=params.max_tokens,
-            n=params.n,
-            parallel_tool_calls=params.parallel_tool_calls,
-            presence_penalty=params.presence_penalty,
-            response_format=params.response_format,
-            seed=params.seed,
-            stop=params.stop,
-            stream=params.stream,
-            stream_options=stream_options,
-            temperature=params.temperature,
-            tool_choice=params.tool_choice,
-            tools=params.tools,
-            top_logprobs=params.top_logprobs,
-            top_p=params.top_p,
-            user=params.user,
-            api_key=self.get_api_key(),
-            api_base=self.api_base,
-            # These are watsonx-specific parameters
-            timeout=self.config.timeout,
-            project_id=self.config.project_id,
-        )
-
-        result = await litellm.acompletion(**request_params)
+        result = await super().openai_chat_completion(params)
 
         # If not streaming, check and inject usage if missing
         if not params.stream:
@@ -175,49 +140,6 @@ async def _normalize_stream(
             logger.error(f"Error normalizing stream: {e}", exc_info=True)
             raise
 
-    async def openai_completion(
-        self,
-        params: OpenAICompletionRequestWithExtraBody,
-    ) -> OpenAICompletion | AsyncIterator[OpenAICompletion]:
-        """
-        Override parent method to add watsonx-specific parameters.
-        """
-        from llama_stack.providers.utils.inference.openai_compat import prepare_openai_completion_params
-
-        model_obj = await self.model_store.get_model(params.model)
-
-        request_params = await prepare_openai_completion_params(
-            model=self.get_litellm_model_name(model_obj.provider_resource_id),
-            prompt=params.prompt,
-            best_of=params.best_of,
-            echo=params.echo,
-            frequency_penalty=params.frequency_penalty,
-            logit_bias=params.logit_bias,
-            logprobs=params.logprobs,
-            max_tokens=params.max_tokens,
-            n=params.n,
-            presence_penalty=params.presence_penalty,
-            seed=params.seed,
-            stop=params.stop,
-            stream=params.stream,
-            stream_options=params.stream_options,
-            temperature=params.temperature,
-            top_p=params.top_p,
-            user=params.user,
-            suffix=params.suffix,
-            api_key=self.get_api_key(),
-            api_base=self.api_base,
-            # These are watsonx-specific parameters
-            timeout=self.config.timeout,
-            project_id=self.config.project_id,
-        )
-        result = await litellm.atext_completion(**request_params)
-
-        if params.stream:
-            return wrap_async_stream(result)  # type: ignore[arg-type]  # LiteLLM streaming types
-
-        return result  # type: ignore[return-value]  # external lib lacks type stubs
-
     async def openai_embeddings(
         self,
         params: OpenAIEmbeddingsRequestWithExtraBody,

src/llama_stack/providers/utils/inference/litellm_openai_mixin.py

@@ -7,13 +7,15 @@
 import base64
 import struct
 from collections.abc import AsyncIterator
+from typing import Any
 
 import litellm
 
 from llama_stack.core.request_headers import NeedsRequestProviderData
 from llama_stack.log import get_logger
 from llama_stack.providers.utils.inference.model_registry import ModelRegistryHelper, ProviderModelEntry
 from llama_stack.providers.utils.inference.openai_compat import (
+    get_stream_options_for_telemetry,
     prepare_openai_completion_params,
 )
 from llama_stack.providers.utils.inference.stream_utils import wrap_async_stream
@@ -180,6 +182,9 @@ async def openai_completion(
         self,
         params: OpenAICompletionRequestWithExtraBody,
     ) -> OpenAICompletion | AsyncIterator[OpenAICompletion]:
+        # Inject stream_options when streaming and telemetry is active
+        stream_options = get_stream_options_for_telemetry(params.stream_options, params.stream)
+
         if not self.model_store:
             raise ValueError("Model store is not initialized")
 
@@ -202,13 +207,14 @@ async def openai_completion(
             seed=params.seed,
             stop=params.stop,
             stream=params.stream,
-            stream_options=params.stream_options,
+            stream_options=stream_options,
             temperature=params.temperature,
             top_p=params.top_p,
             user=params.user,
             suffix=params.suffix,
             api_key=self.get_api_key(),
             api_base=self.api_base,
+            **self._litellm_extra_request_params(params),
         )
         # LiteLLM returns compatible type but mypy can't verify external library
         result = await litellm.atext_completion(**request_params)
@@ -222,14 +228,8 @@ async def openai_chat_completion(
         self,
         params: OpenAIChatCompletionRequestWithExtraBody,
     ) -> OpenAIChatCompletion | AsyncIterator[OpenAIChatCompletionChunk]:
-        # Add usage tracking for streaming when telemetry is active
-
-        stream_options = params.stream_options
-        if params.stream:
-            if stream_options is None:
-                stream_options = {"include_usage": True}
-            elif "include_usage" not in stream_options:
-                stream_options = {**stream_options, "include_usage": True}
+        # Inject stream_options when streaming and telemetry is active
+        stream_options = get_stream_options_for_telemetry(params.stream_options, params.stream)
 
         if not self.model_store:
             raise ValueError("Model store is not initialized")
@@ -265,6 +265,7 @@ async def openai_chat_completion(
             user=params.user,
             api_key=self.get_api_key(),
             api_base=self.api_base,
+            **self._litellm_extra_request_params(params),
         )
         # LiteLLM returns compatible type but mypy can't verify external library
         result = await litellm.acompletion(**request_params)
@@ -288,6 +289,20 @@ async def check_model_availability(self, model: str) -> bool:
 
         return model in litellm.models_by_provider[self.litellm_provider_name]
 
+    def _litellm_extra_request_params(
+        self,
+        params: OpenAIChatCompletionRequestWithExtraBody | OpenAICompletionRequestWithExtraBody,
+    ) -> dict[str, Any]:
+        """
+        Provider hook for extra LiteLLM/OpenAI-compat request params.
+
+        This is intentionally a narrow hook so provider adapters (e.g. WatsonX)
+        can add provider-specific kwargs (timeouts, project IDs, etc.) while the
+        mixin remains the single source of truth for telemetry-driven
+        stream_options injection.
+        """
+        return {}
+
 
 def b64_encode_openai_embeddings_response(
     response_data: list[dict], encoding_format: str | None = "float"

src/llama_stack/providers/utils/inference/openai_compat.py

@@ -235,3 +235,28 @@ def prepare_openai_embeddings_params(
         params["user"] = user
 
     return params
+
+
+def get_stream_options_for_telemetry(
+    stream_options: dict[str, Any] | None,
+    is_streaming: bool,
+) -> dict[str, Any] | None:
+    """
+    Inject stream_options when streaming and telemetry is active.
+
+    Active telemetry takes precedence over caller preference to ensure
+    complete and consistent observability metrics.
+    """
+    if not is_streaming:
+        return stream_options
+
+    from opentelemetry import trace
+
+    span = trace.get_current_span()
+    if not span or not span.is_recording():
+        return stream_options
+
+    if stream_options is None:
+        return {"include_usage": True}
+
+    return {**stream_options, "include_usage": True}

src/llama_stack/providers/utils/inference/openai_mixin.py

@@ -16,7 +16,10 @@
 from llama_stack.core.request_headers import NeedsRequestProviderData
 from llama_stack.log import get_logger
 from llama_stack.providers.utils.inference.model_registry import RemoteInferenceProviderConfig
-from llama_stack.providers.utils.inference.openai_compat import prepare_openai_completion_params
+from llama_stack.providers.utils.inference.openai_compat import (
+    get_stream_options_for_telemetry,
+    prepare_openai_completion_params,
+)
 from llama_stack.providers.utils.inference.prompt_adapter import localize_image_content
 from llama_stack_api import (
     Model,
@@ -270,6 +273,9 @@ async def openai_completion(
         """
         Direct OpenAI completion API call.
         """
+        # Inject stream_options when streaming and telemetry is active
+        stream_options = get_stream_options_for_telemetry(params.stream_options, params.stream or False)
+
         provider_model_id = await self._get_provider_model_id(params.model)
         self._validate_model_allowed(provider_model_id)
 
@@ -287,7 +293,7 @@ async def openai_completion(
             seed=params.seed,
             stop=params.stop,
             stream=params.stream,
-            stream_options=params.stream_options,
+            stream_options=stream_options,
             temperature=params.temperature,
             top_p=params.top_p,
             user=params.user,
@@ -306,6 +312,9 @@ async def openai_chat_completion(
         """
         Direct OpenAI chat completion API call.
         """
+        # Inject stream_options when streaming and telemetry is active
+        stream_options = get_stream_options_for_telemetry(params.stream_options, params.stream or False)
+
         provider_model_id = await self._get_provider_model_id(params.model)
         self._validate_model_allowed(provider_model_id)
 
@@ -346,7 +355,7 @@ async def _localize_image_url(m: OpenAIMessageParam) -> OpenAIMessageParam:
             seed=params.seed,
             stop=params.stop,
             stream=params.stream,
-            stream_options=params.stream_options,
+            stream_options=stream_options,
             temperature=params.temperature,
             tool_choice=params.tool_choice,
             tools=params.tools,