feat(agent-spans): Display used tools (#101506)

Format available tools nicely and highlight used ones.



https://github.com/user-attachments/assets/fd8784a3-e24c-4d67-b2d7-e54f4d9655e9

Author: ArthurKnaus

static/app/views/insights/agents/utils/query.tsx

@@ -1,6 +1,6 @@
 // AI Runs - equivalent to OTEL Invoke Agent span
 // https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-agent-spans.md#invoke-agent-span
-const AI_RUN_OPS = [
+export const AI_RUN_OPS = [
   'ai.run.generateText',
   'ai.run.generateObject',
   'gen_ai.invoke_agent',
@@ -81,6 +81,10 @@ export const getAIGenerationsFilter = () => {
   return `span.op:gen_ai.* !span.op:[${joinValues(NON_GENERATION_OPS)}]`;
 };
 
+export const getToolSpansFilter = () => {
+  return `span.op:"gen_ai.execute_tool"`;
+};
+
 export const getAITracesFilter = () => {
   return `span.op:gen_ai.*`;
 };

static/app/views/insights/agents/utils/referrers.tsx

@@ -2,6 +2,7 @@ export enum Referrer {
   MODELS_TABLE = 'api.insights.agent-monitoring.models-table',
   TOOLS_TABLE = 'api.insights.agent-monitoring.tools-table',
   TRACE_DRAWER = 'api.insights.agent-monitoring.trace-drawer',
+  TRACE_DRAWER_TOOL_USAGE = 'api.insights.agent-monitoring.trace-drawer-tool-usage',
   TRACES_TABLE = 'api.insights.agent-monitoring.traces-table',
   TOKEN_USAGE_WIDGET = 'api.insights.agent-monitoring.token-usage-widget',
   MODEL_COST_WIDGET = 'api.insights.agent-monitoring.token-cost-widget',

static/app/views/performance/newTraceDetails/traceDrawer/details/highlightedAttributes.spec.tsx

@@ -25,6 +25,7 @@ describe('getHighlightedSpanAttributes', () => {
 
     getHighlightedSpanAttributes({
       op: 'gen_ai.chat',
+      spanId: '123',
       attributes,
     });
 
@@ -57,6 +58,7 @@ describe('getHighlightedSpanAttributes', () => {
 
     getHighlightedSpanAttributes({
       op: 'gen_ai.chat',
+      spanId: '123',
       attributes,
     });
 
@@ -70,6 +72,7 @@ describe('getHighlightedSpanAttributes', () => {
 
     getHighlightedSpanAttributes({
       op: 'gen_ai.chat',
+      spanId: '123',
       attributes,
     });
 
@@ -84,6 +87,7 @@ describe('getHighlightedSpanAttributes', () => {
 
     getHighlightedSpanAttributes({
       op: 'http.request',
+      spanId: '123',
       attributes,
     });
 

static/app/views/performance/newTraceDetails/traceDrawer/details/highlightedAttributes.tsx

@@ -1,15 +1,26 @@
 import styled from '@emotion/styled';
 import * as Sentry from '@sentry/react';
 
+import {Tag} from '@sentry/scraps/badge';
+import {Flex} from '@sentry/scraps/layout';
+
 import {Tooltip} from 'sentry/components/core/tooltip';
 import Count from 'sentry/components/count';
 import {StructuredData} from 'sentry/components/structuredEventData';
-import {t} from 'sentry/locale';
+import {t, tn} from 'sentry/locale';
 import {prettifyAttributeName} from 'sentry/views/explore/components/traceItemAttributes/utils';
 import type {TraceItemResponseAttribute} from 'sentry/views/explore/hooks/useTraceItemDetails';
 import {LLMCosts} from 'sentry/views/insights/agents/components/llmCosts';
 import {ModelName} from 'sentry/views/insights/agents/components/modelName';
-import {AI_CREATE_AGENT_OPS, getIsAiSpan} from 'sentry/views/insights/agents/utils/query';
+import {
+  AI_CREATE_AGENT_OPS,
+  AI_RUN_OPS,
+  getIsAiSpan,
+  getToolSpansFilter,
+} from 'sentry/views/insights/agents/utils/query';
+import {Referrer} from 'sentry/views/insights/agents/utils/referrers';
+import {useSpans} from 'sentry/views/insights/common/queries/useDiscover';
+import {SpanFields} from 'sentry/views/insights/types';
 
 type HighlightedAttribute = {
   name: string;
@@ -26,15 +37,17 @@ function tryParseJson(value: string) {
 
 export function getHighlightedSpanAttributes({
   op,
+  spanId,
   attributes = {},
 }: {
   attributes: Record<string, string> | undefined | TraceItemResponseAttribute[];
   op: string | undefined;
+  spanId: string;
 }): HighlightedAttribute[] {
   const attributeObject = ensureAttributeObject(attributes);
 
   if (getIsAiSpan({op})) {
-    return getAISpanAttributes(attributeObject, op);
+    return getAISpanAttributes({attributes: attributeObject, op, spanId});
   }
 
   if (op?.startsWith('mcp.')) {
@@ -62,10 +75,15 @@ function ensureAttributeObject(
   return attributes;
 }
 
-function getAISpanAttributes(
-  attributes: Record<string, string | number | boolean>,
-  op?: string
-) {
+function getAISpanAttributes({
+  op,
+  spanId,
+  attributes = {},
+}: {
+  attributes: Record<string, string | number | boolean>;
+  op: string | undefined;
+  spanId: string;
+}) {
   const highlightedAttributes = [];
 
   const agentName = attributes['gen_ai.agent.name'] || attributes['gen_ai.function_id'];
@@ -142,16 +160,16 @@ function getAISpanAttributes(
   }
 
   const availableTools = attributes['gen_ai.request.available_tools'];
-  if (availableTools && AI_CREATE_AGENT_OPS.includes(op!)) {
+  const toolsArray = tryParseJson(availableTools?.toString() || '');
+  if (
+    toolsArray &&
+    Array.isArray(toolsArray) &&
+    toolsArray.length > 0 &&
+    [...AI_RUN_OPS, ...AI_CREATE_AGENT_OPS].includes(op!)
+  ) {
     highlightedAttributes.push({
       name: t('Available Tools'),
-      value: (
-        <StructuredData
-          value={tryParseJson(availableTools.toString())}
-          withAnnotatedText
-          maxDefaultDepth={0}
-        />
-      ),
+      value: <HighlightedTools availableTools={toolsArray} spanId={spanId} />,
     });
   }
 
@@ -196,6 +214,61 @@ function getMCPAttributes(attributes: Record<string, string | number | boolean>)
   return highlightedAttributes;
 }
 
+function HighlightedTools({
+  availableTools,
+  spanId,
+}: {
+  availableTools: any[];
+  spanId: string;
+}) {
+  const toolNames = availableTools.map(tool => tool.name).filter(Boolean);
+  const hasToolNames = toolNames.length > 0;
+  const toolSpansQuery = useSpans(
+    {
+      search: `parent_span:${spanId} has:${SpanFields.GEN_AI_TOOL_NAME} ${getToolSpansFilter()}`,
+      fields: [SpanFields.GEN_AI_TOOL_NAME],
+      enabled: hasToolNames,
+    },
+    Referrer.TRACE_DRAWER_TOOL_USAGE
+  );
+
+  const usedTools: Map<string, number> = new Map();
+  toolSpansQuery.data?.forEach(span => {
+    const toolName = span[SpanFields.GEN_AI_TOOL_NAME];
+    usedTools.set(toolName, (usedTools.get(toolName) ?? 0) + 1);
+  });
+
+  // Fall back to showing formatted JSON if tool names cannot be parsed
+  if (!hasToolNames) {
+    return (
+      <StructuredData value={availableTools} withAnnotatedText maxDefaultDepth={0} />
+    );
+  }
+
+  return (
+    <Flex direction="row" gap="xs" wrap="wrap">
+      {toolNames.sort().map(tool => {
+        const usageCount = usedTools.get(tool) ?? 0;
+        return (
+          <Tooltip
+            key={tool}
+            disabled={toolSpansQuery.isPending}
+            title={
+              usageCount === 0
+                ? t('Not used by agent')
+                : tn('Used %s time', 'Used %s times', usageCount)
+            }
+          >
+            <Tag key={tool} type={usedTools.has(tool) ? 'info' : 'default'}>
+              {tool}
+            </Tag>
+          </Tooltip>
+        );
+      })}
+    </Flex>
+  );
+}
+
 function HighlightedTokenAttributes({
   inputTokens,
   cachedTokens,

static/app/views/performance/newTraceDetails/traceDrawer/details/span/eapSections/description.tsx

@@ -274,6 +274,7 @@ export function SpanDescription({
       hideNodeActions={hideNodeActions}
       highlightedAttributes={getHighlightedSpanAttributes({
         attributes,
+        spanId: span.event_id,
         op: span.op,
       })}
     />

static/app/views/performance/newTraceDetails/traceDrawer/details/span/sections/description.tsx

@@ -199,6 +199,7 @@ export function SpanDescription({
       hideNodeActions={hideNodeActions}
       highlightedAttributes={getHighlightedSpanAttributes({
         attributes: span.data,
+        spanId: span.span_id,
         op: span.op,
       })}
     />

static/app/views/performance/newTraceDetails/traceDrawer/details/transaction/sections/highlights.tsx

@@ -89,6 +89,7 @@ export function TransactionHighlights(props: HighlightProps) {
       hideNodeActions={props.hideNodeActions}
       highlightedAttributes={getHighlightedSpanAttributes({
         attributes: props.event.contexts.trace?.data,
+        spanId: props.node.value.span_id,
         op: props.node.value['transaction.op'],
       })}
     />