Bedrock modular api (#2526)

Modular API didn't support Bedrock because a Bedrock relies on the
aws-sdk for constructing and signing requests - HTTP requests aren't
naturally exposed to the user with that SDK.

To get around this, we take an intermediate object and serialize that to
JSON to produce something resembling an HTTP request that users can
modify.

To send the request after modification, it must be signed again. We have
new docs describing this process.

Author: imalsogreg

engine/Cargo.lock

@@ -66,6 +66,7 @@ log.workspace = true
 minijinja.workspace = true
 pin-project-lite.workspace = true
 pretty.workspace = true
+percent-encoding = "2.3.1"
 regex.workspace = true
 reqwest-eventsource = "0.6.0"
 scopeguard.workspace = true

engine/baml-runtime/Cargo.toml

@@ -1044,6 +1044,20 @@ impl BamlRuntime {
             .await
             .map(|(prompt, ..)| prompt)?;
 
+        let mut request_id = HttpRequestId::new();
+
+        if let RenderedPrompt::Chat(chat) = &prompt {
+            if let LLMProvider::Primitive(primitive) = provider.as_ref() {
+                if let internal::llm_client::primitive::LLMPrimitiveProvider::Aws(aws_client) =
+                    primitive.as_ref()
+                {
+                    return aws_client
+                        .build_modular_http_request(&ctx, chat, stream, request_id)
+                        .await;
+                }
+            }
+        }
+
         let request = match prompt {
             RenderedPrompt::Chat(chat) => provider
                 .build_request(either::Either::Right(&chat), true, stream, &ctx, self)
@@ -1064,7 +1078,7 @@ impl BamlRuntime {
         // Would also be nice if RequestBuilder had getters so we didn't have to
         // call .build()? above.
         Ok(HTTPRequest::new(
-            HttpRequestId::new(),
+            std::mem::take(&mut request_id),
             request.url().to_string(),
             request.method().to_string(),
             json_headers(request.headers()),

engine/baml-runtime/src/internal/llm_client/primitive/aws/aws_client.rs

@@ -448,6 +448,172 @@ async function run() {
 ```
 </CodeBlocks>
 
+### AWS Bedrock
+
+The modular API now returns requests for Bedrock's Converse API. You can
+modify it, sign it and forward the request with any HTTP client. A signature
+with the SignatureV4 SDK is required, we provide examples of how to do this
+below.
+
+```baml BAML {2}
+function ExtractResume(resume: string) -> Resume {
+  client Bedrock
+  // Prompt here...
+}
+```
+
+<CodeBlocks>
+```python Python
+import asyncio
+import json
+import os
+import httpx
+from botocore.auth import SigV4Auth
+from botocore.awsrequest import AWSRequest
+import boto3
+from baml_client import b
+from urllib.parse import urlsplit
+
+async def run():
+  req = await b.request.ExtractResume("John Doe | Software Engineer | BSc in CS")
+
+  body = req.body.json()
+  # Optional: append your own messages before signing.
+  body["messages"].append({
+    "role": "system",
+    "content": [{"text": "You must respond in JSON."}],
+  })
+  body_string = json.dumps(body)
+  body_bytes = body_string.encode("utf-8")
+
+  session = boto3.Session()
+  credentials = session.get_credentials().get_frozen_credentials()
+  region = (
+    req.client_details.options.get("region")
+    or os.environ.get("AWS_REGION")
+    or os.environ.get("AWS_DEFAULT_REGION")
+    or session.region_name
+    or "us-east-1"
+  )
+
+  url = urlsplit(req.url)
+
+  base_headers = {
+    key: value
+    for key, value in dict(req.headers).items()
+    if value is not None
+  }
+
+  headers = {
+    **base_headers,
+    "content-type": "application/json",
+    "accept": "application/json",
+    "host": url.netloc,
+  }
+
+  aws_request = AWSRequest(
+    method=req.method,
+    url=req.url,
+    data=body_bytes,
+    headers=headers,
+  )
+  SigV4Auth(credentials, "bedrock", region).add_auth(aws_request)
+
+  async with httpx.AsyncClient() as client:
+    response = await client.post(
+      req.url,
+      headers={key: str(value) for key, value in aws_request.headers.items()},
+      content=body_bytes,
+    )
+    if not response.is_success:
+      raise RuntimeError(
+        f"Bedrock request failed: {response.status_code} {response.text}"
+      )
+
+  payload = response.json()
+  message = payload["output"]["message"]["content"][0]["text"]
+  parsed = b.parse.ExtractResume(message)
+  print(parsed)
+
+asyncio.run(run())
+```
+
+```typescript TypeScript
+import { SignatureV4 } from "@smithy/signature-v4"
+import { fromEnv } from "@aws-sdk/credential-providers"
+import { HttpRequest } from "@smithy/protocol-http"
+import { Sha256 } from "@aws-crypto/sha256-js"
+import { b } from 'baml_client'
+
+async function run() {
+  const req = await b.request.ExtractResume("John Doe | Software Engineer | BSc in CS")
+
+  const body = req.body.json() as any
+  body.messages.push({
+    role: "user",
+    content: [{ text: "Add a short TL;DR." }],
+  })
+  const bodyString = JSON.stringify(body)
+
+  const url = new URL(req.url)
+  const region = process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION ?? "us-east-1"
+
+  const signer = new SignatureV4({
+    service: "bedrock",
+    region,
+    credentials: fromEnv(),
+    sha256: Sha256,
+  })
+
+  const baseHeaders = Object.fromEntries(
+    Object.entries(req.headers as Record<string, string | undefined>).filter(
+      ([, value]) => value !== undefined,
+    ),
+  ) as Record<string, string>
+
+  const headers = {
+    ...baseHeaders,
+    host: url.host,
+    "content-type": "application/json",
+    accept: "application/json",
+  }
+
+  const unsigned = new HttpRequest({
+    protocol: url.protocol,
+    hostname: url.hostname,
+    path: url.pathname,
+    method: req.method,
+    headers,
+    body: bodyString,
+  })
+
+  const signed = await signer.sign(unsigned)
+  const signedHeaders = Object.fromEntries(
+    Object.entries(signed.headers).map(([key, value]) => [key, String(value)]),
+  ) as Record<string, string>
+
+  const res = await fetch(req.url, {
+    method: req.method,
+    headers: signedHeaders,
+    body: bodyString,
+  })
+
+  if (!res.ok) {
+    throw new Error(`Bedrock request failed: ${res.status} ${await res.text()}`)
+  }
+
+  const payload = await res.json()
+  const message = payload.output.message.content.find((block: any) => block.text)?.text ?? ''
+  const parsed = b.parse.ExtractResume(message)
+  console.log(parsed)
+}
+```
+</CodeBlocks>
+
+> ℹ️ Streaming modular requests are not yet supported for Bedrock. Call
+> `b.request` (non-streaming) when targeting AWS, and re-sign after any
+> modifications to the body or headers.
+
 ## Type Checking
 
 ### Python

engine/baml-runtime/src/lib.rs

@@ -484,6 +484,106 @@ You can use this to modify the `region`, `access_key_id`, `secret_access_key`, a
 <Markdown src="/snippets/supports-streaming.mdx" />
 <Markdown src="/snippets/finish-reason.mdx" />
 
+## Modular API
+
+- `b.request` returns a fully signed SigV4 `HTTPRequest` pointing at the
+  Converse API.
+- Forward the request as-is. Do not mutate the headers; they already include
+  `Authorization`, `X-Amz-Date`, and (if needed) `X-Amz-Security-Token`.
+- Send the request immediately after building it. The signature is computed at
+  request time, so rebuilding gives you a fresh signature.
+- Streaming modular calls are not yet supported for Bedrock.
+
+```typescript TypeScript
+import { SignatureV4 } from "@aws-sdk/signature-v4"
+import { defaultProvider } from "@aws-sdk/credential-provider-node"
+import { HttpRequest } from "@aws-sdk/protocol-http"
+import { b } from 'baml_client'
+
+async function callBedrock() {
+  const req = await b.request.ExtractResume("John Doe | Software Engineer | BSc in CS")
+
+  const body = req.body.json() as any
+  const bodyString = JSON.stringify(body)
+  const url = new URL(req.url)
+  const region = (req.client_details.options?.region as string) ?? process.env.AWS_REGION ?? "us-east-1"
+
+  const signer = new SignatureV4({
+    service: "bedrock",
+    region,
+    credentials: defaultProvider(),
+  })
+
+  const unsigned = new HttpRequest({
+    protocol: url.protocol,
+    hostname: url.hostname,
+    path: url.pathname,
+    method: req.method,
+    headers: {
+      ...req.headers,
+      host: url.host,
+      "content-type": "application/json",
+    },
+    body: bodyString,
+  })
+
+  const signed = await signer.sign(unsigned)
+
+  const res = await fetch(req.url, {
+    method: req.method,
+    headers: signed.headers as Record<string, string>,
+    body: bodyString,
+  })
+
+  if (!res.ok) {
+    throw new Error(`Bedrock request failed: ${res.status}`)
+  }
+
+  const payload = await res.json()
+  const message = payload.output.message.content.find((block: any) => block.text)?.text ?? ''
+
+  return b.parse.ExtractResume(message)
+}
+```
+
+```python Python
+import json
+import requests
+from botocore.auth import SigV4Auth
+from botocore.awsrequest import AWSRequest
+import boto3
+from baml_client import b
+
+def call_bedrock():
+  req = b.request.ExtractResume("John Doe | Software Engineer | BSc in CS")
+
+  body = req.body.json()
+  body_bytes = json.dumps(body).encode("utf-8")
+
+  session = boto3.Session()
+  credentials = session.get_credentials().get_frozen_credentials()
+  region = req.client_details.options.get("region") or session.region_name or "us-east-1"
+
+  aws_request = AWSRequest(
+    method=req.method,
+    url=req.url,
+    data=body_bytes,
+    headers=dict(req.headers),
+  )
+  SigV4Auth(credentials, "bedrock", region).add_auth(aws_request)
+
+  response = requests.post(
+    req.url,
+    headers=dict(aws_request.headers.items()),
+    data=body_bytes,
+  )
+  response.raise_for_status()
+
+  payload = response.json()
+  message = payload["output"]["message"]["content"][0]["text"]
+  return b.parse.ExtractResume(message)
+```
+
 ## Provider request parameters
 These are other `options` that are passed through to the provider, without modification by BAML. For example if the request has a `temperature` field, you can define it in the client here so every call has that set.
 
@@ -600,4 +700,3 @@ client<llm> MyClient {
 - Use `AWS_PROFILE` to manage multiple profiles
 - Run `aws configure list` to verify configuration
 </Accordion>
-

fern/01-guide/05-baml-advanced/modular-api.mdx

@@ -46,8 +46,9 @@
         buildInputs = (with pkgs; [
           cmake
           git
-	        go
-	        gotools
+        go
+        gotools
+          mise
           openssl
           pkg-config
           lld_17

fern/03-reference/baml/clients/providers/aws-bedrock.mdx

@@ -219,7 +219,7 @@ client<llm> AwsBedrock {
     // max_completion_tokens 100000
     // model "anthropic.claude-3-5-sonnet-20240620-v1:0"
     // model_id "anthropic.claude-3-haiku-20240307-v1:0"
-    model "meta.llama3-8b-instruct-v1:0"
+    model "arn:aws:bedrock:us-east-1:404337120808:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0"
     // region "us-east-1"
     // access_key_id env.AWS_ACCESS_KEY_ID
     // secret_access_key env.AWS_SECRET_ACCESS_KEY
@@ -432,4 +432,4 @@ client<llm> AzureO3WithMaxCompletionTokens {
     api_key env.AZURE_OPENAI_API_KEY
     max_completion_tokens 1000
   }
-}
+}