Add specific error for missing required env vars in clients (BoundaryML#2570)

Issues: BoundaryML#2054 and BoundaryML#2055

Author: antoniosarosi

engine/baml-runtime/src/lib.rs

@@ -1057,7 +1057,10 @@ impl BamlRuntime {
         stream: bool,
     ) -> Result<HTTPRequest> {
         baml_log::set_from_env(&env_vars).unwrap();
-        let ctx = context_manager.create_ctx(tb, cb, env_vars, vec![])?;
+        let mut ctx = context_manager.create_ctx(tb, cb, env_vars, vec![])?;
+
+        // Called from modular API.
+        ctx.set_modular_api(true);
 
         let provider = self.llm_provider_from_function(&function_name, &ctx)?;
 

engine/baml-runtime/src/runtime/runtime_interface.rs

@@ -15,7 +15,7 @@ use internal_baml_core::{
     validate,
 };
 use internal_baml_jinja::RenderedPrompt;
-use internal_llm_client::{AllowedRoleMetadata, ClientSpec};
+use internal_llm_client::{AllowedRoleMetadata, ClientProvider, ClientSpec};
 
 use crate::{
     client_registry::ClientProperty,
@@ -105,17 +105,46 @@ impl<'a> InternalClientLookup<'a> for InternalBamlRuntime {
                     .context(format!("Could not find client with name: {client_name}"))?;
                 // Get required env vars from the client walker
                 let new_client = LLMProvider::try_from((&walker, ctx)).map(Arc::new)?;
-                // Only store the required env vars
-                let filtered_env_vars = ctx
-                    .env_vars()
-                    .iter()
-                    .filter(|(k, _)| walker.required_env_vars().contains(*k))
-                    .map(|(k, v)| (k.clone(), v.clone()))
-                    .collect();
+
+                // Collect required environment variables.
+                let mut required_env_vars = HashMap::new();
+
+                // If the client is Vertex or AWS Bedrock, we don't fail on
+                // missing required environment variables because we run a bunch
+                // of additional logic to resolve required values (i.e for
+                // Vertex if GOOGLE_CLOUD_PROJECT is not provided it can be
+                // found on the credentials property).
+                //
+                // If called with modular API, we don't fail either. User might
+                // want to insert the values later.
+                let fail_on_missing_required_env_vars = !ctx.is_modular_api()
+                    && !matches!(
+                        walker.item.elem.provider,
+                        ClientProvider::AwsBedrock | ClientProvider::Vertex
+                    );
+
+                for key in walker.required_env_vars() {
+                    if let Some(value) = ctx.env_vars().get(&key) {
+                        // Exists but is empty
+                        if fail_on_missing_required_env_vars && value.trim().is_empty() {
+                            baml_log::warn!(
+                                "Required environment variable '{key}' for client '{client_name}' is set but is empty: {key}='{value}'"
+                            );
+                        }
+                        required_env_vars.insert(key, value.to_owned());
+                    } else if fail_on_missing_required_env_vars {
+                        // It's not set and we have to fail, bail
+                        anyhow::bail!(
+                            "LLM client '{client_name}' requires environment variable '{key}' to be set but it is not"
+                        );
+                    }
+                }
+
                 clients.insert(
                     client_name.into(),
-                    CachedClient::new(new_client.clone(), filtered_env_vars),
+                    CachedClient::new(new_client.clone(), required_env_vars),
                 );
+
                 Ok(new_client)
             }
         }

engine/baml-runtime/src/types/runtime_context.rs

@@ -63,6 +63,8 @@ pub struct RuntimeContext {
     // Only the BAML_TRACER depends on this.
     pub call_id_stack: Vec<FunctionCallId>,
     pub recursive_class_overrides: Vec<IndexSet<String>>,
+    /// Called through modular API.
+    is_modular_api: bool,
 }
 
 impl RuntimeContext {
@@ -78,6 +80,14 @@ impl RuntimeContext {
         self.env.get("BOUNDARY_PROXY_URL").map(|s| s.as_str())
     }
 
+    pub fn is_modular_api(&self) -> bool {
+        self.is_modular_api
+    }
+
+    pub fn set_modular_api(&mut self, is_modular_api: bool) {
+        self.is_modular_api = is_modular_api;
+    }
+
     pub fn new(
         baml_src: Arc<BamlSrcReader>,
         env: HashMap<String, String>,
@@ -101,6 +111,7 @@ impl RuntimeContext {
             recursive_type_alias_overrides,
             call_id_stack,
             recursive_class_overrides,
+            is_modular_api: false,
         }
     }
 

integ-tests/baml_src/clients.baml

@@ -30,6 +30,16 @@ client<llm> GPT4o {
   }
 }
 
+// For integ tests
+client<llm> GPT4oBaseUrlNotSet {
+  provider openai
+  options {
+    model gpt-4o
+    api_key env.OPENAI_API_KEY
+    base_url env.OPEN_API_BASE_DO_NOT_SET_THIS
+  }
+}
+
 
 client<llm> GPT4Turbo {
   retry_policy Bar

integ-tests/baml_src/test-files/providers/openai.baml

@@ -28,9 +28,9 @@ function TestOpenAIShorthand(input: string) -> string {
   "#
 }
 
- 
- 
- 
+
+
+
 // Test standard GPT-4 (should add default max_tokens)
 function TestOpenAI(input: string) -> string {
   client GPT4
@@ -39,7 +39,7 @@ function TestOpenAI(input: string) -> string {
     Write a nice haiku, given the user input. Make sure to reference the input in the haiku. Make it 50 paragraphs
 
     Input: {{ input }}
-  "# 
+  "#
 }
 
 // Test O1 model without max_tokens (should not add default)
@@ -129,6 +129,14 @@ function TestOpenAIGPT4oMini3(input: string) -> string {
   "#
 }
 
+function OpenAIGPT4oMissingBaseUrlEnvVar(input: string) -> string {
+  client GPT4oBaseUrlNotSet
+  prompt #"
+    {{ _.role("user") }}
+    Write a nice haiku, given the user input. Make sure to reference the input in the haiku.
+  "#
+}
+
 // Add test cases to verify the behavior
 test TestOpenAIClients {
   functions [