fix: Add AWS environment variable checks for S3 tests (#17519)

Author: Weijun-H

datafusion-cli/src/catalog.rs

@@ -230,6 +230,8 @@ pub fn substitute_tilde(cur: String) -> String {
 }
 #[cfg(test)]
 mod tests {
+    use std::{env, vec};
+
     use super::*;
 
     use datafusion::catalog::SchemaProvider;
@@ -284,6 +286,19 @@ mod tests {
 
     #[tokio::test]
     async fn query_s3_location_test() -> Result<()> {
+        let aws_envs = vec![
+            "AWS_ENDPOINT",
+            "AWS_ACCESS_KEY_ID",
+            "AWS_SECRET_ACCESS_KEY",
+            "AWS_ALLOW_HTTP",
+        ];
+        for aws_env in aws_envs {
+            if env::var(aws_env).is_err() {
+                eprint!("aws envs not set, skipping s3 test");
+                return Ok(());
+            }
+        }
+
         let bucket = "examples3bucket";
         let location = format!("s3://{bucket}/file.parquet");
 

datafusion-cli/src/exec.rs

@@ -582,6 +582,19 @@ mod tests {
     }
     #[tokio::test]
     async fn copy_to_external_object_store_test() -> Result<()> {
+        let aws_envs = vec![
+            "AWS_ENDPOINT",
+            "AWS_ACCESS_KEY_ID",
+            "AWS_SECRET_ACCESS_KEY",
+            "AWS_ALLOW_HTTP",
+        ];
+        for aws_env in aws_envs {
+            if std::env::var(aws_env).is_err() {
+                eprint!("aws envs not set, skipping s3 test");
+                return Ok(());
+            }
+        }
+
         let locations = vec![
             "s3://bucket/path/file.parquet",
             "oss://bucket/path/file.parquet",

datafusion-cli/src/object_storage.rs

@@ -579,6 +579,12 @@ mod tests {
 
     #[tokio::test]
     async fn s3_object_store_builder_default() -> Result<()> {
+        if let Err(DataFusionError::Execution(e)) = check_aws_envs().await {
+            // Skip test if AWS envs are not set
+            eprintln!("{e}");
+            return Ok(());
+        }
+
         let location = "s3://bucket/path/FAKE/file.parquet";
         // Set it to a non-existent file to avoid reading the default configuration file
         std::env::set_var("AWS_CONFIG_FILE", "data/aws.config");
@@ -733,6 +739,11 @@ mod tests {
 
     #[tokio::test]
     async fn s3_object_store_builder_resolves_region_when_none_provided() -> Result<()> {
+        if let Err(DataFusionError::Execution(e)) = check_aws_envs().await {
+            // Skip test if AWS envs are not set
+            eprintln!("{e}");
+            return Ok(());
+        }
         let expected_region = "eu-central-1";
         let location = "s3://test-bucket/path/file.parquet";
         // Set it to a non-existent file to avoid reading the default configuration file
@@ -759,6 +770,12 @@ mod tests {
     #[tokio::test]
     async fn s3_object_store_builder_overrides_region_when_resolve_region_enabled(
     ) -> Result<()> {
+        if let Err(DataFusionError::Execution(e)) = check_aws_envs().await {
+            // Skip test if AWS envs are not set
+            eprintln!("{e}");
+            return Ok(());
+        }
+
         let original_region = "us-east-1";
         let expected_region = "eu-central-1"; // This should be the auto-detected region
         let location = "s3://test-bucket/path/file.parquet";
@@ -860,4 +877,19 @@ mod tests {
             .unwrap();
         table_options
     }
+
+    async fn check_aws_envs() -> Result<()> {
+        let aws_envs = [
+            "AWS_ACCESS_KEY_ID",
+            "AWS_SECRET_ACCESS_KEY",
+            "AWS_REGION",
+            "AWS_ALLOW_HTTP",
+        ];
+        for aws_env in aws_envs {
+            std::env::var(aws_env).map_err(|_| {
+                exec_datafusion_err!("aws envs not set, skipping s3 tests")
+            })?;
+        }
+        Ok(())
+    }
 }