feat: datafusion table provider next

.github/workflows/build.yml

@@ -36,7 +36,7 @@ jobs:
         uses: actions-rust-lang/rustfmt@v1
 
       - name: build and lint with clippy
-        run: cargo clippy --features ${{ env.DEFAULT_FEATURES }} --tests
+        run: cargo clippy --profile=ci --features ${{ env.DEFAULT_FEATURES }} --tests
 
       - name: Run Machete to check for unused dependencies
         uses: bnjbvr/cargo-machete@aa070c61641e482bc2b5d41c97b496ee4755bf37
@@ -83,16 +83,16 @@ jobs:
           rustflags: ""
 
       - name: Default build
-        run: cargo build --package deltalake
+        run: cargo build --profile=ci --package deltalake
 
       - name: Default features with clippy
-        run: cargo clippy --features ${{ env.DEFAULT_FEATURES }}
+        run: cargo clippy --profile=ci --features ${{ env.DEFAULT_FEATURES }}
 
       - name: Spot-check build for native-tls features
-        run: cargo build --no-default-features --features azure,datafusion,s3-native-tls,gcs,glue
+        run: cargo build --profile=ci --no-default-features --features azure,datafusion,s3-native-tls,gcs,glue
 
       - name: Check no default features (except rustls)
-        run: cargo check --no-default-features --features rustls
+        run: cargo check --profile=ci --no-default-features --features rustls
 
   test:
     name: Unit Tests

.github/workflows/integration.yml

@@ -11,11 +11,9 @@ env:
   # https://github.com/rust-lang/cargo/issues/10280
   CARGO_NET_GIT_FETCH_WITH_CLI: "true"
   DEFAULT_FEATURES: "azure,datafusion,s3,gcs,glue,hdfs "
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  RUSTFLAGS: -C debuginfo=line-tables-only
-  # Disable incremental builds by cargo for CI which should save disk space
-  # and hopefully avoid final link "No space left on device"
-  CARGO_INCREMENTAL: 0
+  # Override RUSTFLAGS to not treat warnings as errors (setup-rust-toolchain sets -D warnings by default)
+  # This allows deprecated methods to be warnings instead of errors
+  RUSTFLAGS: ""
   SCCACHE_GHA_ENABLED: "true"
   RUSTC_WRAPPER: "sccache"
 
@@ -63,7 +61,7 @@ jobs:
       - name: Run tests with rustls (default)
         run: |
           gmake setup-dat
-          cargo llvm-cov \
+          cargo llvm-cov --profile=ci \
             --package deltalake-${{ matrix.target.name }} \
             --features integration_test \
             --codecov \
@@ -109,7 +107,7 @@ jobs:
       - name: Run tests with native-tls
         run: |
           gmake setup-dat
-          cargo test \
+          cargo test --profile=ci \
             --package deltalake-aws \
             --no-default-features \
             --features integration_test,native-tls
@@ -147,7 +145,7 @@ jobs:
       - name: Run tests with rustls (default)
         run: |
           gmake setup-dat
-          cargo llvm-cov \
+          cargo llvm-cov --profile=ci \
             --features integration_test \
             --package deltalake-hdfs \
             --codecov \
@@ -192,7 +190,7 @@ jobs:
       - name: Run tests
         run: |
           gmake setup-dat
-          cargo llvm-cov \
+          cargo llvm-cov --profile=ci \
             --package deltalake-lakefs \
             --features integration_test_lakefs \
             --codecov \

.github/workflows/python_benchmark.yml

@@ -13,9 +13,6 @@ jobs:
   benchmark:
     name: Python Benchmark
     runs-on: ubuntu-latest
-    env:
-      RUSTFLAGS: "-C debuginfo=line-tables-only"
-      CARGO_INCREMENTAL: 0
 
     steps:
       - uses: actions/checkout@v5
@@ -25,7 +22,7 @@ jobs:
 
       - name: Build deltalake in release mode
         run: |
-          MATURIN_EXTRA_ARGS=--release make develop
+          PROFILE=ci make develop
 
       # Download previous benchmark result from cache (if exists)
       - name: Download previous benchmark data

.github/workflows/python_release.yml

@@ -45,7 +45,7 @@ jobs:
         with:
           target: ${{ matrix.target }}
           command: publish
-          args: --skip-existing -m python/Cargo.toml --no-sdist
+          args: --skip-existing -m python/Cargo.toml --no-sdist --profile python-release
 
   release-pypi-windows:
     needs: validate-release-tag
@@ -65,7 +65,7 @@ jobs:
         with:
           target: x86_64-pc-windows-msvc
           command: publish
-          args: --skip-existing -m python/Cargo.toml --no-sdist
+          args: --skip-existing -m python/Cargo.toml --no-sdist --profile python-release
 
   release-pypi-manylinux-x86-64:
     needs: validate-release-tag
@@ -83,10 +83,9 @@ jobs:
         env:
           MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
         with:
-          maturin-version: v1.6.0 # https://github.com/PyO3/maturin/issues/2154
           target: x86_64-unknown-linux-gnu
           command: publish
-          args: --skip-existing -m python/Cargo.toml
+          args: --skip-existing -m python/Cargo.toml --profile python-release
 
   release-pypi-manylinux-228-aarch64:
     needs: validate-release-tag
@@ -106,7 +105,7 @@ jobs:
         with:
           target: aarch64-unknown-linux-gnu
           command: publish
-          args: --skip-existing -m python/Cargo.toml --no-sdist
+          args: --skip-existing -m python/Cargo.toml --no-sdist --profile python-release
           manylinux: "2_28"
 
   release-pypi-musl-x86-64:
@@ -125,10 +124,9 @@ jobs:
         env:
           MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
         with:
-          maturin-version: v1.6.0 # https://github.com/PyO3/maturin/issues/2154
           target: x86_64-unknown-linux-musl
           command: publish
-          args: --skip-existing -m python/Cargo.toml --zig
+          args: --skip-existing -m python/Cargo.toml --zig --profile python-release
 
   release-docs:
     needs:

CHANGELOG.md

@@ -20,6 +20,7 @@
 
 **Fixed bugs:**
 
+- Automatically register the AWS, Azure, GCS, HDFS, LakeFS, and Unity storage handlers when the corresponding feature is enabled so `DeltaOps::try_from_uri` no longer errors with unknown schemes such as `gs://`.
 - Significant performance regression when opening S3 table on next branch [\#3667](https://github.com/delta-io/delta-rs/issues/3667)
 - Concurrent overwrite doesn't fail conflict checking [\#3622](https://github.com/delta-io/delta-rs/issues/3622)
 - source distributions missing in v1.1.1 [\#3621](https://github.com/delta-io/delta-rs/issues/3621)

Cargo.toml

@@ -15,16 +15,6 @@ license = "Apache-2.0"
 documentation = "https://docs.rs/deltalake"
 repository = "https://github.com/delta-io/delta.rs"
 
-[profile.release-with-debug]
-inherits = "release"
-debug = true
-
-# Reducing the debuginfo for the test profile in order to trim the disk and RAM
-# usage during development
-# <https://github.com/delta-io/delta-rs/issues/1550?
-[profile.test]
-debug = "line-tables-only"
-
 [workspace.dependencies]
 delta_kernel = { version = "0.16.0", features = [
     "arrow-56",
@@ -73,6 +63,7 @@ uuid = { version = "1" }
 async-trait = { version = "0.1" }
 futures = { version = "0.3" }
 tokio = { version = "1" }
+typed-builder = { version = "0.23.0" }
 
 # opentelemetry
 tracing-opentelemetry = { version = "0.32" }
@@ -103,10 +94,54 @@ AKS = "AKS"
 # to avoid using 'type' as a field name.
 tpe = "tpe"
 
-# for better flamegraphs when benchmarking
+# ============================================================================
+# Profile Configuration
+# ============================================================================
+# Profiles for different use cases:
+# - dev: Fast local development with good debug experience (Cargo defaults)
+# - test: test builds with minimal debug info to save disk/RAM (custom)
+# - release: Production Rust crates - maximum performance (Cargo defaults)
+# - bench: For benchmarking with flamegraphs (cargo bench)
+# - profiling: For performance profiling with release opts + debug info
+# - ci: CI/CD optimized - fast builds, release-like performance
+# - python-release: Python wheel builds - portable, reproducible (PyPI releases)
+# ============================================================================
+
+# Test profile - reduced debug info to save disk/RAM
+# Usage: cargo test (automatic)
+# <https://github.com/delta-io/delta-rs/issues/1550>
+[profile.test]
+debug = "line-tables-only"
+
+# Benchmark profile with debug symbols for flamegraphs
+# Usage: cargo bench (automatic)
 [profile.bench]
 debug = true
 
+# Profiling profile with release optimizations + debug info
+# Usage: cargo build --profile=profiling
 [profile.profiling]
 inherits = "release"
 debug = true
+
+# CI profile - optimized for Continuous Integration
+# Usage: cargo build --profile=ci
+[profile.ci]
+inherits = "dev"           
+debug = false              
+incremental = false        
+
+[profile.ci.package."*"]
+opt-level = 3              
+debug = false             
+debug-assertions = false   
+strip = "debuginfo"       
+incremental = false
+
+# Python wheel release profile - for PyPI distribution
+# Usage: maturin build --profile python-release (via .github/workflows/python_release.yml)
+[profile.python-release]
+inherits = "release"
+opt-level = 3
+codegen-units = 1
+lto = "fat"         

Makefile

@@ -8,12 +8,6 @@
 DAT_VERSION := 0.0.3
 DEFAULT_FEATURES := "azure,datafusion,s3,gcs,glue,hdfs"
 
-# Disable full debug symbol generation to speed up CI build and keep memory down
-export RUSTFLAGS:= -C debuginfo=line-tables-only
-# Disable incremental builds by cargo for CI which should save disk space
-# and hopefully avoid final link "No space left on device"
-export CARGO_INCREMENTAL:=0
-
 ## begin dat related
 ####################
 .PHONY: setup-dat

crates/aws/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltalake-aws"
-version = "0.12.0"
+version = "0.12.1"
 authors.workspace = true
 keywords.workspace = true
 readme.workspace = true
@@ -25,14 +25,15 @@ object_store = { workspace = true, features = ["aws"] }
 regex = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
+typed-builder = { workspace = true }
 uuid = { workspace = true, features = ["serde", "v4"] }
 url = { workspace = true }
 
 # crates.io dependencies
 aws-smithy-runtime-api = { version = "1.8" }
 aws-smithy-runtime = { version = "1.8", optional = true }
 aws-credential-types = { version = "1.2", features = ["hardcoded-credentials"] }
-aws-config = { version = "1.8", default-features = false, features = [
+aws-config = { version = "1.8", features = [
     "behavior-version-latest",
     "rt-tokio",
     "credentials-process",

crates/aws/src/constants.rs

@@ -162,3 +162,7 @@ pub static CONDITION_DELETE_INCOMPLETE: LazyLock<String> = LazyLock::new(|| {
 
 pub const CONDITION_UPDATE_INCOMPLETE: &str = "complete = :f";
 pub const DEFAULT_COMMIT_ENTRY_EXPIRATION_DELAY: Duration = Duration::from_secs(86_400);
+
+pub const DEFAULT_S3_POOL_IDLE_TIMEOUT_SECONDS: u64 = 15;
+pub const DEFAULT_STS_POOL_IDLE_TIMEOUT_SECONDS: u64 = 10;
+pub const DEFAULT_S3_GET_INTERNAL_SERVER_ERROR_RETRIES: usize = 10;