chore: rework the arrow integration tests

This now relies on the features explicitly supported by the kernel crate
rather than released arrow versions which may or may not be supported

Author: rtyler

integration-tests/Cargo.toml

@@ -6,19 +6,4 @@ edition = "2021"
 [workspace]
 
 [dependencies]
-arrow = "=53.0.0"
-delta_kernel = { path = "../kernel", features = ["arrow-conversion", "arrow-expression", "default-engine", "sync-engine"] }
-
-[patch.'file:///../kernel']
-arrow = "=53.0.0"
-arrow-arith = "=53.0.0"
-arrow-array = "=53.0.0"
-arrow-buffer = "=53.0.0"
-arrow-cast = "=53.0.0"
-arrow-data = "=53.0.0"
-arrow-ord = "=53.0.0"
-arrow-json = "=53.0.0"
-arrow-select = "=53.0.0"
-arrow-schema = "=53.0.0"
-parquet = "=53.0.0"
-object_store = "=0.11.1"
+delta_kernel = { path = "../kernel", features = ["default-engine", "sync-engine"] }

integration-tests/src/main.rs

@@ -1,15 +1,16 @@
-fn create_arrow_schema() -> arrow::datatypes::Schema {
-    use arrow::datatypes::{DataType, Field, Schema};
+use delta_kernel::arrow::datatypes::{DataType, Field, Schema};
+
+fn create_arrow_schema() -> Schema {
     let field_a = Field::new("a", DataType::Int64, false);
     let field_b = Field::new("b", DataType::Boolean, false);
     Schema::new(vec![field_a, field_b])
 }
 
 fn create_kernel_schema() -> delta_kernel::schema::Schema {
-    use delta_kernel::schema::{DataType, Schema, StructField};
+    use delta_kernel::schema::{DataType, StructField};
     let field_a = StructField::not_null("a", DataType::LONG);
     let field_b = StructField::not_null("b", DataType::BOOLEAN);
-    Schema::new(vec![field_a, field_b])
+    delta_kernel::schema::Schema::new(vec![field_a, field_b])
 }
 
 fn main() {

integration-tests/test-all-arrow-versions.sh

@@ -2,38 +2,25 @@
 
 set -eu -o pipefail
 
-is_version_le() {
-    [  "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]
-}
-
-is_version_lt() {
-  if [ "$1" = "$2" ]
-  then
-    return 1
-  else
-    is_version_le "$1" "$2"
-  fi
-}
-
 test_arrow_version() {
   ARROW_VERSION="$1"
   echo "== Testing version $ARROW_VERSION =="
-  sed -i'' -e "s/\(arrow[^\"]*=[^\"]*\).*/\1\"=$ARROW_VERSION\"/" Cargo.toml
-  sed -i'' -e "s/\(parquet[^\"]*\).*/\1\"=$ARROW_VERSION\"/" Cargo.toml
   cargo clean
   rm -f Cargo.lock
   cargo update
   cat Cargo.toml
-  cargo run
+  cargo run --features ${ARROW_VERSION}
 }
 
-MIN_ARROW_VER="53.0.0"
-MAX_ARROW_VER="54.0.0"
+FEATURES=$(cat ../kernel/Cargo.toml | grep -e ^arrow_ | awk '{ print $1 }' | sort -u)
 
-for ARROW_VERSION in $(curl -s https://crates.io/api/v1/crates/arrow | jq -r '.versions[].num' | tr -d '\r')
+
+echo "[features]" >> Cargo.toml
+
+for ARROW_VERSION in ${FEATURES}
 do
-  if ! is_version_lt "$ARROW_VERSION" "$MIN_ARROW_VER" && is_version_lt "$ARROW_VERSION" "$MAX_ARROW_VER"
-  then
-    test_arrow_version "$ARROW_VERSION"
-  fi
+  echo "${ARROW_VERSION} = [\"delta_kernel/${ARROW_VERSION}\"]" >> Cargo.toml
+  test_arrow_version $ARROW_VERSION
 done
+
+git checkout Cargo.toml