[Ballista] Fix Ballista IT (#5)

Author: andygrove

.dockerignore

@@ -21,42 +21,6 @@
 # This setup requires to all of our docker containers have arrow's source
 # as a mounted directory.
 
-# exclude everything
-**
-
-# include explicitly
-!ci/**
-!c_glib/Gemfile
-!dev/archery/requirements*.txt
-!python/requirements*.txt
-!python/manylinux1/**
-!python/manylinux2010/**
-!r/DESCRIPTION
-!ruby/Gemfile
-!ruby/red-arrow/Gemfile
-!ruby/red-arrow/lib/arrow/version.rb
-!ruby/red-arrow/red-arrow.gemspec
-!ruby/red-arrow-cuda/Gemfile
-!ruby/red-arrow-cuda/lib/arrow-cuda/version.rb
-!ruby/red-arrow-cuda/red-arrow-cuda.gemspec
-!ruby/red-gandiva/Gemfile
-!ruby/red-gandiva/lib/gandiva/version.rb
-!ruby/red-gandiva/red-gandiva.gemspec
-!ruby/red-parquet/Gemfile
-!ruby/red-parquet/lib/parquet/version.rb
-!ruby/red-parquet/red-parquet.gemspec
-!ruby/red-plasma/Gemfile
-!ruby/red-plasma/lib/plasma/version.rb
-!ruby/red-plasma/red-plasma.gemspec
-!rust/Cargo.toml
-!rust/benchmarks/Cargo.toml
-!rust/arrow/Cargo.toml
-!rust/arrow/benches
-!rust/arrow-flight/Cargo.toml
-!rust/parquet/Cargo.toml
-!rust/parquet/build.rs
-!rust/parquet_derive/Cargo.toml
-!rust/parquet_derive_test/Cargo.toml
-!rust/datafusion/Cargo.toml
-!rust/datafusion/benches
-!rust/integration-testing/Cargo.toml
+ci
+dev
+**/target/*

ballista/.dockerignore

@@ -18,4 +18,4 @@
 # under the License.
 BALLISTA_VERSION=0.4.2-SNAPSHOT
 set -e
-docker build -t ballistacompute/rust-base:$BALLISTA_VERSION -f docker/rust-base.dockerfile .
+docker build -t ballistacompute/rust-base:$BALLISTA_VERSION -f dev/docker/rust-base.dockerfile .

ballista/docker/README.md

@@ -21,4 +21,4 @@ BALLISTA_VERSION=0.4.2-SNAPSHOT
 
 set -e
 
-docker build -t ballistacompute/ballista-rust:$BALLISTA_VERSION -f docker/rust.dockerfile .
+docker build -t ballistacompute/ballista-rust:$BALLISTA_VERSION -f dev/docker/rust.dockerfile .

ballista/dev/build-rust-base.sh renamed to dev/build-rust-base.sh

@@ -23,7 +23,7 @@
 
 
 # Base image extends debian:buster-slim
-FROM rust:1.49.0-buster AS builder
+FROM rust:1.51.0-buster AS builder
 
 RUN apt update && apt -y install musl musl-dev musl-tools libssl-dev openssl
 

ballista/dev/build-rust.sh renamed to dev/build-rust.sh

@@ -22,39 +22,57 @@
 # as a mounted directory.
 
 ARG RELEASE_FLAG=--release
-FROM ballistacompute/rust-base:0.4.0-20210213 AS base
+FROM ballistacompute/rust-base:0.4.2-SNAPSHOT AS base
 WORKDIR /tmp/ballista
 RUN apt-get -y install cmake
 RUN cargo install cargo-chef 
 
 FROM base as planner
-COPY rust .
+RUN mkdir /tmp/ballista/ballista
+RUN mkdir /tmp/ballista/benchmarks
+RUN mkdir /tmp/ballista/datafusion
+RUN mkdir /tmp/ballista/datafusion-examples
+ADD Cargo.toml .
+COPY benchmarks ./benchmarks/
+COPY datafusion ./datafusion/
+COPY datafusion-examples ./datafusion-examples/
+COPY ballista ./ballista/
 RUN cargo chef prepare --recipe-path recipe.json
 
 FROM base as cacher
 COPY --from=planner /tmp/ballista/recipe.json recipe.json
 RUN cargo chef cook $RELEASE_FLAG --recipe-path recipe.json
 
 FROM base as builder
-COPY rust .
+RUN mkdir /tmp/ballista/ballista
+RUN mkdir /tmp/ballista/benchmarks
+RUN mkdir /tmp/ballista/datafusion
+RUN mkdir /tmp/ballista/datafusion-examples
+ADD Cargo.toml .
+COPY benchmarks ./benchmarks/
+COPY datafusion ./datafusion/
+COPY ballista ./ballista/
+COPY datafusion-examples ./datafusion-examples/
 COPY --from=cacher /tmp/ballista/target target
 ARG RELEASE_FLAG=--release
 
 # force build.rs to run to generate configure_me code.
 ENV FORCE_REBUILD='true'
 RUN cargo build $RELEASE_FLAG
+RUN cd ballista/rust && \
+    cargo build $RELEASE_FLAG
 
 # put the executor on /executor (need to be copied from different places depending on FLAG)
 ENV RELEASE_FLAG=${RELEASE_FLAG}
-RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-executor /executor; else mv /tmp/ballista/target/release/ballista-executor /executor; fi
+RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/ballista/rust/target/debug/ballista-executor /executor; else mv /tmp/ballista/ballista/rust/target/release/ballista-executor /executor; fi
 
 # put the scheduler on /scheduler (need to be copied from different places depending on FLAG)
 ENV RELEASE_FLAG=${RELEASE_FLAG}
-RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/ballista-scheduler /scheduler; else mv /tmp/ballista/target/release/ballista-scheduler /scheduler; fi
+RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/ballista/rust/target/debug/ballista-scheduler /scheduler; else mv /tmp/ballista/ballista/rust/target/release/ballista-scheduler /scheduler; fi
 
 # put the tpch on /tpch (need to be copied from different places depending on FLAG)
 ENV RELEASE_FLAG=${RELEASE_FLAG}
-RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/target/debug/tpch /tpch; else mv /tmp/ballista/target/release/tpch /tpch; fi
+RUN if [ -z "$RELEASE_FLAG" ]; then mv /tmp/ballista/ballista/rust/target/debug/tpch /tpch; else mv /tmp/ballista/ballista/rust/target/release/tpch /tpch; fi
 
 # Copy the binary into a new container for a smaller docker image
 FROM ballistacompute/rust-base:0.4.0-20210213

ballista/docker/rust-base.dockerfile renamed to dev/docker/rust-base.dockerfile

@@ -17,8 +17,9 @@
 # specific language governing permissions and limitations
 # under the License.
 set -e
+./dev/build-rust-base.sh
 ./dev/build-rust.sh
-pushd rust/benchmarks/tpch
+pushd ballista/rust/benchmarks/tpch
 ./tpch-gen.sh
 
 docker-compose up -d