[WIP] Bazel fixes, Bazel CI, Dev Containers

.bazelrc

@@ -1,3 +1,6 @@
+try-import %workspace%/.circleci.bazelrc
+try-import %workspace%/.maven.bazelrc
+
 ## Disable remote cache completely when --config=local is passed
 build:local --remote_cache=
 
@@ -19,7 +22,7 @@ common:spark_3.1 --define spark_version=3.1
 common:spark_3.2 --define spark_version=3.2
 common:spark_3.5 --define spark_version=3.5
 # Default Spark version
-common --define spark_version=3.1
+common --define spark_version=3.2
 
 # Flink versions
 common:flink_1.16 --define flink_version=1.16

.bazelversion

@@ -0,0 +1 @@
+6.4.0

.circleci/Dockerfile

@@ -34,9 +34,12 @@ RUN apt-get update && apt-get -y -q install \
     openjdk-8-jdk \
     pkg-config \
     sbt \
-    bazelisk \
     && apt-get clean
 
+# Install bazelisk directly from GitHub releases
+RUN curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-amd64 \
+    && chmod +x /usr/local/bin/bazel
+
 # Install thrift
 RUN curl -sSL "http://archive.apache.org/dist/thrift/$THRIFT_VERSION/thrift-$THRIFT_VERSION.tar.gz" -o thrift.tar.gz \
 	&& mkdir -p /usr/src/thrift \

.circleci/config.yml

@@ -195,6 +195,42 @@ jobs:
             destination: spark_warehouse.tar.gz
             when: on_fail
 
+    "Bazel Tests":
+        executor: docker_baseimg_executor_xxlarge
+        steps:
+            - checkout
+            # TODO build/publish image
+            # Airbnb would have to set up/own this
+            - run:
+                name: Run Bazel Setup
+                command: |
+                    # Add Bazel GPG key and repository
+                    curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor | sudo tee /usr/share/keyrings/bazel-archive-keyring.gpg > /dev/null
+                    echo "deb [arch=amd64 signed-by=/usr/share/keyrings/bazel-archive-keyring.gpg] https://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list
+                    # Update package list and install specific Bazel version
+                    sudo apt update
+                    sudo apt install -y bazel-6.4.0
+                    # Set up bazel-6.4.0 as the default bazel command
+                    sudo update-alternatives --install /usr/bin/bazel bazel /usr/bin/bazel-6.4.0 100
+            - run:
+            # TODO remote cache
+            # Airbnb would have to set up/own this
+                name: Run Bazel Tests
+                shell: /bin/bash -leuxo pipefail
+                command: |
+                    sudo mkdir -p /root/.cache/bazel
+                    useradd --create-home --shell /bin/bash nonroot
+                    chown -R nonroot:nonroot /chronon
+                    chown -R nonroot:nonroot /root/.cache/bazel # Give access to the bazel cache
+                    export JAVA_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=4G -Xmx4G -Xms2G"
+                    sudo -u nonroot -H -s /bin/bash -c "bazel version && bazel build //... && bazel test //... --test_output=streamed --test_summary=detailed --local_ram_resources=HOST_RAM*0.7 --local_cpu_resources=HOST_CPUS*0.7"  
+            - store_test_results:
+                path: bazel-testlogs
+            - store_artifacts:
+                path: bazel-testlogs
+                destination: bazel-test-logs
+                when: on_fail
+
 workflows:
     build_test_deploy:
         jobs:
@@ -222,4 +258,5 @@ workflows:
                       - "Pull Docker Image"
             - "Scala 13 -- Iceberg Table Utils Tests":
                   requires:
-                      - "Pull Docker Image"
+                      - "Pull Docker Image"
+            - "Bazel Tests"

.devcontainer/devcontainer.json

@@ -0,0 +1,25 @@
+{
+  "name": "Chronon Development",
+  "image": "mcr.microsoft.com/devcontainers/base:ubuntu-22.04",
+  "features": {
+    "ghcr.io/devcontainers/features/java:1": {
+      "version": "11"
+    },
+    "ghcr.io/devcontainers/features/python:1": {
+      "version": "3.10"
+    }
+  },
+  "postCreateCommand": ".devcontainer/setup.sh",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "scalameta.metals",
+        "ms-python.python",
+        "ms-python.pylint",
+        "bazelbuild.vscode-bazel"
+      ]
+    }
+  },
+  "forwardPorts": [8080, 3000],
+  "remoteUser": "vscode"
+}

.devcontainer/install_thrift.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -efx
+
+# Clean up any existing thrift_build directory
+rm -rf thrift_build
+mkdir thrift_build
+pushd thrift_build
+
+# Download archive and verify it matches our expected checksum.
+THRIFT_HTTP_ARCHIVE=https://archive.apache.org/dist/thrift/0.13.0/thrift-0.13.0.tar.gz
+THRIFT_ARCHIVE=thrift.tar.gz
+THRIFT_EXPECTED_CHECKSUM_SHA256=7ad348b88033af46ce49148097afe354d513c1fca7c607b59c33ebb6064b5179
+curl "$THRIFT_HTTP_ARCHIVE" -o "$THRIFT_ARCHIVE"
+THRIFT_ACTUAL_CHECKSUM_SHA256=$(sha256sum "$THRIFT_ARCHIVE" | awk '{ print $1 }')
+if [ "$THRIFT_EXPECTED_CHECKSUM_SHA256" != "$THRIFT_ACTUAL_CHECKSUM_SHA256" ]; then
+    echo "Checksum does not match expected value" >&2
+    echo " - location: $THRIFT_HTTP_ARCHIVE" >&2
+    echo " - expected: $THRIFT_EXPECTED_CHECKSUM_SHA256" >&2
+    echo " - obtained: $THRIFT_ACTUAL_CHECKSUM_SHA256" >&2
+    exit 1
+fi
+
+echo "Building Thrift from source"
+# Build thrift from source.
+mkdir src 
+tar zxvf thrift.tar.gz -C src --strip-components=1
+pushd src
+
+# Install build dependencies
+sudo apt update
+sudo apt install -y build-essential libssl-dev pkg-config flex bison
+
+# Configure and build
+./configure --without-python --without-cpp
+make
+
+# Install
+sudo make install
+
+popd
+
+# Verify installation
+thrift -version
+
+popd
+
+echo "Thrift 0.13.0 installation completed successfully"

.devcontainer/setup.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Install Bazel 6.4.0 (required by this project)
+echo "Installing Bazel 6.4.0..."
+
+# Add Bazel GPG key and repository
+curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor | sudo tee /usr/share/keyrings/bazel-archive-keyring.gpg > /dev/null
+echo "deb [arch=amd64 signed-by=/usr/share/keyrings/bazel-archive-keyring.gpg] https://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list
+
+# Update package list and install specific Bazel version
+sudo apt update
+sudo apt install -y bazel-6.4.0
+
+# Set up bazel-6.4.0 as the default bazel command
+sudo update-alternatives --install /usr/bin/bazel bazel /usr/bin/bazel-6.4.0 100
+
+# Install Thrift compiler (0.13.0 to match project requirements)
+echo "Installing Thrift 0.13.0..."
+./.devcontainer/install_thrift.sh
+
+# Verify installations
+echo "Verifying installations..."
+bazel version
+python --version
+java -version
+
+echo "Setup complete!"

.gitignore

@@ -31,7 +31,10 @@ cs
 *.venv
 # Documentation builds
 docs/build/
-
+thrift_build/*
+project/*
+.metals/*
+.claude/*
 # Python distribution and packaging
 api/py/dist/
 api/py/eggs/

.ijwb/.bazelproject

@@ -1,16 +1 @@
-directories:
-  # Add the directories you want added as source here
-  # By default, we've added your entire workspace ('.')
-  .
-
-# Automatically includes all relevant targets under the 'directories' above
-derive_targets_from_directories: true
-
-targets:
-  # If source code isn't resolving, add additional targets that compile it here
-
-additional_languages:
-  # Uncomment any additional languages you want supported
-  python
-  scala
-  java
+import tools/ide_support/intellij/default_view.bazelproject

.jvmopts

@@ -0,0 +1,4 @@
+-Xms2G
+-Xmx4G
+-XX:+CMSClassUnloadingEnabled 
+-XX:MaxPermSize=4G

WORKSPACE

@@ -123,6 +123,12 @@ load("@io_bazel_rules_scala//scala:toolchains.bzl", "scala_register_toolchains")
 
 scala_register_toolchains()
 
+load("@io_bazel_rules_scala//testing:junit.bzl", "junit_repositories", "junit_toolchain")
+
+junit_repositories()
+
+junit_toolchain()
+
 load("@io_bazel_rules_scala//testing:scalatest.bzl", "scalatest_repositories", "scalatest_toolchain")
 
 scalatest_repositories()

aggregator/BUILD.bazel

@@ -57,13 +57,13 @@ scala_library(
            ]),
 )
 
-scala_test_suite(
+scala_junit_test(
     name = "test",
     srcs = glob(["src/test/scala/ai/chronon/aggregator/test/*.scala"]),
+    suffixes = ["Test"],
     visibility = ["//visibility:public"],
     deps = [
                ":aggregator",
-               ":test-lib",
                "//api:api-lib",
                "//api:api-models",
                maven_artifact("junit:junit"),
@@ -100,9 +100,9 @@ genrule(
 
 java_export(
     name = "aggregator-export",
-    maven_coordinates = "ai.chronon:aggregator_(scala_version):$(version)",
+    maven_coordinates = "ai.chronon:aggregator_$(scala_version):$(version)",
     pom_template = ":generate_pom",
     runtime_deps = [
         ":aggregator",
     ],
-)
+)

aggregator/src/test/scala/ai/chronon/aggregator/test/ApproxDistinctTest.scala

@@ -19,8 +19,11 @@ package ai.chronon.aggregator.test
 import ai.chronon.aggregator.base.ApproxDistinctCount
 import junit.framework.TestCase
 import org.junit.Assert._
+import org.junit.Test
 
 class ApproxDistinctTest extends TestCase {
+
+  @Test
   def testErrorBound(uniques: Int, errorBound: Int, lgK: Int): Unit = {
     val uniqueElems = 1 to uniques
     val duplicates = uniqueElems ++ uniqueElems ++ uniqueElems
@@ -32,6 +35,7 @@ class ApproxDistinctTest extends TestCase {
     assertTrue(Math.abs(estimated - uniques) < errorBound)
   }
 
+  @Test
   def testMergingErrorBound(uniques: Int, errorBound: Int, lgK: Int, merges: Int): Unit = {
     val chunkSize = uniques / merges
     assert(chunkSize > 0)
@@ -50,12 +54,14 @@ class ApproxDistinctTest extends TestCase {
     assertTrue(Math.abs(estimated - uniques) < errorBound)
   }
 
+  @Test
   def testErrorBounds(): Unit = {
     testErrorBound(uniques = 100, errorBound = 1, lgK = 10)
     testErrorBound(uniques = 1000, errorBound = 20, lgK = 10)
     testErrorBound(uniques = 10000, errorBound = 300, lgK = 10)
   }
 
+  @Test
   def testMergingErrorBounds(): Unit = {
     testMergingErrorBound(uniques = 100, errorBound = 1, lgK = 10, merges = 10)
     testMergingErrorBound(uniques = 1000, errorBound = 20, lgK = 10, merges = 4)

aggregator/src/test/scala/ai/chronon/aggregator/test/ApproxHistogramTest.scala

@@ -3,11 +3,14 @@ package ai.chronon.aggregator.test
 import ai.chronon.aggregator.base.{ApproxHistogram, ApproxHistogramIr}
 import junit.framework.TestCase
 import org.junit.Assert._
+import org.junit.Test
 
 import java.util
 import scala.jdk.CollectionConverters._
 
 class ApproxHistogramTest extends TestCase {
+
+  @Test
   def testHistogram(): Unit = {
     val approxHistogram = new ApproxHistogram[String](3)
     val counts = (1L to 3).map(i => i.toString -> i).toMap
@@ -18,6 +21,7 @@ class ApproxHistogramTest extends TestCase {
     assertEquals(toHashMap(counts), approxHistogram.finalize(ir))
   }
 
+  @Test
   def testSketch(): Unit = {
     val approxHistogram = new ApproxHistogram[String](3)
     val counts = (1L to 4).map(i => i.toString -> i).toMap
@@ -29,6 +33,7 @@ class ApproxHistogramTest extends TestCase {
     assertEquals(toHashMap(expected), approxHistogram.finalize(ir))
   }
 
+  @Test
   def testMergeSketches(): Unit = {
     val approxHistogram = new ApproxHistogram[String](3)
     val counts1: Map[String, Long] = Map("5" -> 5L, "4" -> 4, "2" -> 2, "1" -> 1)
@@ -52,6 +57,7 @@ class ApproxHistogramTest extends TestCase {
     assertTrue(ir.histogram.isEmpty)
   }
 
+  @Test
   def testMergeHistograms(): Unit = {
     val approxHistogram = new ApproxHistogram[String](3)
     val counts1: Map[String, Long] = Map("4" -> 4L, "2" -> 2)
@@ -76,6 +82,7 @@ class ApproxHistogramTest extends TestCase {
     assertTrue(ir.sketch.isEmpty)
   }
 
+  @Test
   def testMergeHistogramsToSketch(): Unit = {
     val approxHistogram = new ApproxHistogram[String](3)
     val counts1: Map[String, Long] = Map("4" -> 4L, "3" -> 3)
@@ -101,6 +108,7 @@ class ApproxHistogramTest extends TestCase {
     assertTrue(ir.histogram.isEmpty)
   }
 
+  @Test
   def testMergeSketchAndHistogram(): Unit = {
     val approxHistogram = new ApproxHistogram[String](3)
     val counts1: Map[String, Long] = Map("5" -> 5L, "3" -> 3, "2" -> 2, "1" -> 1)
@@ -125,6 +133,7 @@ class ApproxHistogramTest extends TestCase {
     assert(ir.histogram.isEmpty)
   }
 
+  @Test
   def testNormalizeHistogram(): Unit = {
     val approxHistogram = new ApproxHistogram[String](3)
     val counts = (1L to 3).map(i => i.toString -> i).toMap
@@ -135,6 +144,7 @@ class ApproxHistogramTest extends TestCase {
     assertEquals(ir, normalized)
   }
 
+  @Test
   def testNormalizeSketch(): Unit = {
     val approxHistogram = new ApproxHistogram[String](3)
     val counts = (1L to 4).map(i => i.toString -> i).toMap