[WIP] Bazel fixes, Bazel CI, Dev Containers

.bazelrc

@@ -1,3 +1,5 @@
+try-import %workspace%/.maven.bazelrc
+
 ## Disable remote cache completely when --config=local is passed
 build:local --remote_cache=
 
@@ -19,7 +21,7 @@ common:spark_3.1 --define spark_version=3.1
 common:spark_3.2 --define spark_version=3.2
 common:spark_3.5 --define spark_version=3.5
 # Default Spark version
-common --define spark_version=3.1
+common --define spark_version=3.2
 
 # Flink versions
 common:flink_1.16 --define flink_version=1.16
@@ -46,4 +48,23 @@ build:java_11 --java_language_version=11 --java_runtime_version=remotejdk_11
 build:java_21 --java_language_version=21 --java_runtime_version=remotejdk_21
 
 # Use Java 8 as default for Bazel builds
-build --java_language_version=8 --java_runtime_version=remotejdk_8
+build --java_language_version=8 --java_runtime_version=remotejdk_8
+
+# CircleCI-specific configurations for test execution
+test:ci --test_output=streamed
+test:ci --test_summary=detailed
+test:ci --local_ram_resources=HOST_RAM*0.7
+test:ci --local_cpu_resources=HOST_CPUS*0.7
+
+# BuildBuddy shared configuration
+common:buildbuddy --bes_results_url=https://chronon-ai.buildbuddy.io/invocation/
+common:buildbuddy --bes_backend=grpcs://chronon-ai.buildbuddy.io
+common:buildbuddy --remote_cache=grpcs://chronon-ai.buildbuddy.io
+common:buildbuddy --remote_timeout=10m
+
+# BuildBuddy configuration for main branch (Read/Write cache)
+build:main --config=buildbuddy
+
+# BuildBuddy configuration for PR builds (Read-Only cache)
+build:pr --config=buildbuddy
+build:pr --noremote_upload_local_results

.bazelversion

@@ -0,0 +1 @@
+6.4.0

.circleci/Dockerfile

@@ -34,9 +34,12 @@ RUN apt-get update && apt-get -y -q install \
     openjdk-8-jdk \
     pkg-config \
     sbt \
-    bazelisk \
     && apt-get clean
 
+# Install bazelisk directly from GitHub releases
+RUN curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-amd64 \
+    && chmod +x /usr/local/bin/bazel
+
 # Install thrift
 RUN curl -sSL "http://archive.apache.org/dist/thrift/$THRIFT_VERSION/thrift-$THRIFT_VERSION.tar.gz" -o thrift.tar.gz \
 	&& mkdir -p /usr/src/thrift \

.circleci/config.yml

@@ -195,6 +195,84 @@ jobs:
             destination: spark_warehouse.tar.gz
             when: on_fail
 
+    # Job for Main branch builds (Read/Write cache)
+    "Bazel Tests - Main Branch":
+        executor: docker_baseimg_executor_xxlarge
+        steps:
+            - checkout
+            - run:
+                name: Run Bazel Setup
+                command: |
+                    # Add Bazel GPG key and repository
+                    curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor | sudo tee /usr/share/keyrings/bazel-archive-keyring.gpg > /dev/null
+                    echo "deb [arch=amd64 signed-by=/usr/share/keyrings/bazel-archive-keyring.gpg] https://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list
+                    # Update package list and install specific Bazel version
+                    sudo apt update
+                    sudo apt install -y bazel-6.4.0
+                    # Set up bazel-6.4.0 as the default bazel command
+                    sudo update-alternatives --install /usr/bin/bazel bazel /usr/bin/bazel-6.4.0 100
+            - run:
+                name: Setup permissions
+                command: |
+                    sudo mkdir -p /root/.cache/bazel
+                    useradd --create-home --shell /bin/bash nonroot
+                    chown -R nonroot:nonroot /chronon
+                    chown -R nonroot:nonroot /root/.cache/bazel # Give access to the bazel cache
+            - run:
+                name: 🚀 Run Bazel Tests (Read/Write Cache)
+                shell: /bin/bash -leuxo pipefail
+                command: |
+                    export JAVA_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=4G -Xmx4G -Xms2G"
+                    sudo -u nonroot -H -s /bin/bash -c "
+                        bazel version && 
+                        bazel build //... --config=main --remote_header=x-buildbuddy-api-key=$BuildBuddyAPIKey && 
+                        bazel test //... --config=main --config=ci --remote_header=x-buildbuddy-api-key=$BuildBuddyAPIKey"
+            - store_test_results:
+                path: bazel-testlogs
+            - store_artifacts:
+                path: bazel-testlogs
+                destination: bazel-test-logs
+                when: on_fail
+
+    # Job for PR builds (Read-Only cache)
+    "Bazel Tests - PR":
+        executor: docker_baseimg_executor_xxlarge
+        steps:
+            - checkout
+            - run:
+                name: Run Bazel Setup
+                command: |
+                    # Add Bazel GPG key and repository
+                    curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor | sudo tee /usr/share/keyrings/bazel-archive-keyring.gpg > /dev/null
+                    echo "deb [arch=amd64 signed-by=/usr/share/keyrings/bazel-archive-keyring.gpg] https://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list
+                    # Update package list and install specific Bazel version
+                    sudo apt update
+                    sudo apt install -y bazel-6.4.0
+                    # Set up bazel-6.4.0 as the default bazel command
+                    sudo update-alternatives --install /usr/bin/bazel bazel /usr/bin/bazel-6.4.0 100
+            - run:
+                name: Setup permissions
+                command: |
+                    sudo mkdir -p /root/.cache/bazel
+                    useradd --create-home --shell /bin/bash nonroot
+                    chown -R nonroot:nonroot /chronon
+                    chown -R nonroot:nonroot /root/.cache/bazel # Give access to the bazel cache
+            - run:
+                name: Run Bazel Tests
+                shell: /bin/bash -leuxo pipefail
+                command: |
+                    export JAVA_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=4G -Xmx4G -Xms2G"
+                    sudo -u nonroot -H -s /bin/bash -c "
+                        bazel version && 
+                        bazel build //... --config=pr --remote_header=x-buildbuddy-api-key=$BuildBuddyAPIKey && 
+                        bazel test //... --config=pr --config=ci --remote_header=x-buildbuddy-api-key=$BuildBuddyAPIKey"
+            - store_test_results:
+                path: bazel-testlogs
+            - store_artifacts:
+                path: bazel-testlogs
+                destination: bazel-test-logs
+                when: on_fail
+
 workflows:
     build_test_deploy:
         jobs:
@@ -222,4 +300,14 @@ workflows:
                       - "Pull Docker Image"
             - "Scala 13 -- Iceberg Table Utils Tests":
                   requires:
-                      - "Pull Docker Image"
+                      - "Pull Docker Image"
+            - "Bazel Tests - Main Branch":
+                  filters:
+                      branches:
+                          only:
+                              - main
+            - "Bazel Tests - PR":
+                  filters:
+                      branches:
+                          ignore:
+                              - main

.devcontainer/devcontainer.json

@@ -0,0 +1,25 @@
+{
+  "name": "Chronon Development",
+  "image": "mcr.microsoft.com/devcontainers/base:ubuntu-22.04",
+  "features": {
+    "ghcr.io/devcontainers/features/java:1": {
+      "version": "11"
+    },
+    "ghcr.io/devcontainers/features/python:1": {
+      "version": "3.10"
+    }
+  },
+  "postCreateCommand": ".devcontainer/setup.sh",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "scalameta.metals",
+        "ms-python.python",
+        "ms-python.pylint",
+        "bazelbuild.vscode-bazel"
+      ]
+    }
+  },
+  "forwardPorts": [8080, 3000],
+  "remoteUser": "vscode"
+}

.devcontainer/install_thrift.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -efx
+
+# Clean up any existing thrift_build directory
+rm -rf thrift_build
+mkdir thrift_build
+pushd thrift_build
+
+# Download archive and verify it matches our expected checksum.
+THRIFT_HTTP_ARCHIVE=https://archive.apache.org/dist/thrift/0.13.0/thrift-0.13.0.tar.gz
+THRIFT_ARCHIVE=thrift.tar.gz
+THRIFT_EXPECTED_CHECKSUM_SHA256=7ad348b88033af46ce49148097afe354d513c1fca7c607b59c33ebb6064b5179
+curl "$THRIFT_HTTP_ARCHIVE" -o "$THRIFT_ARCHIVE"
+THRIFT_ACTUAL_CHECKSUM_SHA256=$(sha256sum "$THRIFT_ARCHIVE" | awk '{ print $1 }')
+if [ "$THRIFT_EXPECTED_CHECKSUM_SHA256" != "$THRIFT_ACTUAL_CHECKSUM_SHA256" ]; then
+    echo "Checksum does not match expected value" >&2
+    echo " - location: $THRIFT_HTTP_ARCHIVE" >&2
+    echo " - expected: $THRIFT_EXPECTED_CHECKSUM_SHA256" >&2
+    echo " - obtained: $THRIFT_ACTUAL_CHECKSUM_SHA256" >&2
+    exit 1
+fi
+
+echo "Building Thrift from source"
+# Build thrift from source.
+mkdir src 
+tar zxvf thrift.tar.gz -C src --strip-components=1
+pushd src
+
+# Install build dependencies
+sudo apt update
+sudo apt install -y build-essential libssl-dev pkg-config flex bison
+
+# Configure and build
+./configure --without-cpp --without-java --without-python --without-py3 --without-ruby --without-haskell --without-erlang --without-perl --without-php --without-php_extension --without-c_glib --without-csharp --without-go --without-nodejs --without-lua --without-qt5 --without-dart --without-swift --without-js --without-d --without-haxe --without-cl --without-as3 --without-dotnetcore --without-netstd --without-rs
+make
+
+# Install
+sudo make install
+
+popd
+
+# Verify installation
+thrift -version
+
+popd
+
+echo "Thrift 0.13.0 installation completed successfully"

.devcontainer/setup.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Install Bazel 6.4.0 (required by this project)
+echo "Installing Bazel 6.4.0..."
+
+# Add Bazel GPG key and repository
+curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor | sudo tee /usr/share/keyrings/bazel-archive-keyring.gpg > /dev/null
+echo "deb [arch=amd64 signed-by=/usr/share/keyrings/bazel-archive-keyring.gpg] https://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list
+
+# Update package list and install specific Bazel version
+sudo apt update
+sudo apt install -y bazel-6.4.0
+
+# Set up bazel-6.4.0 as the default bazel command
+sudo update-alternatives --install /usr/bin/bazel bazel /usr/bin/bazel-6.4.0 100
+
+# Install Thrift compiler (0.13.0 to match project requirements)
+echo "Installing Thrift 0.13.0..."
+./.devcontainer/install_thrift.sh
+
+# Verify installations
+echo "Verifying installations..."
+bazel version
+python --version
+java -version
+
+echo "Setup complete!"

.gitignore

@@ -31,7 +31,10 @@ cs
 *.venv
 # Documentation builds
 docs/build/
-
+thrift_build/*
+project/*
+.metals/*
+.claude/*
 # Python distribution and packaging
 api/py/dist/
 api/py/eggs/

.ijwb/.bazelproject

@@ -1,16 +1 @@
-directories:
-  # Add the directories you want added as source here
-  # By default, we've added your entire workspace ('.')
-  .
-
-# Automatically includes all relevant targets under the 'directories' above
-derive_targets_from_directories: true
-
-targets:
-  # If source code isn't resolving, add additional targets that compile it here
-
-additional_languages:
-  # Uncomment any additional languages you want supported
-  python
-  scala
-  java
+import tools/ide_support/intellij/default_view.bazelproject

.jvmopts

@@ -0,0 +1,4 @@
+-Xms2G
+-Xmx4G
+-XX:+CMSClassUnloadingEnabled 
+-XX:MaxPermSize=4G

WORKSPACE

@@ -123,6 +123,12 @@ load("@io_bazel_rules_scala//scala:toolchains.bzl", "scala_register_toolchains")
 
 scala_register_toolchains()
 
+load("@io_bazel_rules_scala//testing:junit.bzl", "junit_repositories", "junit_toolchain")
+
+junit_repositories()
+
+junit_toolchain()
+
 load("@io_bazel_rules_scala//testing:scalatest.bzl", "scalatest_repositories", "scalatest_toolchain")
 
 scalatest_repositories()

aggregator/BUILD.bazel

@@ -57,13 +57,13 @@ scala_library(
            ]),
 )
 
-scala_test_suite(
+scala_junit_test(
     name = "test",
     srcs = glob(["src/test/scala/ai/chronon/aggregator/test/*.scala"]),
+    suffixes = ["Test"],
     visibility = ["//visibility:public"],
     deps = [
                ":aggregator",
-               ":test-lib",
                "//api:api-lib",
                "//api:api-models",
                maven_artifact("junit:junit"),
@@ -100,9 +100,9 @@ genrule(
 
 java_export(
     name = "aggregator-export",
-    maven_coordinates = "ai.chronon:aggregator_(scala_version):$(version)",
+    maven_coordinates = "ai.chronon:aggregator_$(scala_version):$(version)",
     pom_template = ":generate_pom",
     runtime_deps = [
         ":aggregator",
     ],
-)
+)

aggregator/src/test/scala/ai/chronon/aggregator/test/ApproxDistinctTest.scala

@@ -19,8 +19,11 @@ package ai.chronon.aggregator.test
 import ai.chronon.aggregator.base.ApproxDistinctCount
 import junit.framework.TestCase
 import org.junit.Assert._
+import org.junit.Test
 
 class ApproxDistinctTest extends TestCase {
+
+  @Test
   def testErrorBound(uniques: Int, errorBound: Int, lgK: Int): Unit = {
     val uniqueElems = 1 to uniques
     val duplicates = uniqueElems ++ uniqueElems ++ uniqueElems
@@ -32,6 +35,7 @@ class ApproxDistinctTest extends TestCase {
     assertTrue(Math.abs(estimated - uniques) < errorBound)
   }
 
+  @Test
   def testMergingErrorBound(uniques: Int, errorBound: Int, lgK: Int, merges: Int): Unit = {
     val chunkSize = uniques / merges
     assert(chunkSize > 0)
@@ -50,12 +54,14 @@ class ApproxDistinctTest extends TestCase {
     assertTrue(Math.abs(estimated - uniques) < errorBound)
   }
 
+  @Test
   def testErrorBounds(): Unit = {
     testErrorBound(uniques = 100, errorBound = 1, lgK = 10)
     testErrorBound(uniques = 1000, errorBound = 20, lgK = 10)
     testErrorBound(uniques = 10000, errorBound = 300, lgK = 10)
   }
 
+  @Test
   def testMergingErrorBounds(): Unit = {
     testMergingErrorBound(uniques = 100, errorBound = 1, lgK = 10, merges = 10)
     testMergingErrorBound(uniques = 1000, errorBound = 20, lgK = 10, merges = 4)