Merge branch 'main' into jbrooks-bounded-unique-count

Author: pengyu-hou

.bazeliskrc

@@ -0,0 +1 @@
+USE_BAZEL_VERSION=6.4.0

.bazelrc

@@ -0,0 +1,49 @@
+## Disable remote cache completely when --config=local is passed
+build:local --remote_cache=
+
+# Scala version config flags:
+# To build with Scala 2.12, pass "--config scala_2.12" to "bazel build"
+# To set a different default Scala version, add the following to
+# user.bazelrc:
+# common --config scala_2.12
+common:scala_2.11 --repo_env=SCALA_VERSION=2.11.12 --define scala_version=2.11
+common:scala_2.12 --repo_env=SCALA_VERSION=2.12.18 --define scala_version=2.12
+common:scala_2.13 --repo_env=SCALA_VERSION=2.13.12 --define scala_version=2.13
+
+# Default scala version to 2.12
+common --repo_env=SCALA_VERSION=2.12.18 --define scala_version=2.12
+
+# Spark versions
+common:spark_2.4 --define spark_version=2.4
+common:spark_3.1 --define spark_version=3.1
+common:spark_3.2 --define spark_version=3.2
+common:spark_3.5 --define spark_version=3.5
+# Default Spark version
+common --define spark_version=3.1
+
+# Flink versions
+common:flink_1.16 --define flink_version=1.16
+# Default Flink version
+common --define flink_version=1.16
+
+# Default version to pass the builds
+common --define version=0.0.0
+
+build --javacopt=-Xep:DoubleBraceInitialization:OFF
+
+# Don't implicitly create __init__.py files
+build --incompatible_default_to_explicit_init_py
+
+# https://github.com/bazelbuild/bazel/issues/2377
+test --spawn_strategy=standalone
+
+# Don't depend on a JAVA_HOME pointing at a system JDK
+# see https://github.com/bazelbuild/rules_jvm_external/issues/445
+build --repo_env=JAVA_HOME=../bazel_tools/jdk
+
+build:java_8 --java_language_version=8 --java_runtime_version=remotejdk_8
+build:java_11 --java_language_version=11 --java_runtime_version=remotejdk_11
+build:java_21 --java_language_version=21 --java_runtime_version=remotejdk_21
+
+# Use Java 8 as default for Bazel builds
+build --java_language_version=8 --java_runtime_version=remotejdk_8

.circleci/Dockerfile

@@ -34,6 +34,7 @@ RUN apt-get update && apt-get -y -q install \
     openjdk-8-jdk \
     pkg-config \
     sbt \
+    bazelisk \
     && apt-get clean
 
 # Install thrift

.circleci/config.yml

@@ -10,6 +10,11 @@ executors:
         working_directory: /chronon
         docker:
             - image: houpy0829/chronon-ci:base--f87f50dc520f7a73894ae024eb78bd305d5b08e2
+    docker_baseimg_executor_xxlarge:
+      resource_class: 2xlarge
+      working_directory: /chronon
+      docker:
+        - image: houpy0829/chronon-ci:base--f87f50dc520f7a73894ae024eb78bd305d5b08e2
 
 jobs:
     "Pull Docker Image":
@@ -27,7 +32,7 @@ jobs:
                     docker pull houpy0829/chronon-ci:base--f87f50dc520f7a73894ae024eb78bd305d5b08e2 || true
 
     "Scala 12 -- Spark 3 Tests":
-        executor: docker_baseimg_executor
+        executor: docker_baseimg_executor_xxlarge
         steps:
             - checkout
             - run:
@@ -62,7 +67,7 @@ jobs:
                   command: |
                       conda activate chronon_py
                       # Increase if we see OOM.
-                      export SBT_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=4G -Xmx4G -Xms2G"
+                      export SBT_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=8G -Xmx8G -Xms4G"
                       sbt "++ 2.13.6 test"
             - store_test_results:
                   path: /chronon/spark/target/test-reports
@@ -102,7 +107,7 @@ jobs:
                       conda activate chronon_py
                       cd /chronon/api/py/ai/chronon
                       pip install importlib-metadata==4.11.4 #Install importlib-metadata < 5
-                      flake8 --extend-ignore=W605,Q000,F631
+                      flake8 --extend-ignore=W605,Q000,F631,E203
 
     "Chronon Python Tests":
         executor: docker_baseimg_executor
@@ -112,14 +117,14 @@ jobs:
                   name: Run Chronon Python tests
                   shell: /bin/bash -leuxo pipefail
                   command: |
-                      conda activate chronon_py
-                      pushd /chronon/api/
-                      thrift --gen py -out /chronon/api/py/ai/chronon\
-                         /chronon/api/thrift/api.thrift                 # Generate thrift files
-                      cd /chronon/api/py                                # Go to Python module
-                      pip install -r requirements/dev.txt               # Install latest requirements
-                      tox                                               # Run tests
-                      popd
+                    conda activate chronon_py
+                    pushd /chronon/api/
+                    thrift --gen py -out /chronon/api/py/ai/chronon\
+                       /chronon/api/thrift/api.thrift                 # Generate thrift files
+                    cd /chronon/                                      # Go to Python module
+                    pip install -r api/py/requirements/dev.txt        # Install latest requirements
+                    tox                                               # Run tests
+                    popd
             - store_artifacts:
                   path: /chronon/api/py/htmlcov
 
@@ -133,6 +138,62 @@ jobs:
                   command: |
                       conda activate chronon_py
                       sbt +scalafmtCheck
+    # run these separately as we need a isolated JVM to not have Spark session settings interfere with other runs
+    # long term goal is to refactor the current testing spark session builder and avoid adding new single test to CI
+    "Scala 13 -- Iceberg Format Tests":
+      executor: docker_baseimg_executor
+      steps:
+        - checkout
+        - run:
+            name: Run Scala 13 tests for Iceberg format
+            environment:
+              format_test: iceberg
+            shell: /bin/bash -leuxo pipefail
+            command: |
+              conda activate chronon_py
+              # Increase if we see OOM.
+              export SBT_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=4G -Xmx4G -Xms2G"
+              sbt ';project spark_embedded; ++ 2.13.6; testOnly ai.chronon.spark.test.TableUtilsFormatTest'
+        - store_test_results:
+            path: /chronon/spark/target/test-reports
+        - store_test_results:
+            path: /chronon/aggregator/target/test-reports
+        - run:
+            name: Compress spark-warehouse
+            command: |
+              cd /tmp/ && tar -czvf spark-warehouse.tar.gz chronon/spark-warehouse
+            when: on_fail
+        - store_artifacts:
+            path: /tmp/spark-warehouse.tar.gz
+            destination: spark_warehouse.tar.gz
+            when: on_fail
+    "Scala 13 -- Iceberg Table Utils Tests":
+      executor: docker_baseimg_executor
+      steps:
+        - checkout
+        - run:
+            name: Run Scala 13 tests for Iceberg Table Utils
+            environment:
+              format_test: iceberg
+            shell: /bin/bash -leuxo pipefail
+            command: |
+              conda activate chronon_py
+              # Increase if we see OOM.
+              export SBT_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=4G -Xmx4G -Xms2G"
+              sbt ';project spark_embedded; ++ 2.13.6; testOnly ai.chronon.spark.test.TableUtilsTest'
+        - store_test_results:
+            path: /chronon/spark/target/test-reports
+        - store_test_results:
+            path: /chronon/aggregator/target/test-reports
+        - run:
+            name: Compress spark-warehouse
+            command: |
+              cd /tmp/ && tar -czvf spark-warehouse.tar.gz chronon/spark-warehouse
+            when: on_fail
+        - store_artifacts:
+            path: /tmp/spark-warehouse.tar.gz
+            destination: spark_warehouse.tar.gz
+            when: on_fail
 
 workflows:
     build_test_deploy:
@@ -156,3 +217,9 @@ workflows:
             - "Chronon Python Lint":
                   requires:
                       - "Pull Docker Image"
+            - "Scala 13 -- Iceberg Format Tests":
+                  requires:
+                      - "Pull Docker Image"
+            - "Scala 13 -- Iceberg Table Utils Tests":
+                  requires:
+                      - "Pull Docker Image"

.gitignore

@@ -10,6 +10,11 @@
 *.logs
 *.iml
 .idea/
+*.jvmopts
+.bloop*
+.metals*
+.venv*
+*metals.sbt*
 .eclipse
 **/.vscode/
 **/__pycache__/
@@ -21,7 +26,9 @@ api/py/.coverage
 api/py/htmlcov/
 **/derby.log
 cs
-
+*.bloop
+*.metals
+*.venv
 # Documentation builds
 docs/build/
 
@@ -33,6 +40,7 @@ api/py/build/
 api/py/ai/chronon/thrift/
 api/py/*.egg-info/
 *.egg
+.coverage
 
 # spark test scratch area
 **/spark-warehouse/
@@ -43,3 +51,5 @@ mvn_settings.xml
 # Release folder
 releases
 
+# bazel
+bazel-*

.ijwb/.bazelproject

@@ -0,0 +1,16 @@
+directories:
+  # Add the directories you want added as source here
+  # By default, we've added your entire workspace ('.')
+  .
+
+# Automatically includes all relevant targets under the 'directories' above
+derive_targets_from_directories: true
+
+targets:
+  # If source code isn't resolving, add additional targets that compile it here
+
+additional_languages:
+  # Uncomment any additional languages you want supported
+  python
+  scala
+  java

AUTHORS

@@ -8,6 +8,7 @@ Vamsee Yarlagadda (Airbnb)
 Hao Cen (Airbnb)
 Donghan Zhang (Airbnb)
 Yuli Han (Airbnb)
+Praveen Kundurthy (Airbnb)
 Ben Mears (Stripe)
 Andrew Lee (Stripe)
 Cam Weston (Stripe)
@@ -16,3 +17,4 @@ Daniel Kristjansson (Stripe)
 Piyush Narang (Stripe)
 Caio Camatta (Stripe)
 Divya Manohar (Stripe)
+Krish Narukulla (Roku)

BUILD.bazel

@@ -0,0 +1,21 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+
+# To update py3_requirements_lock.txt, run:
+compile_pip_requirements(
+    name = "pip",
+    extra_args = [
+        "--allow-unsafe",
+        "--resolver=backtracking",
+    ],
+    requirements_in = "//:requirements.txt",
+    requirements_txt = "//:requirements_lock.txt",
+    # force this to mac-only, since that's where we expect it to run.
+    # remove + adopt requirements_[platform] arguments when we're on rules_python>=0.10.0
+    tags = ["manual"],
+)
+
+exports_files(
+    ["pom.xml.tpl"],
+)

CONTRIBUTING.md

@@ -38,6 +38,18 @@ The process for reporting bugs and requesting smaller features is also outlined
 
 Pull Requests (PRs) should follow these guidelines as much as possible:
 
+### Creating a pull request from a fork
+Below is a general workflow to create a PR from a forked repository. By following these steps, we can avoid granting write permissions to the main repository.
+- [Fork the repository](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo)
+- Clone the forked repository 
+- [Configuring a remote repository for a fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/configuring-a-remote-repository-for-a-fork)
+- Sync the forked repository with the main repository
+- [Syncing a fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork)
+- Create a new branch for your changes
+- Make your changes
+- Push the branch to your fork
+- [Create a PR from your branch to the main repository](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork)
+
 ### Code Guidelines
 
 - Follow our [code style guidelines](docs/source/Code_Guidelines.md)

PROJECT_MANAGEMENT_COMMITTEE

@@ -0,0 +1,17 @@
+# Airbnb (8 seats)
+Haozhen Ding
+Pallavi Adusumilli
+Pengyu Hou
+Praveen Kundurthy
+Rohit Girme
+Sophie Wang
+Sherry Li
+Yuli Han
+
+# Stripe (6 seats)
+Ben Mears
+Caio Camatta
+Cam Weston
+Jeff Brooks
+Jeremy Robin
+Spencer McNamara