Replace direct telemetry deps with the electric_telemetry library

.buildkite/docker-image.yml

@@ -18,6 +18,7 @@ steps:
       - export ELECTRIC_VERSION=$(git describe --abbrev=7 --tags --always --first-parent --match '@core/sync-service@*' | sed -En 's|^@core/sync-service@||p')
       - export SHORT_COMMIT_SHA=$(git rev-parse --short HEAD)
       - docker buildx build --platform linux/arm64/v8,linux/amd64 --push
+        --build-context electric-telemetry=../electric-telemetry
         --build-arg ELECTRIC_VERSION=$${ELECTRIC_VERSION}
         -t $${ELECTRIC_IMAGE_NAME}:canary
         -t $${ELECTRIC_CANARY_IMAGE_NAME}:$${SHORT_COMMIT_SHA}
@@ -31,6 +32,7 @@ steps:
       - cd ./packages/sync-service
       - export ELECTRIC_VERSION=$(jq '.version' -r package.json)
       - docker buildx build --platform linux/arm64/v8,linux/amd64 --push
+        --build-context electric-telemetry=../electric-telemetry
         --build-arg ELECTRIC_VERSION=$${ELECTRIC_VERSION}
         -t $${ELECTRIC_IMAGE_NAME}:$${ELECTRIC_VERSION}
         -t $${ELECTRIC_IMAGE_NAME}:latest

.changeset/eleven-pens-sell.md

@@ -0,0 +1,6 @@
+---
+'@core/electric-telemetry': patch
+'@core/sync-service': patch
+---
+
+Extract telemetry code from Electric into a separate package, for easier modification and sharing of the telemetry code between Electric and Cloud.

.github/workflows/autoformat.yml

@@ -48,7 +48,7 @@ jobs:
   elixir_formatting:
     strategy:
       matrix:
-        package: [sync-service, elixir-client]
+        package: [sync-service, elixir-client, electric-telemetry]
     name: Elixir formatting and linting
     runs-on: ubuntu-latest
     defaults:

.github/workflows/benchmarking.yml

@@ -62,6 +62,7 @@ jobs:
             --push \
             --cache-from ${{ env.REGISTRY }}/electric:canary-builder \
             --cache-from ${{ env.REGISTRY }}/electric:pr-${{ github.event.issue.number }}-builder \
+            --build-context electric-telemetry=../electric-telemetry \
             --build-arg BUILDKIT_INLINE_CACHE=1 \
             --tag ${{ env.REGISTRY }}/electric:pr-${{ github.event.issue.number }}-builder \
             --target builder \

.github/workflows/docker_image_smoketest.yml

@@ -36,6 +36,8 @@ jobs:
       - uses: docker/build-push-action@v6
         with:
           context: packages/sync-service
+          build-contexts: |
+            electric-telemetry=packages/electric-telemetry
           push: false
           load: true
           tags: 'electric-test-image'

.github/workflows/electric_telemetry_tests.yml

@@ -0,0 +1,86 @@
+name: Electric Telemetry CI
+
+on:
+  push:
+    branches: ['main']
+    paths-ignore:
+      - 'website/**'
+      - '**/README.md'
+  pull_request:
+    paths-ignore:
+      - 'website/**'
+      - '**/README.md'
+
+permissions:
+  contents: read
+
+jobs:
+  build_and_test:
+    name: Build and test electric-telemetry
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: packages/electric-telemetry
+    env:
+      MIX_ENV: test
+      MIX_OS_DEPS_COMPILE_PARTITION_COUNT: 4
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: erlef/setup-beam@v1
+        with:
+          version-type: strict
+          version-file: '.tool-versions'
+
+      - name: Cache electric-telemetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: packages/electric-telemetry/deps
+          key: ${{ runner.os }}-electric-telemetry-deps-${{ hashFiles('packages/electric-telemetry/mix.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-electric-telemetry-deps-${{ hashFiles('packages/electric-telemetry/mix.lock') }}
+            ${{ runner.os }}-electric-telemetry-deps-
+
+      - name: Cache compiled code
+        uses: actions/cache@v4
+        with:
+          path: |
+            packages/electric-telemetry/_build/*/lib
+            !packages/electric-telemetry/_build/*/lib/electric_telemetry
+          key: "${{ runner.os }}-electric-telemetry-build-${{ env.MIX_ENV }}-${{ hashFiles('packages/sync-service/mix.lock') }}"
+          restore-keys: |
+            ${{ runner.os }}-electric-telemetry-build-${{ env.MIX_ENV }}-${{ hashFiles('packages/sync-service/mix.lock') }}
+            ${{ runner.os }}-electric-telemetry-build-${{ env.MIX_ENV }}-
+            ${{ runner.os }}-electric-telemetry-build-
+
+      - name: Install dependencies
+        run: mix do deps.get + deps.compile
+
+      - name: Compile
+        # don't bail on compile warnings - let's get the results of the tests
+        # and let the formatting task check for compilation warnings
+        run: mix compile
+
+      - name: Run tests
+        run: mix coveralls.json
+
+      - name: Upload coverage reports to CodeCov
+        uses: codecov/codecov-action@ad3126e916f78f00edff4ed0317cf185271ccc2d
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: elixir,unit-tests,electric-telemetry
+          files: ./cover/excoveralls.json
+
+      - name: Upload test results to CodeCov
+        uses: codecov/test-results-action@f2dba722c67b86c6caa034178c6e4d35335f6706
+        if: ${{ !cancelled() }}
+        env:
+          DUMMY_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}-dummy
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: true
+          flags: elixir,unit-tests,electric-telemetry
+          files: ./junit/test-junit-report.xml
+          # Upload all PR test results to single branch - requires overriding branch and commit
+          override_branch: ${{ github.event_name == 'pull_request' && 'codecov/pr-test-aggregation' || '' }}
+          override_commit: ${{ github.event_name == 'pull_request' && env.DUMMY_COMMIT_SHA || '' }}

.github/workflows/elixir_tests.yml

@@ -95,26 +95,16 @@ jobs:
             ${{ runner.os }}-sync-service-build-
 
       - name: Install dependencies
-        run: |
-          mix deps.get
-          mix deps.compile
-          MIX_TARGET=application mix deps.compile
+        run: mix do deps.get + deps.compile
 
       - name: Compile
         # don't bail on compile warnings - let's get the results of the tests
         # and let the formatting task check for compilation warnings
-        run: |
-          mix compile
-          MIX_TARGET=application mix compile
+        run: mix compile
 
       - name: Run tests
         run: mix coveralls.json --include slow --cover --export-coverage test
 
-      - name: Run telemetry tests
-        env:
-          MIX_TARGET: application
-        run: mix coveralls.json --only telemetry_target --import-cover cover
-
       - name: Upload coverage reports to CodeCov
         uses: codecov/codecov-action@ad3126e916f78f00edff4ed0317cf185271ccc2d
         with:

packages/electric-telemetry/.dockerignore

@@ -0,0 +1,3 @@
+_build/
+deps/
+test/

packages/electric-telemetry/.formatter.exs

@@ -0,0 +1,4 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
+]

packages/electric-telemetry/.gitignore

@@ -0,0 +1,24 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Temporary files, for example, from tests.
+/tmp/
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+electric_telemetry-*.tar
+

packages/electric-telemetry/README.md

@@ -0,0 +1,41 @@
+# ElectricTelemetry
+
+Library used by Electric to gather telemetry and export it to a number of supported destinations. Originally extracted from [electric-sql/electric](https://github.com/electric-sql/electric).
+
+## Installation
+
+Install it by adding `electric_telemetry` to your list of dependencies in `mix.exs`:
+
+```elixir
+def deps do
+  [
+    {:electric_telemetry, github: "electric-sql/electric", sparse: "packages/electric-telemetry"}
+  ]
+end
+```
+
+## Configuration
+
+Runtime configuration is available for the underlying otel_metric_exporter lib:
+
+```elixir
+config :otel_metric_exporter,
+  otlp_endpoint: "...",
+  otlp_headers: %{...}
+  resource: %{...}
+```
+
+The configuration options for `ElectricTelemetry.ApplicationTelemetry` or `ElectricTelemetry.StackTelemetry` must be passed as a keyword list function argument. See `ElectricTelemetry.Opts` for the supported options.
+
+## Overview
+
+At a high level, the library includes these modules:
+
+- `ElectricTelemetry` includes basic utilities such as validating user options.
+
+- `ElectricTelemetry.ApplicationTelemetry` defines metrics and periodic measurements that apply to BEAM as a whole.
+
+- `ElectricTelemetry.StackTelemetry` defines metrics that are specific to the notion of an
+  Electric stack: shape stats, replication client stats, etc. No builtin measurements are defines here.
+
+- Reporter modules. These are enabled individually and are used for exporting metrics to the corresponding destination.

packages/electric-telemetry/lib/electric/telemetry.ex

@@ -0,0 +1,47 @@
+defmodule ElectricTelemetry do
+  def scheduler_ids do
+    num_schedulers = :erlang.system_info(:schedulers)
+    Enum.map(1..num_schedulers, &:"normal_#{&1}") ++ [:cpu, :io]
+  end
+
+  def keep_for_stack(metrics, stack_id) do
+    Enum.map(metrics, fn
+      {key, metric} -> {key, filter_metric(metric, stack_id)}
+      metric when is_map(metric) -> filter_metric(metric, stack_id)
+    end)
+  end
+
+  defp filter_metric(metric, stack_id) do
+    Map.update!(metric, :keep, fn
+      nil -> fn metadata -> metadata[:stack_id] == stack_id end
+      fun -> fn metadata -> fun.(metadata) && metadata[:stack_id] == stack_id end
+    end)
+  end
+
+  @opts_schema NimbleOptions.new!(ElectricTelemetry.Opts.schema())
+
+  def validate_options(user_opts) do
+    with {:ok, validated_opts} <- NimbleOptions.validate(user_opts, @opts_schema) do
+      config =
+        Map.new(validated_opts, fn
+          {k, kwlist} when k in [:reporters, :intervals_and_thresholds] -> {k, Map.new(kwlist)}
+          kv -> kv
+        end)
+
+      {:ok, config}
+    end
+  end
+
+  def export_enabled?(%{reporters: reporters}) do
+    truthy?(
+      reporters.statsd_host ||
+        reporters.call_home_url ||
+        reporters.otel_metrics? ||
+        reporters.prometheus?
+    )
+  end
+
+  defp truthy?(false), do: false
+  defp truthy?(nil), do: false
+  defp truthy?(_), do: true
+end