Use docker compose for kafka integration tests

.github/dependabot.yml

@@ -109,3 +109,24 @@ updates:
     directory: /plugin/storage/cassandra
     schedule:
       interval: daily
+  - package-ecosystem: docker
+    directory: /docker-compose/elasticsearch
+    schedule:
+      interval: daily
+    allow:
+      - dependency-name: "docker.elastic.co/elasticsearch/elasticsearch"
+        update-types: ["version-update:semver-minor"]
+  - package-ecosystem: docker
+    directory: /docker-compose/opensearch
+    schedule:
+      interval: daily
+    allow:
+      - dependency-name: "docker.elastic.co/opensearch/opensearch"
+        update-types: ["version-update:semver-minor"]
+  - package-ecosystem: docker
+    directory: /docker-compose/kafka-integration-test
+    schedule:
+      interval: daily
+    allow:
+      - dependency-name: "bitnami/kafka"
+        update-types: ["version-update:semver-minor"]

.github/workflows/ci-elasticsearch.yml

@@ -22,13 +22,13 @@ jobs:
       fail-fast: false
       matrix:
         version:
-        - major: 7.x
+        - major: 7
           distribution: elasticsearch
           jaeger: v1
-        - major: 8.x
+        - major: 8
           distribution: elasticsearch
           jaeger: v1
-        - major: 8.x
+        - major: 8
           distribution: elasticsearch
           jaeger: v2
     name: ${{ matrix.version.distribution }} ${{ matrix.version.major }} ${{ matrix.version.jaeger }}
@@ -60,7 +60,7 @@ jobs:
 
 
     - name: Output ${{ matrix.version.distribution }} logs
-      run: docker logs ${{ steps.test-execution.outputs.cid }}
+      run: docker compose -f docker-compose/${{ matrix.version.distribution }}/v${{matrix.version.major}}.yml logs 
       if: ${{ failure() }}
 
     - name: Upload coverage to codecov

.github/workflows/ci-kafka.yml

@@ -34,7 +34,7 @@ jobs:
       run: bash scripts/kafka-integration-test.sh -k
 
     - name: Output Kafka logs
-      run: docker logs kafka
+      run: docker compose -f docker-compose/kafka-integration-test/v3.yml  logs kafka
       if: ${{ failure() }}
 
     - name: Upload coverage to codecov

.github/workflows/ci-opensearch.yml

@@ -22,13 +22,13 @@ jobs:
       fail-fast: false
       matrix:
         version:
-        - major: 1.x
+        - major: 1
           distribution: opensearch
           jaeger: v1
-        - major: 2.x
+        - major: 2
           distribution: opensearch
           jaeger: v1
-        - major: 2.x
+        - major: 2
           distribution: opensearch
           jaeger: v2
     name: ${{ matrix.version.distribution }} ${{ matrix.version.major }} ${{ matrix.version.jaeger }}
@@ -61,7 +61,7 @@ jobs:
 
 
     - name: Output ${{ matrix.version.distribution }} logs
-      run: docker logs ${{ steps.test-execution.outputs.cid }}
+      run: docker compose -f docker-compose/${{ matrix.version.distribution }}/v${{ matrix.version.major }}.yml logs 
       if: ${{ failure() }}
 
     - name: Upload coverage to codecov

docker-compose/kafka-integration-test/v3.yml

@@ -0,0 +1,16 @@
+version: '3.8'
+
+services:
+  kafka:
+    image: bitnami/kafka:3.6.0
+    ports:
+      - "9092:9092"
+    environment:
+      - KAFKA_CFG_NODE_ID=0
+      - KAFKA_CFG_PROCESS_ROLES=controller,broker
+      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@localhost:9093
+      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT

scripts/es-integration-test.sh

@@ -10,23 +10,19 @@ usage() {
   echo $"Usage: $0 <elasticsearch|opensearch> <version>"
   exit 1
 }
-
+#check if the number of arguments is correct
 check_arg() {
   if [ ! $# -eq 3 ]; then
-    echo "ERROR: need exactly two arguments, <elasticsearch|opensearch> <image> <jaeger-version>"
+    echo "ERROR: need exactly three arguments, <elasticsearch|opensearch> <image> <jaeger-version>"
     usage
   fi
 }
-
+#start the elasticsearch/opensearch container
 setup_db() {
-  local distro=$1
-  local compose_file=$2
+  local compose_file=$1
   docker compose -f "${compose_file}" up -d
-  local cid 
-  cid=$(docker compose -f "${compose_file}" ps -q "${distro}")
-  echo "cid=${cid}" >> "$GITHUB_OUTPUT"
 }
-
+#check if the storage is up and running
 wait_for_storage() {
   local distro=$1
   local url=$2
@@ -46,7 +42,7 @@ wait_for_storage() {
     echo "Attempt: ${attempt} ${distro} is not yet available at ${url}..."
     sleep 10
   done
-
+  #if after all the attempts if the storage is not up and running terminate it and exit
   if [[ "$(curl "${params[@]}" "${url}")" != "200" ]]; then
     echo "ERROR: ${distro} is not ready at ${url} after $(( attempt * 10 )) seconds"
     echo "::group::${distro} logs"
@@ -71,7 +67,7 @@ bring_up_storage() {
   do
     echo "attempt $retry"
     if [ "${distro}" = "elasticsearch" ] || [ "${distro}" = "opensearch" ]; then
-        setup_db "${distro}" "${compose_file}"
+        setup_db "${compose_file}"
     else
       echo "Unknown distribution $distro. Valid options are opensearch or elasticsearch"
       usage
@@ -89,7 +85,7 @@ bring_up_storage() {
     exit 1
   fi
 }
-
+#terminate the elasticsearch/opensearch container
 teardown_storage() {
   local compose_file=$1
   docker compose -f "${compose_file}" down
@@ -102,7 +98,7 @@ main() {
   local j_version=$2
 
   bring_up_storage "${distro}" "${es_version}"
-
+  # continues with integration tests
   if [[ "${j_version}" == "v2" ]]; then
     STORAGE=${distro} SPAN_STORAGE_TYPE=${distro} make jaeger-v2-storage-integration-test
   else

scripts/kafka-integration-test.sh

@@ -3,48 +3,40 @@
 set -e
 
 export STORAGE=kafka
-
-# Function to start Kafka
-start_kafka() {
-    echo "Starting Kafka..."
-
-    docker run --name kafka -d \
-    -p 9092:9092 \
-    -e KAFKA_CFG_NODE_ID=0 \
-    -e KAFKA_CFG_PROCESS_ROLES=controller,broker \
-    -e KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@localhost:9093 \
-    -e KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 \
-    -e KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 \
-    -e KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT \
-    -e KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER \
-    -e KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT \
-    bitnami/kafka:3.6.0
-}
-
+compose_file="docker-compose/kafka-integration-test/v3.yml"
 # Check if the -k parameter is provided or not
 if [ "$1" == "-k" ]; then
-    start_kafka
+    echo "Starting Kafka using Docker Compose..."
+    docker compose -f $compose_file up -d kafka
 fi
 
 # Set the timeout in seconds
 timeout=180
 # Set the interval between checks in seconds
 interval=5
 
+# Function to check if Kafka is ready
+is_kafka_ready() {
+    docker compose -f $compose_file \
+    exec kafka /opt/bitnami/kafka/bin/kafka-topics.sh \
+    --list --bootstrap-server localhost:9092 \
+    >/dev/null 2>&1
+}
+
 # Calculate the end time
 end_time=$((SECONDS + timeout))
 
 while [ $SECONDS -lt $end_time ]; do
     # Check if Kafka is ready by attempting to describe a topic
-    if docker exec kafka /opt/bitnami/kafka/bin/kafka-topics.sh --list --bootstrap-server localhost:9092 >/dev/null 2>&1; then
+    if is_kafka_ready; then
         break
     fi
     echo "Kafka broker not ready, waiting ${interval} seconds"
     sleep $interval
 done
 
 # Check if Kafka is still not available after the timeout
-if ! docker exec kafka /opt/bitnami/kafka/bin/kafka-topics.sh --list --bootstrap-server localhost:9092 >/dev/null 2>&1; then
+if ! is_kafka_ready; then
     echo "Timed out waiting for Kafka to start"
     exit 1
 fi