AB testing for ads services

.env.template

@@ -23,9 +23,10 @@ NEXT_PUBLIC_DD_VERSION_FRONTEND=1.0.0 # use NEXT_PUBLIC_* to expose var to store
 DD_VERSION_BACKEND=1.0.0 # Use for worker service as well, since they use the same Docker image
 DD_VERSION_DBM=1.0.0
 DD_VERSION_ADS=1.0.0
+DD_VERSION_ADS_PYTHON=1.0.0
 DD_VERSION_DISCOUNTS=1.0.0
-DD_VERSION_POSTGRES=1.0.0
-DD_VERSION_NGINX=1.0.0
+DD_VERSION_POSTGRES=15.0
+DD_VERSION_NGINX=1.28.0
 DD_VERSION_REDIS=6.2
 
 # =============================================
@@ -35,8 +36,15 @@ POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
 DB_HOST=postgres
 DB_PORT=5432
-DB_POOL=25                    # Database connection pool size set in backend service (default: 25)
-MAX_THREADS=5                 # Maximum number of concurrent threads set in backend service (default: 5)
+DB_POOL=25 # Database connection pool size set in backend service (default: 25)
+MAX_THREADS=5 # Maximum number of concurrent threads set in backend service (default: 5)
+
+# =============================================
+# Nginx/Service Proxy Configuration
+# =============================================
+ADS_A_UPSTREAM=ads:3030
+ADS_B_UPSTREAM=ads-python:3030
+ADS_B_PERCENT=0 # Set < 0 > 100 to split traffic between ads services
 
 # =============================================
 # Frontend Service Configuration
@@ -65,4 +73,4 @@ DISABLE_SPRING=1
 STOREDOG_URL=http://service-proxy:80 # base url for storedog service (default: 'http://service-proxy:80')
 PUPPETEER_TIMEOUT=30000 # timeout for puppeteer (default: 30000)
 
-SKIP_SESSION_CLOSE= # skip session close for puppeteer (default: ''). note that the current puppeteer script doesn't make use of this environment variable but can easily be updated to do so
+SKIP_SESSION_CLOSE= # skip session close for puppeteer (default: ''). note that the current puppeteer script doesn't make use of this environment variable but can easily be updated to do so

README.md

@@ -154,15 +154,15 @@ Common Datadog variables that can be set in application services:
 - `DD_LOGS_INJECTION`: Enable log injection into traces. Some languages' trace libraries turn this on by default, but we turn it on explicitly to prevent confusion.
 - `DD_PROFILING_ENABLED`: Enable the Continuous Profiler 
 - `DD_RUNTIME_METRICS_ENABLED`: Enable runtime metrics
-- 
 
 Service-specific versions:
 - `DD_VERSION_FRONTEND`: Frontend version (default: `1.0.0`)
 - `DD_VERSION_BACKEND`: Backend version (default: `1.0.0`)
 - `DD_VERSION_DISCOUNTS`: Discounts service version (default: `1.0.0`)
 - `DD_VERSION_ADS`: Ads service version (default: `1.0.0`)
-- `DD_VERSION_NGINX`: nginx service version (default: `1.0.0`)
-- `DD_VERSION_POSTGRES`: PostgreSQL service version (default: `1.0.0`)
+- `DD_VERSION_ADS_PYTHON`: Ads Python service version (default: `1.0.0`)
+- `DD_VERSION_NGINX`: nginx service version (default: `1.28.0`)
+- `DD_VERSION_POSTGRES`: PostgreSQL service version (default: `15.0`)
 - `DD_VERSION_REDIS`: Redis version (default: `6.2`)
 
 > [!NOTE]
@@ -186,6 +186,17 @@ Puppeteer service configuration:
 - `PUPPETEER_TIMEOUT`: Sets max timeout for Puppeteer, in case the session is unresponsive
 - `SKIP_SESSION_CLOSE`: Skip closing browser sessions
 
+### Nginx/Service Proxy Configuration Variables
+
+These variables control the upstream configuration for the ads services in the Nginx (service-proxy) container, enabling A/B testing and traffic splitting between the Java and Python ads services.
+
+> [!IMPORTANT]
+> When `ADS_B_PERCENT` is greater than zero, the `ADS_B_UPSTREAM` endpoint must be reachable. Otherwise, the service-proxy will crash and restart. Uncomment the `ads-python` section in `docker-compose.yml` to enable the service.
+
+- `ADS_A_UPSTREAM`: Host and port for the primary (A) ads service (default: `ads:3030`)
+- `ADS_B_UPSTREAM`: Host and port for the secondary (B) ads service (default: `ads-python:3030`)
+- `ADS_B_PERCENT`: Percentage of traffic to route to the B (Python) ads service (default: `0`). The remainder goes to the A (Java) ads service. Set to a value between `0` and `100` to control the split.
+
 ## Feature flags
 Some capabilities are hidden behind feature flags, which can be controlled via `services/frontend/site/featureFlags.config.json`. 
 

docker-compose.dev.yml

@@ -165,8 +165,12 @@ services:
     build:
       context: ./services/ads/java
     depends_on:
+      - postgres
       - dd-agent
     environment:
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
+      - POSTGRES_USER=${POSTGRES_USER:-postgres}
+      - POSTGRES_HOST=postgres
       - DD_AGENT_HOST=dd-agent
       - DD_SERVICE=store-ads
       - DD_VERSION=${DD_VERSION_ADS:-1.0.0}
@@ -179,6 +183,25 @@ services:
     labels:
       com.datadoghq.ad.logs: '[{"source": "java"}]'
 
+  # OPTIONAL: Advertisement service (Python)
+  # ads-python:
+  #   build:
+  #     context: ./services/ads/python
+  #   depends_on:
+  #     - postgres
+  #     - dd-agent
+  #   environment:
+  #     - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
+  #     - POSTGRES_USER=${POSTGRES_USER:-postgres}
+  #     - POSTGRES_HOST=postgres
+  #     - DD_AGENT_HOST=dd-agent
+  #     - DD_SERVICE=store-ads-python
+  #     - DD_VERSION=${DD_VERSION_ADS_PYTHON:-1.0.0}
+  #   networks:
+  #     - storedog-network
+  #   labels:
+  #     com.datadoghq.ad.logs: '[{"source": "python"}]'
+
   # Web server and reverse proxy
   service-proxy:
     build:
@@ -192,9 +215,12 @@ services:
       - frontend
       - dd-agent
     environment:
+      - ADS_A_UPSTREAM=${ADS_A_UPSTREAM:-ads:3030} # Only needed if using AB testing
+      - ADS_B_UPSTREAM=${ADS_B_UPSTREAM:-ads-python:3030} # Only needed if using AB testing
+      - ADS_B_PERCENT=${ADS_B_PERCENT:-0} # Percent of traffic to ads-python; remainder goes to ads-java. Only needed if using AB testing
       - DD_AGENT_HOST=dd-agent
       - DD_SERVICE=service-proxy
-      - DD_VERSION=${DD_VERSION_NGINX:-1.0.0}
+      - DD_VERSION=${DD_VERSION_NGINX:-1.28.0}
     labels:
       com.datadoghq.ad.logs: '[{"source": "nginx"}]'
       com.datadoghq.ad.check_names: '["nginx"]'
@@ -217,7 +243,7 @@ services:
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
     labels:
       com.datadoghq.tags.service: 'store-db'
-      com.datadoghq.tags.version: '${DD_VERSION_POSTGRES:-1.0.0}'
+      com.datadoghq.tags.version: '${DD_VERSION_POSTGRES:-15.0}'
       com.datadoghq.ad.check_names: '["postgres"]'
       com.datadoghq.ad.init_configs: '[{}]'
       com.datadoghq.ad.instances: '[{"host":"%%host%%", "port":5432, "username":"datadog", "password":"datadog"}]'
@@ -234,6 +260,7 @@ services:
       - storedog-network
     labels:
       com.datadoghq.tags.service: 'redis'
+      com.datadoghq.tags.env: '${DD_ENV:-development}'
       com.datadoghq.tags.version: '${DD_VERSION_REDIS:-6.2}'
       com.datadoghq.ad.check_names: '["redisdb"]'
       com.datadoghq.ad.init_configs: '[{}]'

docker-compose.yml

@@ -110,11 +110,14 @@ services:
   ads:
     image: ghcr.io/datadog/storedog/ads-java:${STOREDOG_IMAGE_VERSION:-latest}
     depends_on:
+      - postgres
       - dd-agent
     environment:
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
+      - POSTGRES_USER=${POSTGRES_USER:-postgres}
+      - POSTGRES_HOST=postgres
       - DD_AGENT_HOST=dd-agent
       - DD_SERVICE=store-ads
-      - DD_ENV=${DD_ENV:-production}
       - DD_VERSION=${DD_VERSION_ADS:-1.0.0}
       - DD_LOGS_INJECTION=true
       - DD_PROFILING_ENABLED=true
@@ -125,8 +128,23 @@ services:
     labels:
       com.datadoghq.ad.logs: '[{"source": "java"}]'
       com.datadoghq.tags.service: 'store-ads'
-      com.datadoghq.tags.env: '${DD_ENV:-production}'
       com.datadoghq.tags.version: '${DD_VERSION_ADS:-1.0.0}'
+  # ads-python:
+  #   image: ghcr.io/datadog/storedog/ads:${STOREDOG_IMAGE_VERSION:-latest}
+  #   depends_on:
+  #     - postgres
+  #     - dd-agent
+  #   environment:
+  #     - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
+  #     - POSTGRES_USER=${POSTGRES_USER:-postgres}
+  #     - POSTGRES_HOST=postgres
+  #     - DD_AGENT_HOST=dd-agent
+  #     - DD_SERVICE=store-ads-python
+  #     - DD_VERSION=${DD_VERSION_ADS_PYTHON:-1.0.0}
+  #   networks:
+  #     - storedog-network
+  #   labels:
+  #     com.datadoghq.ad.logs: '[{"source": "python"}]'
   service-proxy:
     image: ghcr.io/datadog/storedog/nginx:${STOREDOG_IMAGE_VERSION:-latest}
     restart: always
@@ -138,9 +156,12 @@ services:
       - frontend
       - dd-agent
     environment:
+      - ADS_A_UPSTREAM=${ADS_A_UPSTREAM:-ads:3030} # Only needed if using AB testing
+      - ADS_B_UPSTREAM=${ADS_B_UPSTREAM:-ads-python:3030} # Only needed if using AB testing
+      - ADS_B_PERCENT=${ADS_B_PERCENT:-0} # Percent of traffic to ads-python; remainder goes to ads-java. Only needed if using AB testing
       - DD_AGENT_HOST=dd-agent
       - DD_SERVICE=service-proxy
-      - DD_VERSION=${DD_VERSION_NGINX:-1.0.0}
+      - DD_VERSION=${DD_VERSION_NGINX:-1.28.0}
     labels:
       com.datadoghq.ad.logs: '[{"source": "nginx"}]'
       com.datadoghq.ad.check_names: '["nginx"]'
@@ -160,7 +181,7 @@ services:
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
     labels:
       com.datadoghq.tags.service: 'store-db'
-      com.datadoghq.tags.version: '${DD_VERSION_POSTGRES:-1.0.0}'
+      com.datadoghq.tags.version: '${DD_VERSION_POSTGRES:-15.0}'
       com.datadoghq.ad.check_names: '["postgres"]'
       com.datadoghq.ad.init_configs: '[{}]'
       com.datadoghq.ad.instances: '[{"host":"%%host%%", "port":5432, "username":"datadog", "password":"datadog"}]'

services/ads/java/Dockerfile

@@ -20,13 +20,16 @@ ARG DD_GIT_REPOSITORY_URL
 ARG DD_GIT_COMMIT_SHA
 
 ENV APP_HOME=/usr/app/ \
-  ADS_SERVER_PORT=3030 \
-  DD_GIT_REPOSITORY_URL=${DD_GIT_REPOSITORY_URL} \
-  DD_GIT_COMMIT_SHA=${DD_GIT_COMMIT_SHA} 
+    ADS_SERVER_PORT=3030 \
+    POSTGRES_PASSWORD=postgres \
+    POSTGRES_USER=postgres \
+    POSTGRES_HOST=postgres \
+    DD_GIT_REPOSITORY_URL=${DD_GIT_REPOSITORY_URL} \
+    DD_GIT_COMMIT_SHA=${DD_GIT_COMMIT_SHA}
 
 WORKDIR $APP_HOME
 COPY --from=temp_build_image $APP_HOME/build/libs/*.jar store-ads.jar
 
 RUN wget -O dd-java-agent.jar 'https://dtdg.co/latest-java-tracer'
 
-ENTRYPOINT ["java", "-javaagent:/usr/app/dd-java-agent.jar", "-jar", "store-ads.jar"]
+ENTRYPOINT ["java", "-javaagent:/usr/app/dd-java-agent.jar", "-jar", "store-ads.jar"]

services/ads/java/build.gradle

@@ -15,11 +15,14 @@ repositories {
 dependencies {
 	implementation ( 
     'commons-io:commons-io:2.4',
-    'org.springframework.boot:spring-boot-starter-web'
+    'org.springframework.boot:spring-boot-starter-web',
+    'org.springframework.boot:spring-boot-starter-data-jpa'
   )
+	runtimeOnly 'org.postgresql:postgresql'
 	implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2'
-  implementation 'net.logstash.logback:logstash-logback-encoder:7.0'
+  	implementation 'net.logstash.logback:logstash-logback-encoder:7.0'
 	testImplementation 'org.springframework.boot:spring-boot-starter-test'
+	testImplementation 'com.h2database:h2'
 }
 
 bootJar {

services/ads/java/src/main/java/adsjava/AdsJavaApplication.java

@@ -15,9 +15,13 @@
 import org.springframework.web.bind.annotation.RequestParam;
 import java.util.concurrent.TimeoutException;
 import org.springframework.web.bind.annotation.RequestHeader;
+import org.springframework.beans.factory.annotation.Autowired;
+import java.util.List;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Bean;
+import org.springframework.boot.CommandLineRunner;
 
 
 @SpringBootApplication
@@ -26,6 +30,9 @@ public class AdsJavaApplication {
 
     private static final Logger logger = LoggerFactory.getLogger(AdsJavaApplication.class);
 
+    @Autowired
+    private AdvertisementRepository advertisementRepository;
+
 	@RequestMapping("/")
 	public String home() {
         logger.info("home url for ads called");
@@ -51,7 +58,7 @@ public String home() {
         value = "/ads",
         produces = MediaType.APPLICATION_JSON_VALUE
     )
-	public HashMap[] ads(@RequestHeader HashMap<String, String> headers) {
+    public List<Advertisement> ads(@RequestHeader HashMap<String, String> headers) {
         logger.info("/ads called");
 
         boolean errorFlag = false;
@@ -74,28 +81,25 @@ public HashMap[] ads(@RequestHeader HashMap<String, String> headers) {
                 throw new RuntimeException(e);
             }
         } else {
-            HashMap<String, String> map1 = new HashMap<>();
-            map1.put("id", "1");
-            map1.put("name", "Discount Clothing");
-            map1.put("path", "1.jpg");
-
-            HashMap<String, String> map2 = new HashMap<>();
-            map2.put("id", "2");
-            map2.put("name", "Cool Hats");
-            map2.put("path", "2.jpg");
-
-            HashMap<String, String> map3 = new HashMap<>();
-            map3.put("id", "3");
-            map3.put("name", "Nic Bags");
-            map3.put("path", "3.jpg");
-            HashMap[] myArr = { map1, map2, map3 };
-            logger.info("Total responses available: " + myArr.length);
-            return myArr;
+            List<Advertisement> ads = advertisementRepository.findAll();
+            logger.info("Total ads available: " + ads.size());
+            return ads;
         }
-	}
+    }
 
 	public static void main(String[] args) {
 		SpringApplication.run(AdsJavaApplication.class, args);
 	}
 
+    @Bean
+    public CommandLineRunner initDb(AdvertisementRepository repository) {
+        return args -> {
+            if (repository.count() == 0) {
+                repository.save(new Advertisement("Discount Clothing", "1.jpg"));
+                repository.save(new Advertisement("Cool Hats", "2.jpg"));
+                repository.save(new Advertisement("Nice Bags", "3.jpg"));
+            }
+        };
+    }
+
 }

services/ads/java/src/main/java/adsjava/Advertisement.java

@@ -0,0 +1,26 @@
+package adsjava;
+
+import javax.persistence.*;
+
+@Entity
+public class Advertisement {
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+    private String name;
+    private String path;
+
+    public Advertisement() {}
+
+    public Advertisement(String name, String path) {
+        this.name = name;
+        this.path = path;
+    }
+
+    public Long getId() { return id; }
+    public void setId(Long id) { this.id = id; }
+    public String getName() { return name; }
+    public void setName(String name) { this.name = name; }
+    public String getPath() { return path; }
+    public void setPath(String path) { this.path = path; }
+} 

services/ads/java/src/main/java/adsjava/AdvertisementRepository.java

@@ -0,0 +1,6 @@
+package adsjava;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+
+public interface AdvertisementRepository extends JpaRepository<Advertisement, Long> {
+} 

services/ads/java/src/main/resources/application.yml

@@ -1,2 +1,12 @@
 server:
   port: ${ADS_SERVER_PORT:3030}
+
+spring:
+  datasource:
+    url: jdbc:postgresql://${POSTGRES_HOST:localhost}:5432/${POSTGRES_USER:postgres}
+    username: ${POSTGRES_USER:postgres}
+    password: ${POSTGRES_PASSWORD:postgres}
+  jpa:
+    hibernate:
+      ddl-auto: update
+    show-sql: true

services/ads/java/src/test/resources/application.yml

@@ -0,0 +1,10 @@
+spring:
+  datasource:
+    url: jdbc:h2:mem:testdb;DB_CLOSE_DELAY=-1
+    driver-class-name: org.h2.Driver
+    username: sa
+    password:
+  jpa:
+    hibernate:
+      ddl-auto: create-drop
+    show-sql: false