From 3b2dd258b729c568f2d9af0e738c80a25b003ccd Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Tue, 1 Jul 2025 10:08:49 +0100 Subject: [PATCH 01/19] Changed ads-java to use postgres for ads and h2 for test --- services/ads/java/Dockerfile | 11 +++-- services/ads/java/build.gradle | 7 +++- .../main/java/adsjava/AdsJavaApplication.java | 42 ++++++++++--------- .../src/main/java/adsjava/Advertisement.java | 26 ++++++++++++ .../java/adsjava/AdvertisementRepository.java | 6 +++ .../java/src/main/resources/application.yml | 10 +++++ .../java/src/test/resources/application.yml | 10 +++++ 7 files changed, 87 insertions(+), 25 deletions(-) create mode 100644 services/ads/java/src/main/java/adsjava/Advertisement.java create mode 100644 services/ads/java/src/main/java/adsjava/AdvertisementRepository.java create mode 100644 services/ads/java/src/test/resources/application.yml diff --git a/services/ads/java/Dockerfile b/services/ads/java/Dockerfile index 6117e3f3..9c467914 100644 --- a/services/ads/java/Dockerfile +++ b/services/ads/java/Dockerfile @@ -20,13 +20,16 @@ ARG DD_GIT_REPOSITORY_URL ARG DD_GIT_COMMIT_SHA ENV APP_HOME=/usr/app/ \ - ADS_SERVER_PORT=3030 \ - DD_GIT_REPOSITORY_URL=${DD_GIT_REPOSITORY_URL} \ - DD_GIT_COMMIT_SHA=${DD_GIT_COMMIT_SHA} + ADS_SERVER_PORT=3030 \ + POSTGRES_PASSWORD=postgres \ + POSTGRES_USER=postgres \ + POSTGRES_HOST=postgres \ + DD_GIT_REPOSITORY_URL=${DD_GIT_REPOSITORY_URL} \ + DD_GIT_COMMIT_SHA=${DD_GIT_COMMIT_SHA} WORKDIR $APP_HOME COPY --from=temp_build_image $APP_HOME/build/libs/*.jar store-ads.jar RUN wget -O dd-java-agent.jar 'https://dtdg.co/latest-java-tracer' -ENTRYPOINT ["java", "-javaagent:/usr/app/dd-java-agent.jar", "-jar", "store-ads.jar"] \ No newline at end of file +ENTRYPOINT ["java", "-javaagent:/usr/app/dd-java-agent.jar", "-jar", "store-ads.jar"] diff --git a/services/ads/java/build.gradle b/services/ads/java/build.gradle index e1f316eb..4425949c 100644 --- a/services/ads/java/build.gradle +++ b/services/ads/java/build.gradle @@ -15,11 +15,14 @@ repositories { dependencies { implementation ( 'commons-io:commons-io:2.4', - 'org.springframework.boot:spring-boot-starter-web' + 'org.springframework.boot:spring-boot-starter-web', + 'org.springframework.boot:spring-boot-starter-data-jpa' ) + runtimeOnly 'org.postgresql:postgresql' implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2' - implementation 'net.logstash.logback:logstash-logback-encoder:7.0' + implementation 'net.logstash.logback:logstash-logback-encoder:7.0' testImplementation 'org.springframework.boot:spring-boot-starter-test' + testImplementation 'com.h2database:h2' } bootJar { diff --git a/services/ads/java/src/main/java/adsjava/AdsJavaApplication.java b/services/ads/java/src/main/java/adsjava/AdsJavaApplication.java index 3ba5148b..b375fc65 100644 --- a/services/ads/java/src/main/java/adsjava/AdsJavaApplication.java +++ b/services/ads/java/src/main/java/adsjava/AdsJavaApplication.java @@ -15,9 +15,13 @@ import org.springframework.web.bind.annotation.RequestParam; import java.util.concurrent.TimeoutException; import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.beans.factory.annotation.Autowired; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.boot.CommandLineRunner; @SpringBootApplication @@ -26,6 +30,9 @@ public class AdsJavaApplication { private static final Logger logger = LoggerFactory.getLogger(AdsJavaApplication.class); + @Autowired + private AdvertisementRepository advertisementRepository; + @RequestMapping("/") public String home() { logger.info("home url for ads called"); @@ -51,7 +58,7 @@ public String home() { value = "/ads", produces = MediaType.APPLICATION_JSON_VALUE ) - public HashMap[] ads(@RequestHeader HashMap headers) { + public List ads(@RequestHeader HashMap headers) { logger.info("/ads called"); boolean errorFlag = false; @@ -74,28 +81,25 @@ public HashMap[] ads(@RequestHeader HashMap headers) { throw new RuntimeException(e); } } else { - HashMap map1 = new HashMap<>(); - map1.put("id", "1"); - map1.put("name", "Discount Clothing"); - map1.put("path", "1.jpg"); - - HashMap map2 = new HashMap<>(); - map2.put("id", "2"); - map2.put("name", "Cool Hats"); - map2.put("path", "2.jpg"); - - HashMap map3 = new HashMap<>(); - map3.put("id", "3"); - map3.put("name", "Nic Bags"); - map3.put("path", "3.jpg"); - HashMap[] myArr = { map1, map2, map3 }; - logger.info("Total responses available: " + myArr.length); - return myArr; + List ads = advertisementRepository.findAll(); + logger.info("Total ads available: " + ads.size()); + return ads; } - } + } public static void main(String[] args) { SpringApplication.run(AdsJavaApplication.class, args); } + @Bean + public CommandLineRunner initDb(AdvertisementRepository repository) { + return args -> { + if (repository.count() == 0) { + repository.save(new Advertisement("Discount Clothing", "1.jpg")); + repository.save(new Advertisement("Cool Hats", "2.jpg")); + repository.save(new Advertisement("Nice Bags", "3.jpg")); + } + }; + } + } diff --git a/services/ads/java/src/main/java/adsjava/Advertisement.java b/services/ads/java/src/main/java/adsjava/Advertisement.java new file mode 100644 index 00000000..edd00986 --- /dev/null +++ b/services/ads/java/src/main/java/adsjava/Advertisement.java @@ -0,0 +1,26 @@ +package adsjava; + +import javax.persistence.*; + +@Entity +public class Advertisement { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + private String name; + private String path; + + public Advertisement() {} + + public Advertisement(String name, String path) { + this.name = name; + this.path = path; + } + + public Long getId() { return id; } + public void setId(Long id) { this.id = id; } + public String getName() { return name; } + public void setName(String name) { this.name = name; } + public String getPath() { return path; } + public void setPath(String path) { this.path = path; } +} \ No newline at end of file diff --git a/services/ads/java/src/main/java/adsjava/AdvertisementRepository.java b/services/ads/java/src/main/java/adsjava/AdvertisementRepository.java new file mode 100644 index 00000000..ae7c56cd --- /dev/null +++ b/services/ads/java/src/main/java/adsjava/AdvertisementRepository.java @@ -0,0 +1,6 @@ +package adsjava; + +import org.springframework.data.jpa.repository.JpaRepository; + +public interface AdvertisementRepository extends JpaRepository { +} \ No newline at end of file diff --git a/services/ads/java/src/main/resources/application.yml b/services/ads/java/src/main/resources/application.yml index a8359851..c1e5936e 100644 --- a/services/ads/java/src/main/resources/application.yml +++ b/services/ads/java/src/main/resources/application.yml @@ -1,2 +1,12 @@ server: port: ${ADS_SERVER_PORT:3030} + +spring: + datasource: + url: jdbc:postgresql://${POSTGRES_HOST:localhost}:5432/${POSTGRES_USER:postgres} + username: ${POSTGRES_USER:postgres} + password: ${POSTGRES_PASSWORD:postgres} + jpa: + hibernate: + ddl-auto: update + show-sql: true diff --git a/services/ads/java/src/test/resources/application.yml b/services/ads/java/src/test/resources/application.yml new file mode 100644 index 00000000..67da9f11 --- /dev/null +++ b/services/ads/java/src/test/resources/application.yml @@ -0,0 +1,10 @@ +spring: + datasource: + url: jdbc:h2:mem:testdb;DB_CLOSE_DELAY=-1 + driver-class-name: org.h2.Driver + username: sa + password: + jpa: + hibernate: + ddl-auto: create-drop + show-sql: false \ No newline at end of file From 4d8156dbe90e8aca0812e2492c712893a15039a6 Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Tue, 1 Jul 2025 10:10:46 +0100 Subject: [PATCH 02/19] added envs to ads in docker compose files --- docker-compose.dev.yml | 3 +++ docker-compose.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index a3e64f66..78d8525e 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -167,6 +167,9 @@ services: depends_on: - dd-agent environment: + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} + - POSTGRES_USER=${POSTGRES_USER:-postgres} + - POSTGRES_HOST=postgres - DD_AGENT_HOST=dd-agent - DD_SERVICE=store-ads - DD_VERSION=${DD_VERSION_ADS:-1.0.0} diff --git a/docker-compose.yml b/docker-compose.yml index cf4041a2..9638f957 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,6 +112,9 @@ services: depends_on: - dd-agent environment: + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} + - POSTGRES_USER=${POSTGRES_USER:-postgres} + - POSTGRES_HOST=postgres - DD_AGENT_HOST=dd-agent - DD_SERVICE=store-ads - DD_ENV=${DD_ENV:-production} From d4ec512d36e748b9a54e4ffe799fdc40a51563a4 Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Tue, 1 Jul 2025 10:52:16 +0100 Subject: [PATCH 03/19] adding AB testing to nginx config --- docker-compose.yml | 16 ++++++++++++++++ services/nginx/default.conf.template | 15 +++++++++++++-- services/nginx/docker-entrypoint.sh | 4 ++-- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9638f957..bb146bac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -130,6 +130,19 @@ services: com.datadoghq.tags.service: 'store-ads' com.datadoghq.tags.env: '${DD_ENV:-production}' com.datadoghq.tags.version: '${DD_VERSION_ADS:-1.0.0}' + ads-python: + image: ghcr.io/datadog/storedog/ads-python:${STOREDOG_IMAGE_VERSION:-latest} + depends_on: + - dd-agent + environment: + - DD_AGENT_HOST=dd-agent + - DD_SERVICE=store-ads-python + - DD_ENV=${DD_ENV:-production} + - DD_VERSION=${DD_VERSION_ADS_PYTHON:-1.0.0} + networks: + - storedog-network + labels: + com.datadoghq.ad.logs: '[{"source": "python"}]' service-proxy: image: ghcr.io/datadog/storedog/nginx:${STOREDOG_IMAGE_VERSION:-latest} restart: always @@ -144,6 +157,9 @@ services: - DD_AGENT_HOST=dd-agent - DD_SERVICE=service-proxy - DD_VERSION=${DD_VERSION_NGINX:-1.0.0} + - ADS_A_UPSTREAM=${ADS_A_UPSTREAM:-ads:3030} + - ADS_B_UPSTREAM=${ADS_B_UPSTREAM:-ads-python:3030} + - ADS_B_PERCENT=${ADS_B_PERCENT:-0} labels: com.datadoghq.ad.logs: '[{"source": "nginx"}]' com.datadoghq.ad.check_names: '["nginx"]' diff --git a/services/nginx/default.conf.template b/services/nginx/default.conf.template index 4e33c8a6..90dc8ab1 100644 --- a/services/nginx/default.conf.template +++ b/services/nginx/default.conf.template @@ -1,6 +1,17 @@ server { listen 80; + # A/B split: each request is randomly assigned based on $msec + split_clients "${msec}" $ads_ab_variant { + ${ADS_B_PERCENT}% B; + * A; + } + + map $ads_ab_variant $ads_upstream { + A ${ADS_A_UPSTREAM}; + B ${ADS_B_UPSTREAM}; + } + # Reverse proxy for ads location /services/ads { proxy_set_header Host $host; @@ -9,8 +20,8 @@ server { proxy_set_header X-Forwarded-Proto $scheme; rewrite ^/services/ads(/.*)$ $1 break; - # Proxy to ads-java, ensure trailing slash for proper URI handling - proxy_pass http://ads:3030/; + # Proxy to ads service based on A/B split + proxy_pass http://$ads_upstream/; } # Reverse proxy for discounts diff --git a/services/nginx/docker-entrypoint.sh b/services/nginx/docker-entrypoint.sh index 6228e137..97835f8b 100644 --- a/services/nginx/docker-entrypoint.sh +++ b/services/nginx/docker-entrypoint.sh @@ -5,8 +5,8 @@ set -e # Use 127.0.0.11 as the default resolver for Docker Compose export NGINX_RESOLVER=${NGINX_RESOLVER:-127.0.0.11} -# Substitute the variable in the template and output the final config -envsubst '$NGINX_RESOLVER' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf +# Substitute all relevant variables in the template and output the final config +envsubst '$NGINX_RESOLVER $ADS_A_UPSTREAM $ADS_B_UPSTREAM $ADS_B_PERCENT' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf # Start NGINX in the foreground exec nginx -g 'daemon off;' From 9b52bd422389b8b7f45d37394216624acd0a8020 Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Tue, 1 Jul 2025 11:12:09 +0100 Subject: [PATCH 04/19] added python and variables to compose file. added default values to .env.template --- .env.template | 16 ++++++++++++---- docker-compose.dev.yml | 24 +++++++++++++++++++++++- docker-compose.yml | 13 ++++++++----- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/.env.template b/.env.template index 34778518..da2b3177 100644 --- a/.env.template +++ b/.env.template @@ -23,8 +23,9 @@ NEXT_PUBLIC_DD_VERSION_FRONTEND=1.0.0 # use NEXT_PUBLIC_* to expose var to store DD_VERSION_BACKEND=1.0.0 # Use for worker service as well, since they use the same Docker image DD_VERSION_DBM=1.0.0 DD_VERSION_ADS=1.0.0 +DD_VERSION_ADS_PYTHON=1.0.0 DD_VERSION_DISCOUNTS=1.0.0 -DD_VERSION_POSTGRES=1.0.0 +DD_VERSION_POSTGRES=15.0 DD_VERSION_NGINX=1.0.0 DD_VERSION_REDIS=6.2 @@ -35,8 +36,15 @@ POSTGRES_USER=postgres POSTGRES_PASSWORD=postgres DB_HOST=postgres DB_PORT=5432 -DB_POOL=25 # Database connection pool size set in backend service (default: 25) -MAX_THREADS=5 # Maximum number of concurrent threads set in backend service (default: 5) +DB_POOL=25 # Database connection pool size set in backend service (default: 25) +MAX_THREADS=5 # Maximum number of concurrent threads set in backend service (default: 5) + +# ============================================= +# Nginx/Service Proxy Configuration +# ============================================= +ADS_A_UPSTREAM=ads:3030 +ADS_B_UPSTREAM=ads-python:3030 +ADS_B_PERCENT=0 # Set < 0 > 100 to split traffic between ads services # ============================================= # Frontend Service Configuration @@ -65,4 +73,4 @@ DISABLE_SPRING=1 STOREDOG_URL=http://service-proxy:80 # base url for storedog service (default: 'http://service-proxy:80') PUPPETEER_TIMEOUT=30000 # timeout for puppeteer (default: 30000) -SKIP_SESSION_CLOSE= # skip session close for puppeteer (default: ''). note that the current puppeteer script doesn't make use of this environment variable but can easily be updated to do so \ No newline at end of file +SKIP_SESSION_CLOSE= # skip session close for puppeteer (default: ''). note that the current puppeteer script doesn't make use of this environment variable but can easily be updated to do so diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 78d8525e..d3a4834a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -182,6 +182,25 @@ services: labels: com.datadoghq.ad.logs: '[{"source": "java"}]' + # OPTIONAL: Advertisement service (Python) + ads-python: + build: + context: ./services/ads/python + depends_on: + - dd-agent + environment: + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} + - POSTGRES_USER=${POSTGRES_USER:-postgres} + - POSTGRES_HOST=postgres + - DD_AGENT_HOST=dd-agent + - DD_SERVICE=store-ads-python + - DD_ENV=${DD_ENV:-production} + - DD_VERSION=${DD_VERSION_ADS_PYTHON:-1.0.0} + networks: + - storedog-network + labels: + com.datadoghq.ad.logs: '[{"source": "python"}]' + # Web server and reverse proxy service-proxy: build: @@ -195,6 +214,9 @@ services: - frontend - dd-agent environment: + - ADS_A_UPSTREAM=${ADS_A_UPSTREAM:-ads:3030} + - ADS_B_UPSTREAM=${ADS_B_UPSTREAM:-ads-python:3030} + - ADS_B_PERCENT=${ADS_B_PERCENT:-0} # Set < 0 > 100 to split traffic between ads services - DD_AGENT_HOST=dd-agent - DD_SERVICE=service-proxy - DD_VERSION=${DD_VERSION_NGINX:-1.0.0} @@ -220,7 +242,7 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} labels: com.datadoghq.tags.service: 'store-db' - com.datadoghq.tags.version: '${DD_VERSION_POSTGRES:-1.0.0}' + com.datadoghq.tags.version: '${DD_VERSION_POSTGRES:-15.0}' com.datadoghq.ad.check_names: '["postgres"]' com.datadoghq.ad.init_configs: '[{}]' com.datadoghq.ad.instances: '[{"host":"%%host%%", "port":5432, "username":"datadog", "password":"datadog"}]' diff --git a/docker-compose.yml b/docker-compose.yml index bb146bac..d08332de 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -131,10 +131,13 @@ services: com.datadoghq.tags.env: '${DD_ENV:-production}' com.datadoghq.tags.version: '${DD_VERSION_ADS:-1.0.0}' ads-python: - image: ghcr.io/datadog/storedog/ads-python:${STOREDOG_IMAGE_VERSION:-latest} + image: ghcr.io/datadog/storedog/ads:${STOREDOG_IMAGE_VERSION:-latest} depends_on: - dd-agent environment: + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} + - POSTGRES_USER=${POSTGRES_USER:-postgres} + - POSTGRES_HOST=postgres - DD_AGENT_HOST=dd-agent - DD_SERVICE=store-ads-python - DD_ENV=${DD_ENV:-production} @@ -154,12 +157,12 @@ services: - frontend - dd-agent environment: + - ADS_A_UPSTREAM=${ADS_A_UPSTREAM:-ads:3030} + - ADS_B_UPSTREAM=${ADS_B_UPSTREAM:-ads-python:3030} + - ADS_B_PERCENT=${ADS_B_PERCENT:-0} # Set < 0 > 100 to split traffic between ads services - DD_AGENT_HOST=dd-agent - DD_SERVICE=service-proxy - DD_VERSION=${DD_VERSION_NGINX:-1.0.0} - - ADS_A_UPSTREAM=${ADS_A_UPSTREAM:-ads:3030} - - ADS_B_UPSTREAM=${ADS_B_UPSTREAM:-ads-python:3030} - - ADS_B_PERCENT=${ADS_B_PERCENT:-0} labels: com.datadoghq.ad.logs: '[{"source": "nginx"}]' com.datadoghq.ad.check_names: '["nginx"]' @@ -179,7 +182,7 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} labels: com.datadoghq.tags.service: 'store-db' - com.datadoghq.tags.version: '${DD_VERSION_POSTGRES:-1.0.0}' + com.datadoghq.tags.version: '${DD_VERSION_POSTGRES:-15.0}' com.datadoghq.ad.check_names: '["postgres"]' com.datadoghq.ad.init_configs: '[{}]' com.datadoghq.ad.instances: '[{"host":"%%host%%", "port":5432, "username":"datadog", "password":"datadog"}]' From 24b81962a4b9ecdbb104b49d2d6f55479bed26b7 Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Tue, 1 Jul 2025 11:13:38 +0100 Subject: [PATCH 05/19] commented out python service since it's optional --- docker-compose.dev.yml | 34 +++++++++++++++++----------------- docker-compose.yml | 32 ++++++++++++++++---------------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d3a4834a..cf3f1b1f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -183,23 +183,23 @@ services: com.datadoghq.ad.logs: '[{"source": "java"}]' # OPTIONAL: Advertisement service (Python) - ads-python: - build: - context: ./services/ads/python - depends_on: - - dd-agent - environment: - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} - - POSTGRES_USER=${POSTGRES_USER:-postgres} - - POSTGRES_HOST=postgres - - DD_AGENT_HOST=dd-agent - - DD_SERVICE=store-ads-python - - DD_ENV=${DD_ENV:-production} - - DD_VERSION=${DD_VERSION_ADS_PYTHON:-1.0.0} - networks: - - storedog-network - labels: - com.datadoghq.ad.logs: '[{"source": "python"}]' + # ads-python: + # build: + # context: ./services/ads/python + # depends_on: + # - dd-agent + # environment: + # - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} + # - POSTGRES_USER=${POSTGRES_USER:-postgres} + # - POSTGRES_HOST=postgres + # - DD_AGENT_HOST=dd-agent + # - DD_SERVICE=store-ads-python + # - DD_ENV=${DD_ENV:-production} + # - DD_VERSION=${DD_VERSION_ADS_PYTHON:-1.0.0} + # networks: + # - storedog-network + # labels: + # com.datadoghq.ad.logs: '[{"source": "python"}]' # Web server and reverse proxy service-proxy: diff --git a/docker-compose.yml b/docker-compose.yml index d08332de..190bc38f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -130,22 +130,22 @@ services: com.datadoghq.tags.service: 'store-ads' com.datadoghq.tags.env: '${DD_ENV:-production}' com.datadoghq.tags.version: '${DD_VERSION_ADS:-1.0.0}' - ads-python: - image: ghcr.io/datadog/storedog/ads:${STOREDOG_IMAGE_VERSION:-latest} - depends_on: - - dd-agent - environment: - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} - - POSTGRES_USER=${POSTGRES_USER:-postgres} - - POSTGRES_HOST=postgres - - DD_AGENT_HOST=dd-agent - - DD_SERVICE=store-ads-python - - DD_ENV=${DD_ENV:-production} - - DD_VERSION=${DD_VERSION_ADS_PYTHON:-1.0.0} - networks: - - storedog-network - labels: - com.datadoghq.ad.logs: '[{"source": "python"}]' + # ads-python: + # image: ghcr.io/datadog/storedog/ads:${STOREDOG_IMAGE_VERSION:-latest} + # depends_on: + # - dd-agent + # environment: + # - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} + # - POSTGRES_USER=${POSTGRES_USER:-postgres} + # - POSTGRES_HOST=postgres + # - DD_AGENT_HOST=dd-agent + # - DD_SERVICE=store-ads-python + # - DD_ENV=${DD_ENV:-production} + # - DD_VERSION=${DD_VERSION_ADS_PYTHON:-1.0.0} + # networks: + # - storedog-network + # labels: + # com.datadoghq.ad.logs: '[{"source": "python"}]' service-proxy: image: ghcr.io/datadog/storedog/nginx:${STOREDOG_IMAGE_VERSION:-latest} restart: always From 508c70947dfc0335d5ee9b042a58c4c484a4936b Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Tue, 1 Jul 2025 11:33:21 +0100 Subject: [PATCH 06/19] moved split outside server block --- services/nginx/default.conf.template | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/services/nginx/default.conf.template b/services/nginx/default.conf.template index 90dc8ab1..493f2da3 100644 --- a/services/nginx/default.conf.template +++ b/services/nginx/default.conf.template @@ -1,16 +1,15 @@ -server { - listen 80; +split_clients "${msec}" $ads_ab_variant { + ${ADS_B_PERCENT}% B; + * A; +} - # A/B split: each request is randomly assigned based on $msec - split_clients "${msec}" $ads_ab_variant { - ${ADS_B_PERCENT}% B; - * A; - } +map $ads_ab_variant $ads_upstream { + A ${ADS_A_UPSTREAM}; + B ${ADS_B_UPSTREAM}; +} - map $ads_ab_variant $ads_upstream { - A ${ADS_A_UPSTREAM}; - B ${ADS_B_UPSTREAM}; - } +server { + listen 80; # Reverse proxy for ads location /services/ads { From 25961747a27dd192881ed463ad7f629d06a8201b Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Tue, 1 Jul 2025 12:42:30 +0100 Subject: [PATCH 07/19] changed split behavior and logic --- docker-compose.dev.yml | 2 +- docker-compose.yml | 2 +- services/nginx/default.conf.template | 18 +++++++++++------- services/nginx/docker-entrypoint.sh | 15 ++++++++++++++- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index cf3f1b1f..b302b859 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -216,7 +216,7 @@ services: environment: - ADS_A_UPSTREAM=${ADS_A_UPSTREAM:-ads:3030} - ADS_B_UPSTREAM=${ADS_B_UPSTREAM:-ads-python:3030} - - ADS_B_PERCENT=${ADS_B_PERCENT:-0} # Set < 0 > 100 to split traffic between ads services + - ADS_B_PERCENT=${ADS_B_PERCENT:-0} # Percent of traffic to ads-python; remainder goes to ads-java - DD_AGENT_HOST=dd-agent - DD_SERVICE=service-proxy - DD_VERSION=${DD_VERSION_NGINX:-1.0.0} diff --git a/docker-compose.yml b/docker-compose.yml index 190bc38f..254930d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: environment: - ADS_A_UPSTREAM=${ADS_A_UPSTREAM:-ads:3030} - ADS_B_UPSTREAM=${ADS_B_UPSTREAM:-ads-python:3030} - - ADS_B_PERCENT=${ADS_B_PERCENT:-0} # Set < 0 > 100 to split traffic between ads services + - ADS_B_PERCENT=${ADS_B_PERCENT:-0} # Percent of traffic to ads-python; remainder goes to ads-java - DD_AGENT_HOST=dd-agent - DD_SERVICE=service-proxy - DD_VERSION=${DD_VERSION_NGINX:-1.0.0} diff --git a/services/nginx/default.conf.template b/services/nginx/default.conf.template index 493f2da3..0f7aad0d 100644 --- a/services/nginx/default.conf.template +++ b/services/nginx/default.conf.template @@ -1,11 +1,15 @@ -split_clients "${msec}" $ads_ab_variant { - ${ADS_B_PERCENT}% B; - * A; +upstream ads_service_a { + server ${ADS_A_UPSTREAM} max_fails=1 fail_timeout=1s; } -map $ads_ab_variant $ads_upstream { - A ${ADS_A_UPSTREAM}; - B ${ADS_B_UPSTREAM}; +upstream ads_service_b { + server ${ADS_B_UPSTREAM} max_fails=1 fail_timeout=1s; +} + +# Conditional upstream based on what's enabled +upstream ads_upstream { + # This will be populated based on environment variables + ${UPSTREAM_CONFIG} } server { @@ -20,7 +24,7 @@ server { rewrite ^/services/ads(/.*)$ $1 break; # Proxy to ads service based on A/B split - proxy_pass http://$ads_upstream/; + proxy_pass http://ads_upstream/; } # Reverse proxy for discounts diff --git a/services/nginx/docker-entrypoint.sh b/services/nginx/docker-entrypoint.sh index 97835f8b..1e421917 100644 --- a/services/nginx/docker-entrypoint.sh +++ b/services/nginx/docker-entrypoint.sh @@ -5,8 +5,21 @@ set -e # Use 127.0.0.11 as the default resolver for Docker Compose export NGINX_RESOLVER=${NGINX_RESOLVER:-127.0.0.11} +# Calculate upstream config based on ADS_B_PERCENT +ADS_B_PERCENT=${ADS_B_PERCENT:-0} + +if [ -z "$ADS_B_PERCENT" ] || [ "$ADS_B_PERCENT" -eq 0 ]; then + UPSTREAM_CONFIG="server ${ADS_A_UPSTREAM};" +else + ADS_A_WEIGHT=$((100 - ADS_B_PERCENT)) + ADS_B_WEIGHT=$ADS_B_PERCENT + UPSTREAM_CONFIG="server ${ADS_A_UPSTREAM} weight=${ADS_A_WEIGHT}; server ${ADS_B_UPSTREAM} weight=${ADS_B_WEIGHT};" +fi + +export UPSTREAM_CONFIG + # Substitute all relevant variables in the template and output the final config -envsubst '$NGINX_RESOLVER $ADS_A_UPSTREAM $ADS_B_UPSTREAM $ADS_B_PERCENT' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf +envsubst '$NGINX_RESOLVER $ADS_A_UPSTREAM $ADS_B_UPSTREAM $UPSTREAM_CONFIG' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf # Start NGINX in the foreground exec nginx -g 'daemon off;' From 9afa21ea320be89ba940e1f03a4a2091b2bbd791 Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Tue, 1 Jul 2025 13:13:14 +0100 Subject: [PATCH 08/19] conditional add service block --- services/nginx/default.conf.template | 4 +--- services/nginx/docker-entrypoint.sh | 5 ++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/services/nginx/default.conf.template b/services/nginx/default.conf.template index 0f7aad0d..003dd161 100644 --- a/services/nginx/default.conf.template +++ b/services/nginx/default.conf.template @@ -2,9 +2,7 @@ upstream ads_service_a { server ${ADS_A_UPSTREAM} max_fails=1 fail_timeout=1s; } -upstream ads_service_b { - server ${ADS_B_UPSTREAM} max_fails=1 fail_timeout=1s; -} +${ADS_SERVICE_B_BLOCK} # Conditional upstream based on what's enabled upstream ads_upstream { diff --git a/services/nginx/docker-entrypoint.sh b/services/nginx/docker-entrypoint.sh index 1e421917..453684ee 100644 --- a/services/nginx/docker-entrypoint.sh +++ b/services/nginx/docker-entrypoint.sh @@ -9,17 +9,20 @@ export NGINX_RESOLVER=${NGINX_RESOLVER:-127.0.0.11} ADS_B_PERCENT=${ADS_B_PERCENT:-0} if [ -z "$ADS_B_PERCENT" ] || [ "$ADS_B_PERCENT" -eq 0 ]; then + ADS_SERVICE_B_BLOCK="" UPSTREAM_CONFIG="server ${ADS_A_UPSTREAM};" else + ADS_SERVICE_B_BLOCK="upstream ads_service_b {\n server ${ADS_B_UPSTREAM} max_fails=1 fail_timeout=1s;\n}" ADS_A_WEIGHT=$((100 - ADS_B_PERCENT)) ADS_B_WEIGHT=$ADS_B_PERCENT UPSTREAM_CONFIG="server ${ADS_A_UPSTREAM} weight=${ADS_A_WEIGHT}; server ${ADS_B_UPSTREAM} weight=${ADS_B_WEIGHT};" fi +export ADS_SERVICE_B_BLOCK export UPSTREAM_CONFIG # Substitute all relevant variables in the template and output the final config -envsubst '$NGINX_RESOLVER $ADS_A_UPSTREAM $ADS_B_UPSTREAM $UPSTREAM_CONFIG' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf +envsubst '$NGINX_RESOLVER $ADS_A_UPSTREAM $ADS_B_UPSTREAM $UPSTREAM_CONFIG $ADS_SERVICE_B_BLOCK' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf # Start NGINX in the foreground exec nginx -g 'daemon off;' From 53df946d6bd9389ac14f7b90af0edb2ce0194734 Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Tue, 1 Jul 2025 13:20:31 +0100 Subject: [PATCH 09/19] changed format of service block variable --- services/nginx/docker-entrypoint.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/nginx/docker-entrypoint.sh b/services/nginx/docker-entrypoint.sh index 453684ee..c39437c6 100644 --- a/services/nginx/docker-entrypoint.sh +++ b/services/nginx/docker-entrypoint.sh @@ -12,7 +12,11 @@ if [ -z "$ADS_B_PERCENT" ] || [ "$ADS_B_PERCENT" -eq 0 ]; then ADS_SERVICE_B_BLOCK="" UPSTREAM_CONFIG="server ${ADS_A_UPSTREAM};" else - ADS_SERVICE_B_BLOCK="upstream ads_service_b {\n server ${ADS_B_UPSTREAM} max_fails=1 fail_timeout=1s;\n}" + read -r -d '' ADS_SERVICE_B_BLOCK < Date: Tue, 1 Jul 2025 13:24:15 +0100 Subject: [PATCH 10/19] changed syntax of B_BLOCK variable to better shell compliance --- services/nginx/docker-entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/nginx/docker-entrypoint.sh b/services/nginx/docker-entrypoint.sh index c39437c6..8e64bdf1 100644 --- a/services/nginx/docker-entrypoint.sh +++ b/services/nginx/docker-entrypoint.sh @@ -12,11 +12,12 @@ if [ -z "$ADS_B_PERCENT" ] || [ "$ADS_B_PERCENT" -eq 0 ]; then ADS_SERVICE_B_BLOCK="" UPSTREAM_CONFIG="server ${ADS_A_UPSTREAM};" else - read -r -d '' ADS_SERVICE_B_BLOCK < Date: Tue, 1 Jul 2025 13:38:17 +0100 Subject: [PATCH 11/19] fixing env tags --- docker-compose.dev.yml | 2 +- docker-compose.yml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index b302b859..96171522 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -194,7 +194,6 @@ services: # - POSTGRES_HOST=postgres # - DD_AGENT_HOST=dd-agent # - DD_SERVICE=store-ads-python - # - DD_ENV=${DD_ENV:-production} # - DD_VERSION=${DD_VERSION_ADS_PYTHON:-1.0.0} # networks: # - storedog-network @@ -259,6 +258,7 @@ services: - storedog-network labels: com.datadoghq.tags.service: 'redis' + com.datadoghq.tags.env: '${DD_ENV:-development}' com.datadoghq.tags.version: '${DD_VERSION_REDIS:-6.2}' com.datadoghq.ad.check_names: '["redisdb"]' com.datadoghq.ad.init_configs: '[{}]' diff --git a/docker-compose.yml b/docker-compose.yml index 254930d9..c61e3c3e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -117,7 +117,6 @@ services: - POSTGRES_HOST=postgres - DD_AGENT_HOST=dd-agent - DD_SERVICE=store-ads - - DD_ENV=${DD_ENV:-production} - DD_VERSION=${DD_VERSION_ADS:-1.0.0} - DD_LOGS_INJECTION=true - DD_PROFILING_ENABLED=true @@ -128,7 +127,6 @@ services: labels: com.datadoghq.ad.logs: '[{"source": "java"}]' com.datadoghq.tags.service: 'store-ads' - com.datadoghq.tags.env: '${DD_ENV:-production}' com.datadoghq.tags.version: '${DD_VERSION_ADS:-1.0.0}' # ads-python: # image: ghcr.io/datadog/storedog/ads:${STOREDOG_IMAGE_VERSION:-latest} @@ -140,7 +138,6 @@ services: # - POSTGRES_HOST=postgres # - DD_AGENT_HOST=dd-agent # - DD_SERVICE=store-ads-python - # - DD_ENV=${DD_ENV:-production} # - DD_VERSION=${DD_VERSION_ADS_PYTHON:-1.0.0} # networks: # - storedog-network From f155a3ab4326eb73b172cc399f7fa16625fc0362 Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Tue, 1 Jul 2025 13:52:57 +0100 Subject: [PATCH 12/19] added depends on postrgres for ads services --- docker-compose.dev.yml | 2 ++ docker-compose.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 96171522..b9d03724 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -165,6 +165,7 @@ services: build: context: ./services/ads/java depends_on: + - postgres - dd-agent environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} @@ -187,6 +188,7 @@ services: # build: # context: ./services/ads/python # depends_on: + # - postgres # - dd-agent # environment: # - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} diff --git a/docker-compose.yml b/docker-compose.yml index c61e3c3e..5673501b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -110,6 +110,7 @@ services: ads: image: ghcr.io/datadog/storedog/ads-java:${STOREDOG_IMAGE_VERSION:-latest} depends_on: + - postgres - dd-agent environment: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} @@ -131,6 +132,7 @@ services: # ads-python: # image: ghcr.io/datadog/storedog/ads:${STOREDOG_IMAGE_VERSION:-latest} # depends_on: + # - postgres # - dd-agent # environment: # - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} From e098b284cd5f42ce78cbb56a92ae374526dfe365 Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Tue, 1 Jul 2025 14:29:39 +0100 Subject: [PATCH 13/19] added details to readme --- .env.template | 2 +- README.md | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.env.template b/.env.template index da2b3177..c757ec50 100644 --- a/.env.template +++ b/.env.template @@ -26,7 +26,7 @@ DD_VERSION_ADS=1.0.0 DD_VERSION_ADS_PYTHON=1.0.0 DD_VERSION_DISCOUNTS=1.0.0 DD_VERSION_POSTGRES=15.0 -DD_VERSION_NGINX=1.0.0 +DD_VERSION_NGINX=1.28.0 DD_VERSION_REDIS=6.2 # ============================================= diff --git a/README.md b/README.md index e6cfa666..1b543add 100644 --- a/README.md +++ b/README.md @@ -154,15 +154,15 @@ Common Datadog variables that can be set in application services: - `DD_LOGS_INJECTION`: Enable log injection into traces. Some languages' trace libraries turn this on by default, but we turn it on explicitly to prevent confusion. - `DD_PROFILING_ENABLED`: Enable the Continuous Profiler - `DD_RUNTIME_METRICS_ENABLED`: Enable runtime metrics -- Service-specific versions: - `DD_VERSION_FRONTEND`: Frontend version (default: `1.0.0`) - `DD_VERSION_BACKEND`: Backend version (default: `1.0.0`) - `DD_VERSION_DISCOUNTS`: Discounts service version (default: `1.0.0`) - `DD_VERSION_ADS`: Ads service version (default: `1.0.0`) -- `DD_VERSION_NGINX`: nginx service version (default: `1.0.0`) -- `DD_VERSION_POSTGRES`: PostgreSQL service version (default: `1.0.0`) +- `DD_VERSION_ADS_PYTHON`: Ads Python service version (default: `1.0.0`) +- `DD_VERSION_NGINX`: nginx service version (default: `1.28.0`) +- `DD_VERSION_POSTGRES`: PostgreSQL service version (default: `15.0`) - `DD_VERSION_REDIS`: Redis version (default: `6.2`) > [!NOTE] @@ -186,6 +186,17 @@ Puppeteer service configuration: - `PUPPETEER_TIMEOUT`: Sets max timeout for Puppeteer, in case the session is unresponsive - `SKIP_SESSION_CLOSE`: Skip closing browser sessions +### Nginx/Service Proxy Configuration Variables + +These variables control the upstream configuration for the ads services in the Nginx (service-proxy) container, enabling A/B testing and traffic splitting between the Java and Python ads services. + +> [!IMPORTANT] +> When `ADS_B_PERCENT` is greater than zero, the `ADS_B_UPSTREAM` endpoint must be reachable. Otherwise, the service-proxy will crash and restart. + +- `ADS_A_UPSTREAM`: Host and port for the primary (A) ads service (default: `ads:3030`) +- `ADS_B_UPSTREAM`: Host and port for the secondary (B) ads service (default: `ads-python:3030`) +- `ADS_B_PERCENT`: Percentage of traffic to route to the B (Python) ads service (default: `0`). The remainder goes to the A (Java) ads service. Set to a value between `0` and `100` to control the split. + ## Feature flags Some capabilities are hidden behind feature flags, which can be controlled via `services/frontend/site/featureFlags.config.json`. From 74c6fcbd64a8980bff95a44572b700ecaac6e33a Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Tue, 1 Jul 2025 14:31:26 +0100 Subject: [PATCH 14/19] added comments for ab testing --- docker-compose.dev.yml | 6 +++--- docker-compose.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index b9d03724..4f544af1 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -215,9 +215,9 @@ services: - frontend - dd-agent environment: - - ADS_A_UPSTREAM=${ADS_A_UPSTREAM:-ads:3030} - - ADS_B_UPSTREAM=${ADS_B_UPSTREAM:-ads-python:3030} - - ADS_B_PERCENT=${ADS_B_PERCENT:-0} # Percent of traffic to ads-python; remainder goes to ads-java + - ADS_A_UPSTREAM=${ADS_A_UPSTREAM:-ads:3030} # Only needed if using AB testing + - ADS_B_UPSTREAM=${ADS_B_UPSTREAM:-ads-python:3030} # Only needed if using AB testing + - ADS_B_PERCENT=${ADS_B_PERCENT:-0} # Percent of traffic to ads-python; remainder goes to ads-java. Only needed if using AB testing - DD_AGENT_HOST=dd-agent - DD_SERVICE=service-proxy - DD_VERSION=${DD_VERSION_NGINX:-1.0.0} diff --git a/docker-compose.yml b/docker-compose.yml index 5673501b..221725ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -156,9 +156,9 @@ services: - frontend - dd-agent environment: - - ADS_A_UPSTREAM=${ADS_A_UPSTREAM:-ads:3030} - - ADS_B_UPSTREAM=${ADS_B_UPSTREAM:-ads-python:3030} - - ADS_B_PERCENT=${ADS_B_PERCENT:-0} # Percent of traffic to ads-python; remainder goes to ads-java + - ADS_A_UPSTREAM=${ADS_A_UPSTREAM:-ads:3030} # Only needed if using AB testing + - ADS_B_UPSTREAM=${ADS_B_UPSTREAM:-ads-python:3030} # Only needed if using AB testing + - ADS_B_PERCENT=${ADS_B_PERCENT:-0} # Percent of traffic to ads-python; remainder goes to ads-java. Only needed if using AB testing - DD_AGENT_HOST=dd-agent - DD_SERVICE=service-proxy - DD_VERSION=${DD_VERSION_NGINX:-1.0.0} From e2a3a90f09ac0432c130b8dee03d023c9c722194 Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Tue, 1 Jul 2025 14:32:37 +0100 Subject: [PATCH 15/19] added attention block --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b543add..690f353e 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,7 @@ Puppeteer service configuration: These variables control the upstream configuration for the ads services in the Nginx (service-proxy) container, enabling A/B testing and traffic splitting between the Java and Python ads services. > [!IMPORTANT] -> When `ADS_B_PERCENT` is greater than zero, the `ADS_B_UPSTREAM` endpoint must be reachable. Otherwise, the service-proxy will crash and restart. +> When `ADS_B_PERCENT` is greater than zero, the `ADS_B_UPSTREAM` endpoint must be reachable. Otherwise, the service-proxy will crash and restart. Uncomment the `ads-python` section in `docker-compose.yml` to enable the service. - `ADS_A_UPSTREAM`: Host and port for the primary (A) ads service (default: `ads:3030`) - `ADS_B_UPSTREAM`: Host and port for the secondary (B) ads service (default: `ads-python:3030`) From 7f58de4f4890d833df1e390842797828efd5708f Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Tue, 1 Jul 2025 14:39:10 +0100 Subject: [PATCH 16/19] set default ads_a --- services/nginx/docker-entrypoint.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/nginx/docker-entrypoint.sh b/services/nginx/docker-entrypoint.sh index 8e64bdf1..45a2b1df 100644 --- a/services/nginx/docker-entrypoint.sh +++ b/services/nginx/docker-entrypoint.sh @@ -4,6 +4,8 @@ set -e # Use 127.0.0.11 as the default resolver for Docker Compose export NGINX_RESOLVER=${NGINX_RESOLVER:-127.0.0.11} +# Set the default ADS_A_UPSTREAM to the Java ads service +export ADS_A_UPSTREAM=${ADS_A_UPSTREAM:-ads:3030} # Calculate upstream config based on ADS_B_PERCENT ADS_B_PERCENT=${ADS_B_PERCENT:-0} From 2c1869289fc8157b610217eed288bc69c27e8427 Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Tue, 1 Jul 2025 14:57:33 +0100 Subject: [PATCH 17/19] updated default nginx version --- docker-compose.dev.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 4f544af1..11c9c77b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -220,7 +220,7 @@ services: - ADS_B_PERCENT=${ADS_B_PERCENT:-0} # Percent of traffic to ads-python; remainder goes to ads-java. Only needed if using AB testing - DD_AGENT_HOST=dd-agent - DD_SERVICE=service-proxy - - DD_VERSION=${DD_VERSION_NGINX:-1.0.0} + - DD_VERSION=${DD_VERSION_NGINX:-1.28.0} labels: com.datadoghq.ad.logs: '[{"source": "nginx"}]' com.datadoghq.ad.check_names: '["nginx"]' diff --git a/docker-compose.yml b/docker-compose.yml index 221725ce..f66d9a25 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -161,7 +161,7 @@ services: - ADS_B_PERCENT=${ADS_B_PERCENT:-0} # Percent of traffic to ads-python; remainder goes to ads-java. Only needed if using AB testing - DD_AGENT_HOST=dd-agent - DD_SERVICE=service-proxy - - DD_VERSION=${DD_VERSION_NGINX:-1.0.0} + - DD_VERSION=${DD_VERSION_NGINX:-1.28.0} labels: com.datadoghq.ad.logs: '[{"source": "nginx"}]' com.datadoghq.ad.check_names: '["nginx"]' From c09e2442580e9248d073a8349fdde485e049cf3b Mon Sep 17 00:00:00 2001 From: Shawn Potts <77298184+spottsdd@users.noreply.github.com> Date: Thu, 3 Jul 2025 09:13:43 -0700 Subject: [PATCH 18/19] Update .env.template Co-authored-by: Tara Schofield --- .env.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.template b/.env.template index c757ec50..af77ed45 100644 --- a/.env.template +++ b/.env.template @@ -73,4 +73,4 @@ DISABLE_SPRING=1 STOREDOG_URL=http://service-proxy:80 # base url for storedog service (default: 'http://service-proxy:80') PUPPETEER_TIMEOUT=30000 # timeout for puppeteer (default: 30000) -SKIP_SESSION_CLOSE= # skip session close for puppeteer (default: ''). note that the current puppeteer script doesn't make use of this environment variable but can easily be updated to do so +SKIP_SESSION_CLOSE= # skip session close for puppeteer (default: ''). Note that the current puppeteer script doesn't make use of this environment variable but can easily be updated to do so From 14dd83ea65c265d2b5e4e3025fbb1835408143d1 Mon Sep 17 00:00:00 2001 From: Shawn Potts Date: Thu, 3 Jul 2025 17:52:52 +0100 Subject: [PATCH 19/19] added details to readme --- README.md | 32 +++++++++++++++++++++++++------- services/ads/README.md | 9 +++++---- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 690f353e..868cdabf 100644 --- a/README.md +++ b/README.md @@ -195,9 +195,14 @@ These variables control the upstream configuration for the ads services in the N - `ADS_A_UPSTREAM`: Host and port for the primary (A) ads service (default: `ads:3030`) - `ADS_B_UPSTREAM`: Host and port for the secondary (B) ads service (default: `ads-python:3030`) -- `ADS_B_PERCENT`: Percentage of traffic to route to the B (Python) ads service (default: `0`). The remainder goes to the A (Java) ads service. Set to a value between `0` and `100` to control the split. +- `ADS_B_PERCENT`: Percentage of traffic to route to the B (Python) ads service (default: `0`). The remainder goes to the A (Java) ads service. + - Set to a value between `0` and `100` to control the split. -## Feature flags +## Optional features + +There are several features that can be enabled by setting environment variables and feature flags. + +### Feature flags Some capabilities are hidden behind feature flags, which can be controlled via `services/frontend/site/featureFlags.config.json`. > [!NOTE] @@ -208,7 +213,7 @@ Some capabilities are hidden behind feature flags, which can be controlled via ` > - ./services/frontend/site/featureFlags.config.json:/app/featureFlags.config.json > ``` -### dbm +#### dbm Enables a product ticker on the homepage with a long-running query to demonstrate DBM. **How to use**: @@ -220,7 +225,7 @@ Enables a product ticker on the homepage with a long-running query to demonstrat You can modify the ticker functionality in `services/frontend/components/common/NavBar.tsx`. -### error-tracking +#### error-tracking Introduces an exception in the Ads services to demonstrate Error Tracking by setting a header in to a value that is not expected by the Ads service. **How to use**: @@ -231,7 +236,7 @@ Introduces an exception in the Ads services to demonstrate Error Tracking by set Modify this functionality in `services/frontend/components/common/Ad/Ad.tsx` and respective Ads service being used. -### api-errors +#### api-errors This introduces random errors that occur in the frontend service's `/api` routes. **How to use**: @@ -241,7 +246,7 @@ This introduces random errors that occur in the frontend service's `/api` routes Modify this functionality in `services/frontend/pages/api/*`. -### product-card-frustration +#### product-card-frustration This will swap out the product card component with a version that doesn't have the thumbnails linked to the product page. When paired with the Puppeteer service, this can be used to demonstrate Frustration Signals in RUM. **How to use**: @@ -251,6 +256,19 @@ This will swap out the product card component with a version that doesn't have t Modify this functionality in `services/frontend/components/Product/ProductCard.tsx` and `services/frontend/components/Product/ProductCard-v2.tsx`. +### A/B Testing Ads services + +Run two Ads services and split traffic between them. The amount of traffic sent to each service is set with a percent value. + +**How to use** +1. Add a second Ads service to the `docker-compose.yml` +1. Add and set these environment variables to the `service-proxy` service: + - `ADS_A_UPSTREAM`: Host and port for the primary (A) ads service (default: `ads:3030`) + - `ADS_B_UPSTREAM`: Host and port for the secondary (B) ads service (default: `ads-python:3030`) + - `ADS_B_PERCENT`: Percentage of traffic to route to the B (Python) ads service (default: `0`). The remainder goes to the A (Java) ads service. + - Set to a value between `0` and `100` to control the split. +1. Start the app via `docker compose up` + ## Image publication Images are stored in GHCR. On PR merges, only the affected services will be pushed to GHCR, using the `latest` tag. For example, if you only made changes to the `backend` service, then only the `backend` Github workflow will trigger and publish `ghcr.io/datadog/storedog/backend:latest`. @@ -263,7 +281,7 @@ All of the services in the Storedog application are Dockerized and run in contai Below is a breakdown of services and some instructions on how to use them. ### Ads -There are two advertisement services, the default service is built in Java and there is another option available in Python. These services do the same thing, have the same endpoints, run on the same port (`3030`), and have the same failure modes. The biggest difference is the Python service uses a Postgres database to store the ads, while the Java service uses an in-memory list. These ads are served through the `Ads.tsx` component in the frontend service. +There are two advertisement services, the default service is built in Java and there is another option available in Python. These services do the same thing, have the same endpoints, run on the same port (`3030`), and have the same failure modes. These ads are served through the `Ads.tsx` component in the frontend service. To switch between the Java and Python services, see the instructions in the [Ads service README](./services/ads/README.md). diff --git a/services/ads/README.md b/services/ads/README.md index 3b6d1eb0..d2bdcfb9 100644 --- a/services/ads/README.md +++ b/services/ads/README.md @@ -4,6 +4,10 @@ This service is responsible for managing the banner advertisements served to the frontend service of the application. There are two variations of this service, one uses Python and the other uses Java. +## Java service + +The Java service is the default used with Storedog. It uses the Spring framework. It uses the PostgreSQL JDBC driver to connect to a PostgreSQL database. It uses an H2 in-memory database when running build tests. + ## Python service The Python service is a Flask application that uses SQLAlchemy to connect to a PostgreSQL database. The service is packaged as a Docker image and typically used in a Docker Compose file (see the root of this repo). @@ -163,7 +167,4 @@ ads: volumes: - ./services/ads/python:/app labels: - com.datadoghq.ad.logs: '[{"source": "python", "service": "store-ads"}]' - com.datadoghq.tags.env: '${DD_ENV-dev}' - com.datadoghq.tags.service: 'store-ads' - com.datadoghq.tags.version: ${DD_VERSION_ADS-1.0.0} + com.datadoghq.ad.logs: '[{"source": "python"}]'