Skip to content

Commit 05622b1

Browse files
pablomartinezbernardopablomartinezbernardo
authored
Split integration tests in groups (#7733)
## Summary of changes * Extract integration test running logic to template * Split `DockerTest` job in two arbitrary groups * Categorize tests with a new `DockerGroup` trait indicating which group they belong to * Categorize docker containers using [service profiles](https://docs.docker.com/compose/how-tos/profiles/) ## Reason for change The number of containers in a single job was starting to be problematic for startup. This approach splits images in two groups (attempting to keep them similarly weighted) easily extensible to more groups.
1 parent 0e80efe commit 05622b1

40 files changed

+97
-43
lines changed

‎.azure-pipelines/ultimate-pipeline.yml‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2302,6 +2302,7 @@ stages:
23022302
variables:
23032303
TestAllPackageVersions: true
23042304
IncludeMinorPackageVersions: $[eq(variables.perform_comprehensive_testing, 'true')]
2305+
IntegrationTestFilter: DockerGroup=$(dockerGroup)
23052306

23062307
pool:
23072308
name: $(linuxX64Pool)
@@ -2330,19 +2331,20 @@ stages:
23302331
retryCountForRunCommand: 3
23312332

23322333
- script: |
2333-
docker-compose -p $(DockerComposeProjectName) build --build-arg baseImage=$(baseImage) --build-arg framework=$(publishTargetFramework) --build-arg CodeCoverageEnabled=$(CodeCoverageEnabled) --build-arg Filter=$(IntegrationTestFilter) --build-arg SampleName=$(IntegrationTestSampleName) IntegrationTests
2334-
docker-compose -p $(DockerComposeProjectName) run --rm StartDependencies
2334+
docker-compose -p $(DockerComposeProjectName)-g$(dockerGroup) build --build-arg baseImage=$(baseImage) --build-arg framework=$(publishTargetFramework) --build-arg CodeCoverageEnabled=$(CodeCoverageEnabled) --build-arg Filter=$(IntegrationTestFilter) --build-arg SampleName=$(IntegrationTestSampleName) IntegrationTests
2335+
docker-compose -p $(DockerComposeProjectName)-g$(dockerGroup) run --rm StartDependencies.Group$(dockerGroup)
23352336
env:
23362337
baseImage: $(baseImage)
23372338
framework: $(publishTargetFramework)
23382339
Filter: $(IntegrationTestFilter)
23392340
SampleName: $(IntegrationTestSampleName)
23402341
DD_LOGGER_DD_API_KEY: $(ddApiKey)
2341-
displayName: docker-compose build IntegrationTests and run StartDependencies
2342+
COMPOSE_PROFILES: group$(dockerGroup)
2343+
displayName: docker-compose build IntegrationTests and run StartDependencies (Group $(dockerGroup))
23422344
retryCountOnTaskFailure: 5
23432345
23442346
- script: |
2345-
docker-compose -f docker-compose.yml -p $(DockerComposeProjectName) \
2347+
docker-compose -f docker-compose.yml -p $(DockerComposeProjectName)-g$(dockerGroup) \
23462348
run --rm \
23472349
-e baseImage=$(baseImage) \
23482350
-e framework=$(publishTargetFramework) \
@@ -2352,22 +2354,25 @@ stages:
23522354
-e SampleName=$(IntegrationTestSampleName) \
23532355
-e Area=$(area) \
23542356
IntegrationTests
2355-
displayName: docker-compose run IntegrationTests
2357+
displayName: docker-compose run IntegrationTests (Group $(dockerGroup))
23562358
env:
23572359
DD_LOGGER_DD_API_KEY: $(ddApiKey)
23582360
baseImage: $(baseImage) # for interpolation in the docker-compose file
2361+
COMPOSE_PROFILES: group$(dockerGroup)
23592362
2360-
- script: docker-compose -f docker-compose.yml -p $(DockerComposeProjectName) logs
2363+
- script: docker-compose -f docker-compose.yml -p $(DockerComposeProjectName)-g$(dockerGroup) logs
23612364
displayName: docker-compose logs
23622365
env:
23632366
DD_LOGGER_DD_API_KEY: $(ddApiKey)
2367+
COMPOSE_PROFILES: group$(dockerGroup)
23642368
condition: succeededOrFailed()
23652369
continueOnError: true
23662370

2367-
- script: docker-compose -f docker-compose.yml -p $(DockerComposeProjectName) down
2371+
- script: docker-compose -f docker-compose.yml -p $(DockerComposeProjectName)-g$(dockerGroup) down
23682372
displayName: docker-compose stop services
23692373
env:
23702374
DD_LOGGER_DD_API_KEY: $(ddApiKey)
2375+
COMPOSE_PROFILES: group$(dockerGroup)
23712376
condition: succeededOrFailed()
23722377
continueOnError: true
23732378

‎docker-compose.yml‎

Lines changed: 42 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ services:
8989
# Dependencies
9090
localstack:
9191
image: localstack/localstack
92+
profiles: ["group2"]
9293
environment:
9394
- SERVICES=sns,sqs,kinesis,dynamodb,events,s3,stepfunctions,lambda
9495
- DEBUG=1
@@ -101,52 +102,60 @@ services:
101102

102103
rabbitmq:
103104
image: rabbitmq:3-management
105+
profiles: ["group1"]
104106
command: rabbitmq-server
105107
ports:
106108
- "127.0.0.1:5672:5672"
107109
- "127.0.0.1:15672:15672"
108110

109111
servicestackredis:
110112
image: redis:4-alpine
113+
profiles: ["group1"]
111114
command: redis-server --bind 0.0.0.0
112115
ports:
113116
- "127.0.0.1:6379:6379"
114117

115118
stackexchangeredis:
116119
image: redis:4-alpine
120+
profiles: ["group1"]
117121
hostname: stackexchangeredis
118122
command: redis-server --bind 0.0.0.0
119123
ports:
120124
- "127.0.0.1:6389:6379"
121125

122126
stackexchangeredis-replica:
123127
image: redis:4-alpine
128+
profiles: ["group1"]
124129
hostname: stackexchangeredis-replica
125130
command: redis-server --bind 0.0.0.0 --slaveof stackexchangeredis 6379
126131
ports:
127132
- "127.0.0.1:6390:6379"
128133

129134
stackexchangeredis-single:
130135
image: redis:4-alpine
136+
profiles: ["group1"]
131137
hostname: stackexchangeredis-single
132138
command: redis-server --bind 0.0.0.0
133139
ports:
134140
- "127.0.0.1:6391:6379"
135141

136142
mongo:
137143
image: mongo:4.0.9
144+
profiles: ["group2"]
138145
ports:
139146
- "127.0.0.1:27017:27017"
140147
command: mongod
141148

142149
couchbase:
143150
image: bentonam/couchbase-docker:community-5.0.1
151+
profiles: ["group1"]
144152
ports:
145153
- "8091-8094:8091-8094"
146154
- "11210:11210"
147155

148156
elasticsearch7:
149157
image: docker.elastic.co/elasticsearch/elasticsearch:7.14.1
158+
profiles: ["group2"]
150159
ports:
151160
- "127.0.0.1:9210:9200"
152161
- "127.0.0.1:9310:9300"
@@ -156,6 +165,7 @@ services:
156165

157166
elasticsearch6:
158167
image: docker.elastic.co/elasticsearch/elasticsearch:6.4.2
168+
profiles: ["group2"]
159169
ports:
160170
- "127.0.0.1:9200:9200"
161171
- "127.0.0.1:9300:9300"
@@ -165,6 +175,7 @@ services:
165175

166176
elasticsearch5:
167177
image: docker.elastic.co/elasticsearch/elasticsearch:5.6.16
178+
profiles: ["group2"]
168179
ports:
169180
- "127.0.0.1:9205:9200"
170181
- "127.0.0.1:9305:9300"
@@ -174,6 +185,7 @@ services:
174185

175186
postgres:
176187
image: postgres:10.5-alpine
188+
profiles: ["group1"]
177189
environment:
178190
- POSTGRES_PASSWORD=postgres
179191
- POSTGRES_USER=postgres
@@ -183,6 +195,7 @@ services:
183195

184196
mysql:
185197
image: mysql/mysql-server:8.0
198+
profiles: ["group1"]
186199
environment:
187200
- MYSQL_DATABASE=world
188201
- MYSQL_ROOT_PASSWORD=mysqldb
@@ -193,6 +206,7 @@ services:
193206

194207
mysql57:
195208
image: mysql/mysql-server:5.7
209+
profiles: ["group1"]
196210
environment:
197211
- MYSQL_DATABASE=world
198212
- MYSQL_ROOT_PASSWORD=mysqldb
@@ -203,6 +217,7 @@ services:
203217

204218
sqlserver:
205219
image: mcr.microsoft.com/mssql/server:latest
220+
profiles: ["group1"]
206221
ports:
207222
- "127.0.0.1:1433:1433"
208223
environment:
@@ -211,14 +226,14 @@ services:
211226

212227
sqledge:
213228
image: mcr.microsoft.com/azure-sql-edge:latest
229+
profiles: ["group2"]
214230
environment:
215231
- ACCEPT_EULA=Y
216232
- MSSQL_SA_PASSWORD=Strong!Passw0rd
217233

218234
azureservicebus-emulator:
219235
image: mcr.microsoft.com/azure-messaging/servicebus-emulator:1.1.2
220-
hostname: azureservicebus-emulator
221-
container_name: azureservicebus-emulator
236+
profiles: ["group2"]
222237
depends_on:
223238
- sqledge
224239
environment:
@@ -230,6 +245,7 @@ services:
230245

231246
azurite:
232247
image: mcr.microsoft.com/azure-storage/azurite:latest
248+
profiles: ["group2"]
233249
hostname: azurite
234250
container_name: azurite
235251
ports:
@@ -239,8 +255,7 @@ services:
239255

240256
azure-eventhubs-emulator:
241257
image: mcr.microsoft.com/azure-messaging/eventhubs-emulator:latest
242-
hostname: azure-eventhubs-emulator
243-
container_name: azure-eventhubs-emulator
258+
profiles: ["group2"]
244259
depends_on:
245260
- azurite
246261
ports:
@@ -269,6 +284,7 @@ services:
269284
# For original definitions
270285
kafka-zookeeper:
271286
image: confluentinc/cp-zookeeper:6.1.1
287+
profiles: ["group1"]
272288
hostname: kafka-zookeeper
273289
container_name: kafka-zookeeper
274290
ports:
@@ -280,6 +296,7 @@ services:
280296

281297
kafka-broker:
282298
image: confluentinc/cp-server:6.1.1
299+
profiles: ["group1"]
283300
hostname: kafka-broker
284301
container_name: kafka-broker
285302
depends_on:
@@ -311,6 +328,7 @@ services:
311328

312329
kafka-schema-registry:
313330
image: confluentinc/cp-schema-registry:6.1.1
331+
profiles: ["group1"]
314332
hostname: kafka-schema-registry
315333
container_name: kafka-schema-registry
316334
depends_on:
@@ -325,6 +343,7 @@ services:
325343

326344
kafka-control-center:
327345
image: confluentinc/cp-enterprise-control-center:6.1.1
346+
profiles: ["group1"]
328347
hostname: kafka-control-center
329348
container_name: kafka-control-center
330349
depends_on:
@@ -344,6 +363,7 @@ services:
344363

345364
kafka-rest-proxy:
346365
image: confluentinc/cp-kafka-rest:6.1.1
366+
profiles: ["group1"]
347367
depends_on:
348368
- kafka-broker
349369
- kafka-schema-registry
@@ -501,29 +521,6 @@ services:
501521
- RANDOM_SEED
502522
- TEST_AGENT_HOST=test-agent
503523
hostname: integrationtests
504-
depends_on:
505-
- servicestackredis
506-
- stackexchangeredis
507-
- stackexchangeredis-replica
508-
- stackexchangeredis-single
509-
- elasticsearch7
510-
- elasticsearch6
511-
- elasticsearch5
512-
- sqlserver
513-
# - oracle disabled for now, image is too big for CI
514-
- mongo
515-
- postgres
516-
- mysql
517-
- mysql57
518-
- rabbitmq
519-
- kafka-broker
520-
- kafka-zookeeper
521-
- localstack
522-
- couchbase
523-
- azureservicebus-emulator
524-
- azurite
525-
- azure-eventhubs-emulator
526-
- test-agent
527524

528525
IntegrationTests.Debugger:
529526
build:
@@ -683,34 +680,44 @@ services:
683680
- DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
684681
- DD_LOGGER_DD_TAGS
685682

686-
StartDependencies:
683+
StartDependencies.Group1:
687684
image: andrewlock/wait-for-dependencies
685+
profiles: ["group1"]
688686
depends_on:
689687
- servicestackredis
690688
- stackexchangeredis
691689
- stackexchangeredis-replica
692690
- stackexchangeredis-single
693-
- elasticsearch7
694-
- elasticsearch6
695-
- elasticsearch5
696691
- sqlserver
697-
# - oracle disabled for now, image is too big for CI
698-
- mongo
699692
- postgres
700693
- mysql
701694
- mysql57
702695
- rabbitmq
703696
- kafka-broker
704697
- kafka-zookeeper
705-
- localstack
706698
- couchbase
699+
- test-agent
700+
environment:
701+
- TIMEOUT_LENGTH=120
702+
command: servicestackredis:6379 stackexchangeredis:6379 stackexchangeredis-replica:6379 stackexchangeredis-single:6379 sqlserver:1433 postgres:5432 mysql:3306 mysql57:3306 rabbitmq:5672 kafka-broker:9092 kafka-zookeeper:2181 couchbase:11210 test-agent:8126
703+
704+
StartDependencies.Group2:
705+
image: andrewlock/wait-for-dependencies
706+
profiles: ["group2"]
707+
depends_on:
708+
- elasticsearch7
709+
- elasticsearch6
710+
- elasticsearch5
711+
- mongo
712+
- localstack
713+
- sqledge
707714
- azureservicebus-emulator
708715
- azurite
709716
- azure-eventhubs-emulator
710717
- test-agent
711718
environment:
712719
- TIMEOUT_LENGTH=120
713-
command: servicestackredis:6379 stackexchangeredis:6379 stackexchangeredis-replica:6379 stackexchangeredis-single:6379 elasticsearch5:9200 elasticsearch6:9200 elasticsearch7:9200 sqlserver:1433 mongo:27017 postgres:5432 mysql:3306 mysql57:3306 rabbitmq:5672 kafka-broker:9092 kafka-zookeeper:2181 localstack:4566 couchbase:11210 azureservicebus-emulator:5672 azure-eventhubs-emulator:5672 test-agent:8126 # oracle:1521
720+
command: elasticsearch5:9200 elasticsearch6:9200 elasticsearch7:9200 mongo:27017 localstack:4566 sqledge:1433 azureservicebus-emulator:5672 azure-eventhubs-emulator:5672 test-agent:8126
714721

715722
IntegrationTests.ARM64:
716723
build:

‎tracer/build/_build/Build.VariableGenerations.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,11 @@ void GenerateIntegrationTestsLinuxMatrix(bool dockerTest)
365365
{
366366
if (dockerTest)
367367
{
368-
matrix.Add($"{baseImage}_{framework}", new { publishTargetFramework = framework, baseImage = baseImage, artifactSuffix = artifactSuffix });
368+
var dockerGroups = new[] { 1, 2 };
369+
foreach (var dockerGroup in dockerGroups)
370+
{
371+
matrix.Add($"{baseImage}_{framework}_group{dockerGroup}", new { publishTargetFramework = framework, baseImage = baseImage, artifactSuffix = artifactSuffix, dockerGroup = dockerGroup });
372+
}
369373
}
370374
else
371375
{

‎tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsDynamoDbTests.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
namespace Datadog.Trace.ClrProfiler.IntegrationTests.AWS
1818
{
1919
[Trait("RequiresDockerDependency", "true")]
20+
[Trait("DockerGroup", "2")]
2021
[UsesVerify]
2122
public class AwsDynamoDbTests : TracingIntegrationTest
2223
{

‎tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsEventBridgeTests.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
namespace Datadog.Trace.ClrProfiler.IntegrationTests.AWS
1818
{
1919
[Trait("RequiresDockerDependency", "true")]
20+
[Trait("DockerGroup", "2")]
2021
[UsesVerify]
2122
public class AwsEventBridgeTests : TracingIntegrationTest
2223
{

‎tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsKinesisTests.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
namespace Datadog.Trace.ClrProfiler.IntegrationTests.AWS
1818
{
1919
[Trait("RequiresDockerDependency", "true")]
20+
[Trait("DockerGroup", "2")]
2021
[UsesVerify]
2122
public class AwsKinesisTests : TracingIntegrationTest
2223
{

‎tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsS3Tests.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
namespace Datadog.Trace.ClrProfiler.IntegrationTests.AWS
1919
{
2020
[Trait("RequiresDockerDependency", "true")]
21+
[Trait("DockerGroup", "2")]
2122
[UsesVerify]
2223
public class AwsS3Tests : TracingIntegrationTest
2324
{

‎tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsSnsTests.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
namespace Datadog.Trace.ClrProfiler.IntegrationTests.AWS
1919
{
2020
[Trait("RequiresDockerDependency", "true")]
21+
[Trait("DockerGroup", "2")]
2122
[UsesVerify]
2223
public class AwsSnsTests : TracingIntegrationTest
2324
{

‎tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsSqsTests.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests.AWS
1919
{
2020
[Collection(nameof(AwsSqsTestsCollection))]
2121
[Trait("RequiresDockerDependency", "true")]
22+
[Trait("DockerGroup", "2")]
2223
[UsesVerify]
2324
public class AwsSqsTests : TracingIntegrationTest
2425
{

0 commit comments

Comments
 (0)