Skip to content

V15 QA added parallelization for the integration tests #17713

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Zeegaan merged 9 commits into from
Dec 3, 2024
Merged

V15 QA added parallelization for the integration tests #17713

Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
dee7696
Added parallelization for our integration tests
andr317c Dec 2, 2024
882d9e8
Updated yaml
andr317c Dec 3, 2024
ff6a564
Moved filter command to variable definition
andr317c Dec 3, 2024
d4b49f9
Escape quotes
andr317c Dec 3, 2024
bb93661
Updated testFilter
andr317c Dec 3, 2024
05f37aa
Added testFilter for SQlite
andr317c Dec 3, 2024
82194be
Cleaned up filter
andr317c Dec 3, 2024
ece313c
Added comments
andr317c Dec 3, 2024
88f0b5b
Merge branch 'v15/dev' into v15/QA/parallize-integration-tests
andr317c Dec 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 64 additions & 13 deletions build/azure-pipelines.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,10 +297,31 @@ stages:
matrix:
# Windows:
# vmImage: 'windows-latest'
Linux:
# We split the tests into 3 parts for each OS to reduce the time it takes to run them on the pipeline
LinuxPart1Of3:
vmImage: "ubuntu-24.04"
macOS:
# Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace
testFilter: '(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)'
LinuxPart2Of3:
vmImage: "ubuntu-24.04"
# Filter tests that are part of the Umbraco.Infrastructure.Service namespace
testFilter: '(FullyQualifiedName~Umbraco.Infrastructure.Service)'
LinuxPart3Of3:
vmImage: "ubuntu-24.04"
# Filter tests that are not part of the Umbraco.Infrastructure namespace. So this will run all tests that are not part of the Umbraco.Infrastructure namespace
testFilter: '(FullyQualifiedName!~Umbraco.Infrastructure)'
macOSPart1Of3:
vmImage: "macOS-latest"
# Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace
testFilter: '(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)'
macOSPart2Of3:
vmImage: "macOS-latest"
# Filter tests that are part of the Umbraco.Infrastructure.Service namespace
testFilter: '(FullyQualifiedName~Umbraco.Infrastructure.Service)'
macOSPart3Of3:
vmImage: "macOS-latest"
# Filter tests that are not part of the Umbraco.Infrastructure namespace.
testFilter: '(FullyQualifiedName!~Umbraco.Infrastructure)'
pool:
vmImage: $(vmImage)
variables:
Expand Down Expand Up @@ -331,14 +352,13 @@ stages:
projects: "tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj"
testRunTitle: Integration Tests SQLite - $(Agent.OS)
${{ if and(eq(variables['Agent.OS'],'Windows_NT'), or(variables.releaseTestFilter, parameters.forceReleaseTestFilter)) }}:
arguments: "--configuration $(buildConfiguration) --no-build ${{parameters.integrationReleaseTestFilter}}"
arguments: '--filter "$(testFilter)" --configuration $(buildConfiguration) --no-build ${{parameters.integrationReleaseTestFilter}}'
${{ elseif eq(variables['Agent.OS'],'Windows_NT') }}:
arguments: "--configuration $(buildConfiguration) --no-build ${{parameters.integrationNonReleaseTestFilter}}"
arguments: '--filter "$(testFilter)" --configuration $(buildConfiguration) --no-build ${{parameters.integrationNonReleaseTestFilter}}'
${{ elseif or(variables.releaseTestFilter, parameters.forceReleaseTestFilter) }}:
arguments: "--configuration $(buildConfiguration) --no-build ${{parameters.nonWindowsIntegrationReleaseTestFilter}}"
arguments: '--filter "$(testFilter)" --configuration $(buildConfiguration) --no-build ${{parameters.nonWindowsIntegrationReleaseTestFilter}}'
${{ else }}:
arguments: "--configuration $(buildConfiguration) --no-build ${{parameters.nonWindowsIntegrationNonReleaseTestFilter}}"

arguments: '--filter "$(testFilter)" --configuration $(buildConfiguration) --no-build ${{parameters.nonWindowsIntegrationNonReleaseTestFilter}}'
# Integration Tests (SQL Server)
- job:
timeoutInMinutes: 180
Expand All @@ -347,15 +367,46 @@ stages:
displayName: Integration Tests (SQL Server)
strategy:
matrix:
Windows:
# We split the tests into 3 parts for each OS to reduce the time it takes to run them on the pipeline
WindowsPart1Of3:
vmImage: "windows-latest"
Tests__Database__DatabaseType: LocalDb
Tests__Database__SQLServerMasterConnectionString: N/A
Linux:
# Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace
testFilter: '(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)'
WindowsPart2Of3:
vmImage: "windows-latest"
Tests__Database__DatabaseType: LocalDb
Tests__Database__SQLServerMasterConnectionString: N/A
# Filter tests that are part of the Umbraco.Infrastructure.Service namespace
testFilter: '(FullyQualifiedName~Umbraco.Infrastructure.Service)'
WindowsPart3Of3:
vmImage: "windows-latest"
Tests__Database__DatabaseType: LocalDb
Tests__Database__SQLServerMasterConnectionString: N/A
# Filter tests that are not part of the Umbraco.Infrastructure namespace. So this will run all tests that are not part of the Umbraco.Infrastructure namespace
testFilter: '(FullyQualifiedName!~Umbraco.Infrastructure)'
LinuxPart1Of3:
vmImage: "ubuntu-latest"
SA_PASSWORD: UmbracoIntegration123!
Tests__Database__DatabaseType: SqlServer
Tests__Database__SQLServerMasterConnectionString: "Server=(local);User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
# Filter tests that are part of the Umbraco.Infrastructure namespace but not part of the Umbraco.Infrastructure.Service namespace
testFilter: '(FullyQualifiedName~Umbraco.Infrastructure) & (FullyQualifiedName!~Umbraco.Infrastructure.Service)'
LinuxPart2Of3:
vmImage: "ubuntu-latest"
SA_PASSWORD: UmbracoIntegration123!
Tests__Database__DatabaseType: SqlServer
Tests__Database__SQLServerMasterConnectionString: "Server=(local);User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
# Filter tests that are part of the Umbraco.Infrastructure.Service namespace
testFilter: '(FullyQualifiedName~Umbraco.Infrastructure.Service)'
LinuxPart3Of3:
vmImage: "ubuntu-latest"
SA_PASSWORD: UmbracoIntegration123!
Tests__Database__DatabaseType: SqlServer
Tests__Database__SQLServerMasterConnectionString: "Server=(local);User Id=sa;Password=$(SA_PASSWORD);TrustServerCertificate=True"
# Filter tests that are not part of the Umbraco.Infrastructure namespace. So this will run all tests that are not part of the Umbraco.Infrastructure namespace
testFilter: '(FullyQualifiedName!~Umbraco.Infrastructure)'
pool:
vmImage: $(vmImage)
steps:
Expand Down Expand Up @@ -388,13 +439,13 @@ stages:
projects: "tests/Umbraco.Tests.Integration/Umbraco.Tests.Integration.csproj"
testRunTitle: Integration Tests SQL Server - $(Agent.OS)
${{ if and(eq(variables['Agent.OS'],'Windows_NT'), or(variables.releaseTestFilter, parameters.forceReleaseTestFilter)) }}:
arguments: "--configuration $(buildConfiguration) --no-build ${{parameters.integrationReleaseTestFilter}}"
arguments: '--filter "$(testFilter)" --configuration $(buildConfiguration) --no-build ${{parameters.integrationReleaseTestFilter}}'
${{ elseif eq(variables['Agent.OS'],'Windows_NT') }}:
arguments: "--configuration $(buildConfiguration) --no-build ${{parameters.integrationNonReleaseTestFilter}}"
arguments: '--filter "$(testFilter)" --configuration $(buildConfiguration) --no-build ${{parameters.integrationNonReleaseTestFilter}}'
${{ elseif or(variables.releaseTestFilter, parameters.forceReleaseTestFilter) }}:
arguments: "--configuration $(buildConfiguration) --no-build ${{parameters.nonWindowsIntegrationReleaseTestFilter}}"
arguments: '--filter "$(testFilter)" --configuration $(buildConfiguration) --no-build ${{parameters.nonWindowsIntegrationReleaseTestFilter}}'
${{ else }}:
arguments: "--configuration $(buildConfiguration) --no-build ${{parameters.nonWindowsIntegrationNonReleaseTestFilter}}"
arguments: '--filter "$(testFilter)" --configuration $(buildConfiguration) --no-build ${{parameters.nonWindowsIntegrationNonReleaseTestFilter}}'

# Stop SQL Server
- pwsh: docker stop mssql
Expand Down