Skip to content

Commit 0bbd713

Browse files
ko3n1gyuanzhedong
ko3n1g
authored and
yuanzhedong
committed
ci: Improve no-fail-fast mechanism (#13370)
* ci: Improve no-fail-fast mechanism Signed-off-by: oliver könig <[email protected]> * f Signed-off-by: oliver könig <[email protected]> * maybe like this? Signed-off-by: oliver könig <[email protected]> * test Signed-off-by: oliver könig <[email protected]> * fix Signed-off-by: oliver könig <[email protected]> * fix Signed-off-by: oliver könig <[email protected]> * test Signed-off-by: oliver könig <[email protected]> * test Signed-off-by: oliver könig <[email protected]> * test Signed-off-by: oliver könig <[email protected]> * fix Signed-off-by: oliver könig <[email protected]> * jobs Signed-off-by: oliver könig <[email protected]> * revert debugging Signed-off-by: oliver könig <[email protected]> * test Signed-off-by: oliver könig <[email protected]> * fix Signed-off-by: oliver könig <[email protected]> --------- Signed-off-by: oliver könig <[email protected]> Signed-off-by: Yuanzhe Dong <[email protected]>
1 parent 0baddce commit 0bbd713

File tree

2 files changed

+127
-17
lines changed

2 files changed

+127
-17
lines changed

‎.github/scripts/components_to_run.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ def main(source_sha: str, target_sha: str):
7777
test_modules = list(set(test_modules))
7878

7979
if len(test_modules) == 0:
80-
test_modules = ["all"]
80+
test_modules = ["nemo2", "automodel", "export-deploy", "speech"]
8181

8282
with open("test_modules.json", "w", encoding="utf-8") as f:
8383
json.dump(test_modules, f)

‎.github/workflows/cicd-main.yml‎

Lines changed: 126 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,9 @@ jobs:
4242
runs-on: ubuntu-latest
4343
outputs:
4444
test_to_run: ${{ steps.test_to_run.outputs.main }}
45+
is_ci_workload: ${{ steps.is_ci_workload.outputs.main }}
46+
no_fail_fast: ${{ steps.no_fail_fast.outputs.main }}
4547
components_to_run: ${{ steps.components_to_run.outputs.main }}
46-
branch_name: ${{ steps.branch_name.outputs.main }}
4748
env:
4849
TESTS_TO_RUN: ${{ inputs.test_to_run }}
4950
EVENT_NAME: ${{ github.event_name }}
@@ -63,7 +64,7 @@ jobs:
6364
if [[ "$EVENT_NAME" == "pull_request" ]]; then
6465
python .github/scripts/components_to_run.py --source-sha ${{ github.event.pull_request.head.sha }} --target-sha ${{ github.event.pull_request.base.sha }}
6566
else
66-
echo '["all"]' | tee -a test_modules.json
67+
echo '["nemo2", "automodel", "export-deploy", "speech"]' | tee -a test_modules.json
6768
fi
6869
6970
components_to_run=$(cat test_modules.json)
@@ -98,21 +99,45 @@ jobs:
9899
parsed_string=$(echo "$TESTS_TO_RUN" | jq -c --raw-input 'split(",")')
99100
echo "main=${parsed_string}" | tee -a "$GITHUB_OUTPUT"
100101
101-
- name: Extract branch name
102+
- name: Check if this is a CI workload
102103
shell: bash
103-
run: echo "main=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" | tee -a "$GITHUB_OUTPUT"
104-
id: branch_name
104+
id: is_ci_workload
105+
run: |
106+
branch_name=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
107+
108+
if [[ "$branch_name" =~ ^bump-ci-container ]]; then
109+
is_ci_workload=true
110+
echo "main=true" | tee -a "$GITHUB_OUTPUT"
111+
else
112+
is_ci_workload=false
113+
fi
114+
115+
echo "main=$is_ci_workload" | tee -a "$GITHUB_OUTPUT"
116+
117+
- name: Check if no-fail-fast is set
118+
shell: bash
119+
id: no_fail_fast
120+
run: |
121+
if [[ "${{ contains(github.event.pull_request.labels.*.name, 'no-fail-fast') }}" == "true" ]]; then
122+
no_fail_fast=true
123+
else
124+
no_fail_fast=false
125+
fi
126+
127+
echo "main=$no_fail_fast" | tee -a "$GITHUB_OUTPUT"
105128
106129
code-linting:
107-
if: ${{ needs.pre-flight.outputs.test_to_run != '[]' }}
130+
if: needs.pre-flight.outputs.test_to_run != '[]'
108131
needs: [pre-flight]
109132
uses: ./.github/workflows/code-linting.yml
110133

111134
cicd-wait-in-queue:
112135
needs: [pre-flight]
113136
runs-on: ubuntu-latest
114137
environment: test
115-
if: ${{ needs.pre-flight.outputs.test_to_run != '[]' && !startsWith(needs.pre-flight.outputs.branch_name, 'bump-ci-container') }}
138+
if: |
139+
needs.pre-flight.outputs.test_to_run != '[]'
140+
&& needs.pre-flight.outputs.is_ci_workload == 'false'
116141
steps:
117142
- name: Running CI tests
118143
run: |
@@ -122,14 +147,30 @@ jobs:
122147
uses: ./.github/workflows/_build_container.yml
123148
needs: [pre-flight, code-linting, cicd-wait-in-queue]
124149
if: |
125-
needs.pre-flight.outputs.test_to_run != '[]' && (success() || needs.cicd-wait-in-queue.result == 'skipped')
150+
needs.pre-flight.outputs.test_to_run != '[]'
151+
&& (
152+
success()
153+
|| (
154+
needs.cicd-wait-in-queue.result == 'skipped'
155+
&& needs.pre-flight.outputs.is_ci_workload == 'true'
156+
)
157+
)
158+
&& !cancelled()
126159
with:
127160
image-name: nemo_container
128161
dockerfile: Dockerfile.ci
129162

130163
cicd-import-tests:
131164
if: |
132-
needs.pre-flight.outputs.test_to_run != '[]' && (success() || needs.cicd-wait-in-queue.result == 'skipped')
165+
needs.pre-flight.outputs.test_to_run != '[]'
166+
&& (
167+
success()
168+
|| (
169+
needs.cicd-wait-in-queue.result == 'skipped'
170+
&& needs.pre-flight.outputs.is_ci_workload == 'true'
171+
)
172+
)
173+
&& !cancelled()
133174
needs: [cicd-test-container-build, pre-flight]
134175
runs-on: self-hosted-azure-gpus-1
135176
steps:
@@ -167,10 +208,18 @@ jobs:
167208
'
168209
169210
L0_Setup_Test_Data_And_Models:
170-
needs: [pre-flight, cicd-test-container-build]
211+
needs: [pre-flight, cicd-test-container-build, cicd-wait-in-queue]
171212
runs-on: self-hosted-azure
172213
if: |
173-
(success() || needs.cicd-wait-in-queue.result == 'skipped')
214+
needs.pre-flight.outputs.test_to_run != '[]'
215+
&& (
216+
success()
217+
|| (
218+
needs.cicd-wait-in-queue.result == 'skipped'
219+
&& needs.pre-flight.outputs.is_ci_workload == 'true'
220+
)
221+
)
222+
&& !cancelled()
174223
steps:
175224
- name: Checkout
176225
uses: actions/checkout@v4
@@ -188,39 +237,100 @@ jobs:
188237
needs: [pre-flight, cicd-test-container-build]
189238
uses: ./.github/workflows/cicd-main-unit-tests.yml
190239
if: |
191-
(success() || needs.cicd-wait-in-queue.result == 'skipped')
240+
needs.pre-flight.outputs.test_to_run != '[]'
241+
&& (
242+
success()
243+
|| (
244+
needs.cicd-wait-in-queue.result == 'skipped'
245+
&& needs.pre-flight.outputs.is_ci_workload == 'true'
246+
)
247+
)
248+
&& !cancelled()
192249
with:
193250
test_to_run: ${{ needs.pre-flight.outputs.test_to_run }}
194251

195252
cicd-main-export-deploy:
196253
needs: [pre-flight, cicd-test-container-build, cicd-main-unit-tests]
197254
uses: ./.github/workflows/cicd-main-export-deploy.yml
198255
if: |
199-
(success() || needs.cicd-wait-in-queue.result == 'skipped') && (contains(fromJson(needs.pre-flight.outputs.components_to_run), 'export-deploy') || needs.pre-flight.outputs.components_to_run == '["all"]')
256+
(
257+
needs.pre-flight.outputs.test_to_run != '[]'
258+
&& (
259+
contains(fromJson(needs.pre-flight.outputs.components_to_run), 'export-deploy')
260+
)
261+
)
262+
&& (
263+
success()
264+
|| (
265+
needs.cicd-wait-in-queue.result == 'skipped'
266+
&& needs.pre-flight.outputs.is_ci_workload == 'true'
267+
)
268+
)
269+
&& !cancelled()
200270
with:
201271
test_to_run: ${{ needs.pre-flight.outputs.test_to_run }}
202272

203273
cicd-main-speech:
204274
needs: [pre-flight, cicd-test-container-build, cicd-main-unit-tests]
205275
uses: ./.github/workflows/cicd-main-speech.yml
206276
if: |
207-
(success() || needs.cicd-wait-in-queue.result == 'skipped') && (contains(fromJson(needs.pre-flight.outputs.components_to_run), 'speech') || needs.pre-flight.outputs.components_to_run == '["all"]')
277+
(
278+
needs.pre-flight.outputs.test_to_run != '[]'
279+
&& (
280+
contains(fromJson(needs.pre-flight.outputs.components_to_run), 'speech')
281+
)
282+
)
283+
&& (
284+
success()
285+
|| (
286+
needs.cicd-wait-in-queue.result == 'skipped'
287+
&& needs.pre-flight.outputs.is_ci_workload == 'true'
288+
)
289+
)
290+
&& !cancelled()
208291
with:
209292
test_to_run: ${{ needs.pre-flight.outputs.test_to_run }}
210293

211294
cicd-main-automodel:
212295
needs: [pre-flight, cicd-test-container-build, cicd-main-unit-tests]
213296
uses: ./.github/workflows/cicd-main-automodel.yml
214297
if: |
215-
(success() || needs.cicd-wait-in-queue.result == 'skipped') && ( contains(fromJson(needs.pre-flight.outputs.components_to_run), 'automodel') || needs.pre-flight.outputs.components_to_run == '["all"]')
298+
(
299+
needs.pre-flight.outputs.test_to_run != '[]'
300+
&& (
301+
contains(fromJson(needs.pre-flight.outputs.components_to_run), 'automodel')
302+
)
303+
)
304+
&& (
305+
success()
306+
|| (
307+
needs.cicd-wait-in-queue.result == 'skipped'
308+
&& needs.pre-flight.outputs.is_ci_workload == 'true'
309+
)
310+
)
311+
&& !cancelled()
216312
with:
217313
test_to_run: ${{ needs.pre-flight.outputs.test_to_run }}
218314

219315
cicd-main-nemo2:
220316
needs: [pre-flight, cicd-test-container-build, cicd-main-unit-tests]
221317
uses: ./.github/workflows/cicd-main-nemo2.yml
222318
if: |
223-
(success() || needs.cicd-wait-in-queue.result == 'skipped') && (contains(fromJson(needs.pre-flight.outputs.components_to_run), 'nemo2') || needs.pre-flight.outputs.components_to_run == '["all"]')
319+
(
320+
needs.pre-flight.outputs.test_to_run != '[]'
321+
&& (
322+
contains(fromJson(needs.pre-flight.outputs.components_to_run), 'nemo2')
323+
|| needs.pre-flight.outputs.components_to_run == '["all"]'
324+
)
325+
)
326+
&& (
327+
success()
328+
|| (
329+
needs.cicd-wait-in-queue.result == 'skipped'
330+
&& needs.pre-flight.outputs.is_ci_workload == 'true'
331+
)
332+
)
333+
&& !cancelled()
224334
with:
225335
test_to_run: ${{ needs.pre-flight.outputs.test_to_run }}
226336

0 commit comments

Comments
 (0)