Skip to content

Commit 18afb17

Browse files
kavillakavilla
committed
update workflow to run faster
Signed-off-by: Kawika Avilla <[email protected]> skip test per suggestion Signed-off-by: Kawika Avilla <[email protected]> some optimizations Signed-off-by: Kawika Avilla <[email protected]> restore chromium Signed-off-by: Kawika Avilla <[email protected]> fix regex Signed-off-by: Kawika Avilla <[email protected]> set limits for heap Signed-off-by: Kawika Avilla <[email protected]> bump types and cypress Signed-off-by: Kawika Avilla <[email protected]> bump semi space size Signed-off-by: Kawika Avilla <[email protected]> revert node types bump Signed-off-by: Kawika Avilla <[email protected]> update lock file and cypress workflow Signed-off-by: Kawika Avilla <[email protected]> wait until visible Signed-off-by: Kawika Avilla <[email protected]> split it out for 15 as well Signed-off-by: Kawika Avilla <[email protected]> add a wait Signed-off-by: Kawika Avilla <[email protected]> escape and make wait a command Signed-off-by: Kawika Avilla <[email protected]> set max semi space as well Signed-off-by: Kawika Avilla <[email protected]> skip problematic tests Signed-off-by: Kawika Avilla <[email protected]> skip recents Signed-off-by: Kawika Avilla <[email protected]>
1 parent 2e545ce commit 18afb17

File tree

26 files changed

+222
-195
lines changed

26 files changed

+222
-195
lines changed

‎.github/workflows/build_and_test_workflow.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ env:
3131
TEST_OPENSEARCH_TRANSPORT_PORT: 9403
3232
TEST_OPENSEARCH_PORT: 9400
3333
OSD_SNAPSHOT_SKIP_VERIFY_CHECKSUM: true
34-
NODE_OPTIONS: '--max-old-space-size=6144 --dns-result-order=ipv4first'
34+
OSD_OPTIMIZER_MAX_WORKERS: 8
35+
NODE_OPTIONS: '--max-old-space-size=8192 --max-semi-space-size=64 --dns-result-order=ipv4first'
3536

3637
jobs:
3738
build-test:
@@ -47,6 +48,9 @@ jobs:
4748
- os: windows-latest
4849
name: Windows
4950
runs-on: ${{ matrix.os }}
51+
env:
52+
# Override just OSD_OPTIMIZER_MAX_WORKERS for ciGroup1 because it contains a test that checks number of workers
53+
OSD_OPTIMIZER_MAX_WORKERS: ${{ matrix.group != 1 && 8 || '' }}
5054
steps:
5155
- name: Configure git's autocrlf (Windows only)
5256
if: matrix.os == 'windows-latest'

‎.github/workflows/build_with_plugins_workflow.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Build Release & Plugins
1+
name: Build and install plugins
22

33
on:
44
push:
@@ -20,7 +20,7 @@ jobs:
2020
runs-on: ubuntu-latest
2121
steps:
2222
- name: Checkout OpenSearch Dashboards
23-
uses: actions/checkout@v3
23+
uses: actions/checkout@v4
2424

2525
- name: Setup Node
2626
uses: actions/setup-node@v4
@@ -29,7 +29,7 @@ jobs:
2929
registry-url: 'https://registry.npmjs.org'
3030

3131
- name: Install Yarn
32-
run: npm install -g [email protected]
32+
run: npm install -g yarn@^1.22.10
3333

3434
- name: Bootstrap OSD Core
3535
run: yarn osd bootstrap --single-version=loose
@@ -98,7 +98,7 @@ jobs:
9898
ref: main
9999
steps:
100100
- name: Checkout OpenSearch Dashboards
101-
uses: actions/checkout@v3
101+
uses: actions/checkout@v4
102102

103103
- name: Setup Node
104104
uses: actions/setup-node@v4
@@ -107,10 +107,10 @@ jobs:
107107
registry-url: 'https://registry.npmjs.org'
108108

109109
- name: Install Yarn
110-
run: npm install -g [email protected]
110+
run: npm install -g yarn@^1.22.10
111111

112112
- name: Checkout Plugin
113-
uses: actions/checkout@v3
113+
uses: actions/checkout@v4
114114
with:
115115
repository: ${{ matrix.plugin.repo }}
116116
ref: ${{ matrix.plugin.ref }}

‎.github/workflows/create_doc_issue.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ jobs:
2222
installation_id: 22958780
2323

2424
- name: Checkout code
25-
uses: actions/checkout@v2
25+
uses: actions/checkout@v4
2626

2727
- name: Edit the issue template
2828
run: |

‎.github/workflows/cypress_workflow.yml‎

Lines changed: 62 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ on:
66
branches: ['**']
77
paths-ignore:
88
- '**/*.md'
9+
- 'docs/**'
910
- '.lycheeignore'
11+
- 'CODEOWNERS'
1012
- 'changelogs/fragments/**'
1113
workflow_dispatch:
1214
inputs:
@@ -35,8 +37,18 @@ env:
3537
CYPRESS_BROWSER: 'chromium'
3638
CYPRESS_VISBUILDER_ENABLED: true
3739
CYPRESS_DATASOURCE_MANAGEMENT_ENABLED: false
40+
CYPRESS_MEMORY_LIMIT: '10240'
41+
CYPRESS_CACHE_FOLDER: '${{ github.workspace }}/.cypress-cache'
42+
# Optimize Chrome
43+
CYPRESS_CHROME_ARGS: >-
44+
--disable-dev-shm-usage
45+
--no-sandbox
46+
--disable-gpu
47+
--disable-software-rasterizer
48+
--js-flags="--max-old-space-size=10240"
3849
OSD_SNAPSHOT_SKIP_VERIFY_CHECKSUM: true
39-
NODE_OPTIONS: '--max-old-space-size=6144 --dns-result-order=ipv4first'
50+
OSD_OPTIMIZER_MAX_WORKERS: 8
51+
NODE_OPTIONS: '--max-old-space-size=10240 --max-semi-space-size=64 --dns-result-order=ipv4first'
4052
COMMENT_TAG: '[MANUAL CYPRESS TEST RUN RESULTS]'
4153
COMMENT_SUCCESS_MSG: ':white_check_mark: Cypress test run succeeded!'
4254
COMMENT_FAILURE_MSG: ':x: Cypress test run failed!'
@@ -78,7 +90,11 @@ jobs:
7890
config: standard
7991
test_location: ftr
8092
# Dashboard tests with query enhanced - group 1
81-
- group: 10
93+
- group: 10Fast
94+
config: query_enhanced
95+
test_location: source
96+
# Dashboard tests with query enhanced - group 1 but slow tests
97+
- group: 10Slow
8298
config: query_enhanced
8399
test_location: source
84100
# Dashboard tests with no query enhanced
@@ -98,12 +114,15 @@ jobs:
98114
config: query_enhanced
99115
test_location: source
100116
# Dashboard tests with query enhanced - group 5
101-
- group: 15
117+
- group: 15Fast
118+
config: query_enhanced
119+
test_location: source
120+
- group: 15Slow
102121
config: query_enhanced
103122
test_location: source
104123
container:
105124
image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
106-
options: --user 1001
125+
options: '--user 1001 --shm-size=2g'
107126
env:
108127
START_CMD: ${{ matrix.config == 'query_enhanced' &&
109128
'node scripts/opensearch_dashboards --dev --no-base-path --no-watch --savedObjects.maxImportPayloadBytes=10485760 --server.maxPayloadBytes=1759977 --logging.json=false --data.search.aggs.shardDelay.enabled=true --csp.warnLegacyBrowsers=false --uiSettings.overrides["query:enhancements:enabled"]=true --uiSettings.overrides[''home:useNewHomePage'']=true --data_source.enabled=true --workspace.enabled=true --opensearch.ignoreVersionMismatch=true --data.savedQueriesNewUI.enabled=true' ||
@@ -142,21 +161,21 @@ jobs:
142161
echo "SOURCE_BRANCH=${{ steps.get_pr_info.outputs.head_ref }}" >> $GITHUB_ENV
143162
144163
- name: Checkout code
145-
uses: actions/checkout@v2
164+
uses: actions/checkout@v4
146165
with:
147166
repository: ${{ env.SOURCE_REPO }}
148167
ref: '${{ env.SOURCE_BRANCH }}'
149168

150169
- name: Setup Node
151-
uses: actions/setup-node@v2
170+
uses: actions/setup-node@v4
152171
with:
153172
node-version-file: '.nvmrc'
154173
registry-url: 'https://registry.npmjs.org'
155174

156175
- name: Setup Yarn
157176
run: |
158177
npm uninstall -g yarn
159-
npm i -g [email protected]
178+
npm i -g yarn@^1.22.10
160179
161180
- name: Run bootstrap
162181
run: yarn osd bootstrap
@@ -165,11 +184,12 @@ jobs:
165184
run: node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers 12
166185

167186
- name: Checkout FT repo
168-
uses: actions/checkout@v2
187+
uses: actions/checkout@v4
169188
with:
170189
path: ${{ env.FTR_PATH }}
171190
repository: ${{ env.TEST_REPO }}
172191
ref: '${{ env.TEST_BRANCH }}'
192+
clean: true
173193

174194
- name: Setup spec files by input
175195
if: ${{ inputs.specs != '' }}
@@ -218,7 +238,7 @@ jobs:
218238
# Run tests based on configuration
219239
- name: Run FT repo tests
220240
if: matrix.test_location == 'ftr'
221-
uses: cypress-io/github-action@v2
241+
uses: cypress-io/github-action@v6
222242
with:
223243
working-directory: ${{ env.FTR_PATH }}
224244
start: ${{ env.OPENSEARCH_SNAPSHOT_CMD }}, ${{ env.START_CMD }}
@@ -247,7 +267,7 @@ jobs:
247267
- name: Run OpenSearch
248268
if: matrix.config == 'query_enhanced'
249269
run: |
250-
/bin/bash -c "./opensearch-2.17.0/opensearch-tar-install.sh &"
270+
/bin/bash -c "./opensearch-${{ env.LATEST_VERSION }}/opensearch-tar-install.sh &"
251271
sleep 30
252272
shell: bash
253273

@@ -265,6 +285,7 @@ jobs:
265285
start: ${{ env.START_CMD }}
266286
wait-on: 'http://localhost:9200, http://localhost:5601'
267287
command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.DASHBOARDS_SPEC }}
288+
268289
- name: Run Dashboards Cypress tests without query enhancements
269290
if: matrix.test_location == 'source' && matrix.config == 'dashboard'
270291
uses: cypress-io/github-action@v6
@@ -274,58 +295,46 @@ jobs:
274295
wait-on: 'http://localhost:9200, http://localhost:5601'
275296
command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.DASHBOARDS_SPEC }}
276297

277-
# Screenshots are only captured on failure, will change this once we do visual regression tests
278-
- name: Upload FT repo screenshots
279-
uses: actions/upload-artifact@v4
280-
if: failure() && matrix.test_location == 'ftr'
281-
with:
282-
name: ftr-cypress-screenshots-${{ matrix.group }}
283-
path: ${{ env.FTR_PATH }}/cypress/screenshots
284-
retention-days: 1
285-
overwrite: true
286-
287-
- name: Upload FT repo videos
288-
uses: actions/upload-artifact@v4
289-
if: failure() && matrix.test_location == 'ftr'
290-
with:
291-
name: ftr-cypress-videos-${{ matrix.group }}
292-
path: ${{ env.FTR_PATH }}/cypress/videos
293-
retention-days: 1
294-
overwrite: true
295-
296-
- name: Upload FT repo results
297-
uses: actions/upload-artifact@v4
298+
# Compress and upload artifacts for FT repo
299+
- name: Compress FT repo artifacts
298300
if: always() && matrix.test_location == 'ftr'
299-
with:
300-
name: ftr-cypress-results-${{ matrix.group }}
301-
path: ${{ env.FTR_PATH }}/cypress/results
302-
retention-days: 1
303-
overwrite: true
301+
run: |
302+
mkdir -p ftr-artifacts
303+
for dir in screenshots videos results; do
304+
if [ -d "${{ env.FTR_PATH }}/cypress/$dir" ]; then
305+
cp -r "${{ env.FTR_PATH }}/cypress/$dir" ftr-artifacts/
306+
fi
307+
done
308+
tar -czf ftr-cypress-artifacts-${{ matrix.group }}.tar.gz -C ftr-artifacts .
309+
shell: bash
304310

305-
- name: Upload Dashboards screenshots
306-
if: failure() && matrix.test_location == 'source'
311+
- name: Upload FT repo artifacts
312+
if: always() && matrix.test_location == 'ftr'
307313
uses: actions/upload-artifact@v4
308314
with:
309-
name: dashboards-cypress-screenshots-${{ matrix.group }}
310-
path: cypress/screenshots
315+
name: ftr-cypress-artifacts-${{ matrix.group }}
316+
path: ftr-cypress-artifacts-${{ matrix.group }}.tar.gz
311317
retention-days: 1
312-
overwrite: true
313318

314-
- name: Upload Dashboards repo videos
315-
uses: actions/upload-artifact@v4
316-
if: failure() && matrix.test_location == 'source'
317-
with:
318-
name: dashboards-cypress-videos-${{ matrix.group }}
319-
path: cypress/videos
320-
retention-days: 1
321-
overwrite: true
319+
# Compress and upload artifacts for Dashboards repo
320+
- name: Compress Dashboards repo artifacts
321+
if: always() && matrix.test_location == 'source'
322+
run: |
323+
mkdir -p dashboard-artifacts
324+
for dir in screenshots videos results; do
325+
if [ -d "cypress/$dir" ]; then
326+
cp -r "cypress/$dir" dashboard-artifacts/
327+
fi
328+
done
329+
tar -czf dashboards-cypress-artifacts-${{ matrix.group }}.tar.gz -C dashboard-artifacts .
330+
shell: bash
322331

323-
- name: Upload Dashboards repo results
324-
uses: actions/upload-artifact@v4
332+
- name: Upload Dashboards repo artifacts
325333
if: always() && matrix.test_location == 'source'
334+
uses: actions/upload-artifact@v4
326335
with:
327-
name: dashboards-cypress-results-${{ matrix.group }}
328-
path: cypress/results
336+
name: dashboards-cypress-artifacts-${{ matrix.group }}
337+
path: dashboards-cypress-artifacts-${{ matrix.group }}.tar.gz
329338
retention-days: 1
330339
overwrite: true
331340

‎.github/workflows/cypress_workflow_with_s3.yml‎

Lines changed: 23 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ on:
66
branches: ['main', '[0-9]+\.x', '[0-9]+\.[0-9]+']
77
paths-ignore:
88
- '**/*.md'
9+
- 'docs/**'
910
- '.lycheeignore'
11+
- 'CODEOWNERS'
1012
- 'changelogs/fragments/**'
1113
workflow_dispatch:
1214
inputs:
@@ -45,18 +47,18 @@ jobs:
4547
name: Run cypress tests with S3
4648
steps:
4749
- name: Checkout code
48-
uses: actions/checkout@v2
50+
uses: actions/checkout@v4
4951

5052
- name: Setup Node
51-
uses: actions/setup-node@v2
53+
uses: actions/setup-node@v4
5254
with:
5355
node-version-file: '.nvmrc'
5456
registry-url: 'https://registry.npmjs.org'
5557

5658
- name: Setup Yarn
5759
run: |
5860
npm uninstall -g yarn
59-
npm i -g [email protected]
61+
npm i -g yarn@^1.22.10
6062
6163
- name: Run bootstrap
6264
run: yarn osd bootstrap
@@ -90,7 +92,7 @@ jobs:
9092

9193
- name: Run OpenSearch
9294
run: |
93-
/bin/bash -c "./opensearch-2.17.0/opensearch-tar-install.sh &"
95+
/bin/bash -c "./opensearch-${{ env.LATEST_VERSION }}/opensearch-tar-install.sh &"
9496
sleep 30
9597
shell: bash
9698

@@ -108,30 +110,25 @@ jobs:
108110
start: ${{ env.START_CMD }}
109111
wait-on: 'http://localhost:9200, http://localhost:5601'
110112
command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.DASHBOARDS_SPEC }}
113+
# Compress and upload artifacts for Dashboards repo
114+
115+
- name: Compress Dashboards repo artifacts
116+
if: always()
117+
run: |
118+
mkdir -p dashboard-artifacts-s3
119+
for dir in screenshots videos results; do
120+
if [ -d "cypress/$dir" ]; then
121+
cp -r "cypress/$dir" dashboard-artifacts-s3/
122+
fi
123+
done
124+
tar -czf dashboards-cypress-artifacts-s3.tar.gz -C dashboard-artifacts-s3 .
125+
shell: bash
111126

112-
- name: Upload Dashboards screenshots
113-
if: failure()
114-
uses: actions/upload-artifact@v4
115-
with:
116-
name: dashboards-cypress-screenshots-s3
117-
path: cypress/screenshots
118-
retention-days: 1
119-
overwrite: true
120-
121-
- name: Upload Dashboards repo videos
122-
uses: actions/upload-artifact@v4
123-
if: failure()
124-
with:
125-
name: dashboards-cypress-videos-s3
126-
path: cypress/videos
127-
retention-days: 1
128-
overwrite: true
129-
130-
- name: Upload Dashboards repo results
131-
uses: actions/upload-artifact@v4
127+
- name: Upload Dashboards repo artifacts
132128
if: always()
129+
uses: actions/upload-artifact@v4
133130
with:
134-
name: dashboards-cypress-results-s3
135-
path: cypress/results
131+
name: dashboards-cypress-artifacts-s3
132+
path: dashboards-cypress-artifacts-s3.tar.gz
136133
retention-days: 1
137134
overwrite: true

0 commit comments

Comments
 (0)