Skip to content

Commit b8825b3

Browse files
camchenrycamchenry
committed
ci: add profiling diff workflow
1 parent 36f5ab9 commit b8825b3

File tree

1 file changed

+267
-0
lines changed

1 file changed

+267
-0
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 267 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,267 @@
1+
name: Benchmark PR
2+
3+
permissions:
4+
pull-requests: write
5+
contents: read
6+
7+
on:
8+
pull_request:
9+
types: [labeled]
10+
11+
jobs:
12+
benchmark:
13+
if: github.event.label.name == 'bench'
14+
runs-on: ubuntu-latest
15+
timeout-minutes: 60
16+
steps:
17+
- name: Checkout PR
18+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
19+
with:
20+
fetch-depth: 0
21+
submodules: true
22+
persist-credentials: false
23+
ref: ${{ github.event.pull_request.head.sha }}
24+
25+
- uses: oxc-project/setup-node@fdbf0dfd334c4e6d56ceeb77d91c76339c2a0885 # v1.0.4
26+
27+
- uses: ./.github/actions/setup
28+
29+
- name: Install jq for payload generation
30+
run: sudo apt-get update && sudo apt-get install -y jq
31+
32+
- name: Clone vscode for benchmarking
33+
run: |
34+
mkdir -p benchmarks
35+
cd benchmarks
36+
# Clone vscode at pinned commit (v1.99.0 release)
37+
git clone --depth 1 --single-branch --branch 1.99.0 https://github.com/microsoft/vscode
38+
39+
- name: Setup vscode for benchmarking
40+
run: |
41+
cd benchmarks/vscode
42+
# Install minimal dependencies needed for TypeScript compilation
43+
npm install --ignore-scripts
44+
45+
- name: Build PR binary
46+
run: just build
47+
48+
- name: Generate headless payload for vscode
49+
run: |
50+
mkdir -p benchmark-results/pr
51+
# Generate payload with all TypeScript files in vscode/src
52+
find benchmarks/vscode/src -type f \( -name "*.ts" -o -name "*.tsx" \) | \
53+
jq -R -s '
54+
split("\n")[:-1] |
55+
{
56+
version: 2,
57+
configs: [
58+
{
59+
file_paths: .,
60+
rules: [
61+
{ name: "await-thenable" },
62+
{ name: "no-array-delete" },
63+
{ name: "no-base-to-string" },
64+
{ name: "no-confusing-void-expression" },
65+
{ name: "no-duplicate-type-constituents" },
66+
{ name: "no-floating-promises" },
67+
{ name: "no-for-in-array" },
68+
{ name: "no-implied-eval" },
69+
{ name: "no-meaningless-void-operator" },
70+
{ name: "no-misused-promises" },
71+
{ name: "no-unnecessary-type-assertion" },
72+
{ name: "no-unsafe-argument" },
73+
{ name: "no-unsafe-assignment" },
74+
{ name: "no-unsafe-call" },
75+
{ name: "no-unsafe-member-access" },
76+
{ name: "no-unsafe-return" }
77+
]
78+
}
79+
]
80+
}
81+
' > benchmark-results/pr/payload.json
82+
echo "Generated payload with $(jq '.configs[0].file_paths | length' benchmark-results/pr/payload.json) files"
83+
84+
- name: Run PR benchmark with CPU profiling
85+
run: |
86+
./tsgolint headless --cpuprof benchmark-results/pr/cpu.prof < benchmark-results/pr/payload.json
87+
88+
- name: Generate PR profile summary
89+
run: |
90+
go tool pprof -top benchmark-results/pr/cpu.prof > benchmark-results/pr/profile.txt
91+
92+
- name: Checkout base branch
93+
run: |
94+
git fetch origin ${{ github.event.pull_request.base.ref }}
95+
git checkout origin/${{ github.event.pull_request.base.ref }}
96+
git submodule update --init --recursive
97+
98+
- name: Apply typescript-go patches for base
99+
run: |
100+
pushd typescript-go
101+
git am --3way --no-gpg-sign ../patches/*.patch
102+
popd
103+
104+
- name: Expose typescript-go collections package for base
105+
run: |
106+
mkdir -p internal/collections
107+
find ./typescript-go/internal/collections -type f ! -name '*_test.go' -exec cp {} internal/collections/ \;
108+
109+
- name: Build base binary
110+
run: just build
111+
112+
- name: Generate headless payload for vscode (base)
113+
run: |
114+
mkdir -p benchmark-results/base
115+
# Generate payload with all TypeScript files in vscode/src
116+
find benchmarks/vscode/src -type f \( -name "*.ts" -o -name "*.tsx" \) | \
117+
jq -R -s '
118+
split("\n")[:-1] |
119+
{
120+
version: 2,
121+
configs: [
122+
{
123+
file_paths: .,
124+
rules: [
125+
{ name: "await-thenable" },
126+
{ name: "no-array-delete" },
127+
{ name: "no-base-to-string" },
128+
{ name: "no-confusing-void-expression" },
129+
{ name: "no-duplicate-type-constituents" },
130+
{ name: "no-floating-promises" },
131+
{ name: "no-for-in-array" },
132+
{ name: "no-implied-eval" },
133+
{ name: "no-meaningless-void-operator" },
134+
{ name: "no-misused-promises" },
135+
{ name: "no-unnecessary-type-assertion" },
136+
{ name: "no-unsafe-argument" },
137+
{ name: "no-unsafe-assignment" },
138+
{ name: "no-unsafe-call" },
139+
{ name: "no-unsafe-member-access" },
140+
{ name: "no-unsafe-return" }
141+
]
142+
}
143+
]
144+
}
145+
' > benchmark-results/base/payload.json
146+
echo "Generated payload with $(jq '.configs[0].file_paths | length' benchmark-results/base/payload.json) files"
147+
148+
- name: Run base benchmark with CPU profiling
149+
run: |
150+
./tsgolint headless --cpuprof benchmark-results/base/cpu.prof < benchmark-results/base/payload.json
151+
152+
- name: Generate base profile summary
153+
run: |
154+
go tool pprof -top benchmark-results/base/cpu.prof > benchmark-results/base/profile.txt
155+
156+
- name: Compare profiles
157+
id: compare
158+
run: |
159+
echo "## Benchmark Results" > benchmark-results/comment.md
160+
echo "" >> benchmark-results/comment.md
161+
echo "**Benchmark Mode:** Headless ($(jq '.configs[0].file_paths | length' benchmark-results/pr/payload.json) files, $(jq '.configs[0].rules | length' benchmark-results/pr/payload.json) rules)" >> benchmark-results/comment.md
162+
echo "" >> benchmark-results/comment.md
163+
echo "### CPU Profile Comparison" >> benchmark-results/comment.md
164+
echo "" >> benchmark-results/comment.md
165+
echo "#### Base Branch (${{ github.event.pull_request.base.ref }})" >> benchmark-results/comment.md
166+
echo '```' >> benchmark-results/comment.md
167+
head -n 20 benchmark-results/base/profile.txt >> benchmark-results/comment.md
168+
echo '```' >> benchmark-results/comment.md
169+
echo "" >> benchmark-results/comment.md
170+
echo "#### PR Branch (${{ github.event.pull_request.head.ref }})" >> benchmark-results/comment.md
171+
echo '```' >> benchmark-results/comment.md
172+
head -n 20 benchmark-results/pr/profile.txt >> benchmark-results/comment.md
173+
echo '```' >> benchmark-results/comment.md
174+
echo "" >> benchmark-results/comment.md
175+
echo "### Differential Profile" >> benchmark-results/comment.md
176+
echo "" >> benchmark-results/comment.md
177+
echo "Comparing PR against base (positive values = PR is slower):" >> benchmark-results/comment.md
178+
echo '```' >> benchmark-results/comment.md
179+
go tool pprof -top -base benchmark-results/base/cpu.prof benchmark-results/pr/cpu.prof | head -n 25 >> benchmark-results/comment.md || echo "No significant differences found" >> benchmark-results/comment.md
180+
echo '```' >> benchmark-results/comment.md
181+
echo "" >> benchmark-results/comment.md
182+
echo "<details>" >> benchmark-results/comment.md
183+
echo "<summary>View CPU Profiles</summary>" >> benchmark-results/comment.md
184+
echo "" >> benchmark-results/comment.md
185+
echo "**Note:** Download the benchmark artifacts to analyze CPU profiles locally with pprof." >> benchmark-results/comment.md
186+
echo "" >> benchmark-results/comment.md
187+
echo '```bash' >> benchmark-results/comment.md
188+
echo "# Interactive analysis" >> benchmark-results/comment.md
189+
echo "go tool pprof -http=:8080 -base base/cpu.prof pr/cpu.prof" >> benchmark-results/comment.md
190+
echo '```' >> benchmark-results/comment.md
191+
echo "" >> benchmark-results/comment.md
192+
echo "</details>" >> benchmark-results/comment.md
193+
194+
- name: Upload benchmark artifacts
195+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.0
196+
with:
197+
name: benchmark-profiles
198+
path: benchmark-results/
199+
retention-days: 30
200+
201+
- name: Post benchmark results
202+
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
203+
with:
204+
script: |
205+
const fs = require('fs');
206+
const comment = fs.readFileSync('benchmark-results/comment.md', 'utf8');
207+
208+
const { data: comments } = await github.rest.issues.listComments({
209+
owner: context.repo.owner,
210+
repo: context.repo.repo,
211+
issue_number: context.issue.number,
212+
});
213+
214+
const botComment = comments.find(comment =>
215+
comment.user.type === 'Bot' &&
216+
comment.body.includes('## Benchmark Results')
217+
);
218+
219+
const commentBody = comment + '\n\n---\n\n📊 Download the [benchmark artifacts](' +
220+
`https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` +
221+
') to view flame graphs and detailed profiles.';
222+
223+
if (botComment) {
224+
await github.rest.issues.updateComment({
225+
owner: context.repo.owner,
226+
repo: context.repo.repo,
227+
comment_id: botComment.id,
228+
body: commentBody
229+
});
230+
} else {
231+
await github.rest.issues.createComment({
232+
owner: context.repo.owner,
233+
repo: context.repo.repo,
234+
issue_number: context.issue.number,
235+
body: commentBody
236+
});
237+
}
238+
239+
- name: Generate profile comparison report
240+
run: |
241+
echo "## Profile Analysis" > benchmark-results/analysis.md
242+
echo "" >> benchmark-results/analysis.md
243+
echo "### Instructions for Viewing Flame Graphs" >> benchmark-results/analysis.md
244+
echo "" >> benchmark-results/analysis.md
245+
echo "1. Download the benchmark-profiles artifact from this workflow run" >> benchmark-results/analysis.md
246+
echo "2. Extract the archive" >> benchmark-results/analysis.md
247+
echo "3. Open \`base/flamegraph.svg\` and \`pr/flamegraph.svg\` in a browser" >> benchmark-results/analysis.md
248+
echo "4. Compare the two flame graphs side by side" >> benchmark-results/analysis.md
249+
echo "" >> benchmark-results/analysis.md
250+
echo "### Using pprof for Interactive Analysis" >> benchmark-results/analysis.md
251+
echo "" >> benchmark-results/analysis.md
252+
echo '```bash' >> benchmark-results/analysis.md
253+
echo "# View differential profile interactively" >> benchmark-results/analysis.md
254+
echo "go tool pprof -http=:8080 -base base/cpu.prof pr/cpu.prof" >> benchmark-results/analysis.md
255+
echo "" >> benchmark-results/analysis.md
256+
echo "# Generate diff flame graph" >> benchmark-results/analysis.md
257+
echo "go tool pprof -svg -base base/cpu.prof pr/cpu.prof > diff.svg" >> benchmark-results/analysis.md
258+
echo '```' >> benchmark-results/analysis.md
259+
260+
cat benchmark-results/analysis.md
261+
262+
- name: Upload analysis instructions
263+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.0
264+
with:
265+
name: analysis-instructions
266+
path: benchmark-results/analysis.md
267+
retention-days: 30

0 commit comments

Comments
 (0)