Skip to content

Commit 4706787

Browse files
camchenrycamchenry
committed
ci: add profiling diff workflow
1 parent 36f5ab9 commit 4706787

File tree

1 file changed

+283
-0
lines changed

1 file changed

+283
-0
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
name: Benchmark PR
2+
3+
permissions:
4+
pull-requests: write
5+
contents: read
6+
7+
on:
8+
pull_request:
9+
types: [labeled]
10+
11+
jobs:
12+
benchmark:
13+
if: github.event.label.name == 'bench'
14+
runs-on: ubuntu-latest
15+
timeout-minutes: 60
16+
steps:
17+
- name: Checkout PR
18+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
19+
with:
20+
fetch-depth: 0
21+
submodules: true
22+
persist-credentials: false
23+
ref: ${{ github.event.pull_request.head.sha }}
24+
25+
- uses: oxc-project/setup-node@fdbf0dfd334c4e6d56ceeb77d91c76339c2a0885 # v1.0.4
26+
27+
- uses: ./.github/actions/setup
28+
29+
- name: Install dependencies for benchmarking
30+
run: sudo apt-get update && sudo apt-get install -y graphviz jq
31+
32+
- name: Clone vscode for benchmarking
33+
run: |
34+
mkdir -p benchmarks
35+
cd benchmarks
36+
# Clone vscode at pinned commit (v1.99.0 release)
37+
git clone --depth 1 --single-branch --branch 1.99.0 https://github.com/microsoft/vscode
38+
39+
- name: Setup vscode for benchmarking
40+
run: |
41+
cd benchmarks/vscode
42+
# Install minimal dependencies needed for TypeScript compilation
43+
npm install --ignore-scripts
44+
45+
- name: Build PR binary
46+
run: just build
47+
48+
- name: Generate headless payload for vscode
49+
run: |
50+
mkdir -p benchmark-results/pr
51+
# Generate payload with all TypeScript files in vscode/src
52+
find benchmarks/vscode/src -type f \( -name "*.ts" -o -name "*.tsx" \) | \
53+
jq -R -s '
54+
split("\n")[:-1] |
55+
{
56+
version: 2,
57+
configs: [
58+
{
59+
file_paths: .,
60+
rules: [
61+
{ name: "await-thenable" },
62+
{ name: "no-array-delete" },
63+
{ name: "no-base-to-string" },
64+
{ name: "no-confusing-void-expression" },
65+
{ name: "no-duplicate-type-constituents" },
66+
{ name: "no-floating-promises" },
67+
{ name: "no-for-in-array" },
68+
{ name: "no-implied-eval" },
69+
{ name: "no-meaningless-void-operator" },
70+
{ name: "no-misused-promises" },
71+
{ name: "no-unnecessary-type-assertion" },
72+
{ name: "no-unsafe-argument" },
73+
{ name: "no-unsafe-assignment" },
74+
{ name: "no-unsafe-call" },
75+
{ name: "no-unsafe-member-access" },
76+
{ name: "no-unsafe-return" }
77+
]
78+
}
79+
]
80+
}
81+
' > benchmark-results/pr/payload.json
82+
echo "Generated payload with $(jq '.configs[0].file_paths | length' benchmark-results/pr/payload.json) files"
83+
84+
- name: Run PR benchmark with CPU profiling
85+
run: |
86+
export TSGOLINT_BENCHMARK_PROJECT=vscode
87+
# Run in headless mode, capture output size but don't log to console
88+
OUTPUT=$(./tsgolint --cpuprof benchmark-results/pr/cpu.prof --headless < benchmark-results/pr/payload.json)
89+
OUTPUT_SIZE=$(echo "$OUTPUT" | wc -c)
90+
echo "Output size: $OUTPUT_SIZE bytes" > benchmark-results/pr/output-size.txt
91+
cat benchmark-results/pr/output-size.txt
92+
continue-on-error: true
93+
94+
- name: Generate PR flame graph
95+
run: |
96+
go tool pprof -svg benchmark-results/pr/cpu.prof > benchmark-results/pr/flamegraph.svg
97+
go tool pprof -top benchmark-results/pr/cpu.prof > benchmark-results/pr/profile.txt
98+
99+
- name: Checkout base branch
100+
run: |
101+
git fetch origin ${{ github.event.pull_request.base.ref }}
102+
git checkout origin/${{ github.event.pull_request.base.ref }}
103+
git submodule update --init --recursive
104+
105+
- name: Apply typescript-go patches for base
106+
run: |
107+
pushd typescript-go
108+
git am --3way --no-gpg-sign ../patches/*.patch
109+
popd
110+
111+
- name: Expose typescript-go collections package for base
112+
run: |
113+
mkdir -p internal/collections
114+
find ./typescript-go/internal/collections -type f ! -name '*_test.go' -exec cp {} internal/collections/ \;
115+
116+
- name: Build base binary
117+
run: just build
118+
119+
- name: Generate headless payload for vscode (base)
120+
run: |
121+
mkdir -p benchmark-results/base
122+
# Generate payload with all TypeScript files in vscode/src
123+
find benchmarks/vscode/src -type f \( -name "*.ts" -o -name "*.tsx" \) | \
124+
jq -R -s '
125+
split("\n")[:-1] |
126+
{
127+
version: 2,
128+
configs: [
129+
{
130+
tsconfig: "benchmarks/vscode/src/tsconfig.json",
131+
file_paths: .,
132+
rules: [
133+
{ name: "await-thenable" },
134+
{ name: "no-array-delete" },
135+
{ name: "no-base-to-string" },
136+
{ name: "no-confusing-void-expression" },
137+
{ name: "no-duplicate-type-constituents" },
138+
{ name: "no-floating-promises" },
139+
{ name: "no-for-in-array" },
140+
{ name: "no-implied-eval" },
141+
{ name: "no-meaningless-void-operator" },
142+
{ name: "no-misused-promises" },
143+
{ name: "no-unnecessary-type-assertion" },
144+
{ name: "no-unsafe-argument" },
145+
{ name: "no-unsafe-assignment" },
146+
{ name: "no-unsafe-call" },
147+
{ name: "no-unsafe-member-access" },
148+
{ name: "no-unsafe-return" }
149+
]
150+
}
151+
]
152+
}
153+
' > benchmark-results/base/payload.json
154+
echo "Generated payload with $(jq '.configs[0].file_paths | length' benchmark-results/base/payload.json) files"
155+
156+
- name: Run base benchmark with CPU profiling
157+
run: |
158+
export TSGOLINT_BENCHMARK_PROJECT=vscode
159+
# Run in headless mode, capture output size but don't log to console
160+
OUTPUT=$(./tsgolint --cpuprof benchmark-results/base/cpu.prof --headless < benchmark-results/base/payload.json)
161+
OUTPUT_SIZE=$(echo "$OUTPUT" | wc -c)
162+
echo "Output size: $OUTPUT_SIZE bytes" > benchmark-results/base/output-size.txt
163+
cat benchmark-results/base/output-size.txt
164+
continue-on-error: true
165+
166+
- name: Generate base flame graph
167+
run: |
168+
go tool pprof -svg benchmark-results/base/cpu.prof > benchmark-results/base/flamegraph.svg
169+
go tool pprof -top benchmark-results/base/cpu.prof > benchmark-results/base/profile.txt
170+
171+
- name: Compare profiles
172+
id: compare
173+
run: |
174+
echo "## Benchmark Results" > benchmark-results/comment.md
175+
echo "" >> benchmark-results/comment.md
176+
echo "**Benchmark Mode:** Headless ($(jq '.configs[0].file_paths | length' benchmark-results/pr/payload.json) files, $(jq '.configs[0].rules | length' benchmark-results/pr/payload.json) rules)" >> benchmark-results/comment.md
177+
echo "" >> benchmark-results/comment.md
178+
echo "### Output Sizes" >> benchmark-results/comment.md
179+
echo "" >> benchmark-results/comment.md
180+
echo "- **Base:** $(cat benchmark-results/base/output-size.txt)" >> benchmark-results/comment.md
181+
echo "- **PR:** $(cat benchmark-results/pr/output-size.txt)" >> benchmark-results/comment.md
182+
echo "" >> benchmark-results/comment.md
183+
echo "### CPU Profile Comparison" >> benchmark-results/comment.md
184+
echo "" >> benchmark-results/comment.md
185+
echo "#### Base Branch (${{ github.event.pull_request.base.ref }})" >> benchmark-results/comment.md
186+
echo '```' >> benchmark-results/comment.md
187+
head -n 20 benchmark-results/base/profile.txt >> benchmark-results/comment.md
188+
echo '```' >> benchmark-results/comment.md
189+
echo "" >> benchmark-results/comment.md
190+
echo "#### PR Branch (${{ github.event.pull_request.head.ref }})" >> benchmark-results/comment.md
191+
echo '```' >> benchmark-results/comment.md
192+
head -n 20 benchmark-results/pr/profile.txt >> benchmark-results/comment.md
193+
echo '```' >> benchmark-results/comment.md
194+
echo "" >> benchmark-results/comment.md
195+
echo "### Differential Profile" >> benchmark-results/comment.md
196+
echo "" >> benchmark-results/comment.md
197+
echo "Comparing PR against base (positive values = PR is slower):" >> benchmark-results/comment.md
198+
echo '```' >> benchmark-results/comment.md
199+
go tool pprof -top -base benchmark-results/base/cpu.prof benchmark-results/pr/cpu.prof | head -n 25 >> benchmark-results/comment.md || echo "No significant differences found" >> benchmark-results/comment.md
200+
echo '```' >> benchmark-results/comment.md
201+
echo "" >> benchmark-results/comment.md
202+
echo "<details>" >> benchmark-results/comment.md
203+
echo "<summary>View Full Flame Graphs</summary>" >> benchmark-results/comment.md
204+
echo "" >> benchmark-results/comment.md
205+
echo "**Note:** Flame graphs have been generated but GitHub doesn't render SVGs in comments." >> benchmark-results/comment.md
206+
echo "Download the artifacts to view them locally." >> benchmark-results/comment.md
207+
echo "" >> benchmark-results/comment.md
208+
echo "</details>" >> benchmark-results/comment.md
209+
210+
- name: Upload benchmark artifacts
211+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.0
212+
with:
213+
name: benchmark-profiles
214+
path: benchmark-results/
215+
retention-days: 30
216+
217+
- name: Post benchmark results
218+
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
219+
with:
220+
script: |
221+
const fs = require('fs');
222+
const comment = fs.readFileSync('benchmark-results/comment.md', 'utf8');
223+
224+
const { data: comments } = await github.rest.issues.listComments({
225+
owner: context.repo.owner,
226+
repo: context.repo.repo,
227+
issue_number: context.issue.number,
228+
});
229+
230+
const botComment = comments.find(comment =>
231+
comment.user.type === 'Bot' &&
232+
comment.body.includes('## Benchmark Results')
233+
);
234+
235+
const commentBody = comment + '\n\n---\n\n📊 Download the [benchmark artifacts](' +
236+
`https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` +
237+
') to view flame graphs and detailed profiles.';
238+
239+
if (botComment) {
240+
await github.rest.issues.updateComment({
241+
owner: context.repo.owner,
242+
repo: context.repo.repo,
243+
comment_id: botComment.id,
244+
body: commentBody
245+
});
246+
} else {
247+
await github.rest.issues.createComment({
248+
owner: context.repo.owner,
249+
repo: context.repo.repo,
250+
issue_number: context.issue.number,
251+
body: commentBody
252+
});
253+
}
254+
255+
- name: Generate profile comparison report
256+
run: |
257+
echo "## Profile Analysis" > benchmark-results/analysis.md
258+
echo "" >> benchmark-results/analysis.md
259+
echo "### Instructions for Viewing Flame Graphs" >> benchmark-results/analysis.md
260+
echo "" >> benchmark-results/analysis.md
261+
echo "1. Download the benchmark-profiles artifact from this workflow run" >> benchmark-results/analysis.md
262+
echo "2. Extract the archive" >> benchmark-results/analysis.md
263+
echo "3. Open \`base/flamegraph.svg\` and \`pr/flamegraph.svg\` in a browser" >> benchmark-results/analysis.md
264+
echo "4. Compare the two flame graphs side by side" >> benchmark-results/analysis.md
265+
echo "" >> benchmark-results/analysis.md
266+
echo "### Using pprof for Interactive Analysis" >> benchmark-results/analysis.md
267+
echo "" >> benchmark-results/analysis.md
268+
echo '```bash' >> benchmark-results/analysis.md
269+
echo "# View differential profile interactively" >> benchmark-results/analysis.md
270+
echo "go tool pprof -http=:8080 -base base/cpu.prof pr/cpu.prof" >> benchmark-results/analysis.md
271+
echo "" >> benchmark-results/analysis.md
272+
echo "# Generate diff flame graph" >> benchmark-results/analysis.md
273+
echo "go tool pprof -svg -base base/cpu.prof pr/cpu.prof > diff.svg" >> benchmark-results/analysis.md
274+
echo '```' >> benchmark-results/analysis.md
275+
276+
cat benchmark-results/analysis.md
277+
278+
- name: Upload analysis instructions
279+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.0
280+
with:
281+
name: analysis-instructions
282+
path: benchmark-results/analysis.md
283+
retention-days: 30

0 commit comments

Comments
 (0)