Skip to content

Commit 7041c39

Browse files
jeremylongshoreclaude
andcommitted
fix(auth): disable undici+gRPC OTel instrumentations causing login deadlock
The previous fix disabled instrumentation-http but Firebase Admin SDK on Node 18+ uses undici/native fetch, not the http module. The auto-enabled instrumentation-undici was still wrapping every outbound call, creating the same circular dependency with the Cloud Trace gRPC exporter. Also disable instrumentation-grpc to prevent any exporter self-instrumentation. Production logs confirm 4x 300s hangs on set-session after the prior fix. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent cfb9c36 commit 7041c39

2 files changed

Lines changed: 9 additions & 5 deletions

File tree

.beads/issues.jsonl

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,13 @@
5656
{"id":"hustle-5g3","title":"FR-56: Personal best tracking","description":"Track and display personal best juggling count with celebration","status":"open","priority":2,"issue_type":"feature","created_at":"2025-12-13T17:12:51.215685885-06:00","updated_at":"2025-12-13T17:12:51.215685885-06:00","labels":["logging","mvp","skills"],"dependencies":[{"issue_id":"hustle-5g3","depends_on_id":"hustle-lso","type":"parent-child","created_at":"2025-12-13T17:12:56.904522722-06:00","created_by":"daemon","metadata":"{}"}]}
5757
{"id":"hustle-5m0","title":"Hustle MVP v1.0 - Youth Soccer Development Tracking","description":"Complete MVP implementation as defined in 001-PP-PROD-hustle-mvp-v1.md. Target: 40% DAU, 5+ logs/user/week. Key features: multi-child family accounts, verified stats, global comparison, gamification, PWA.","status":"open","priority":0,"issue_type":"epic","created_at":"2025-12-13T17:09:58.576458711-06:00","updated_at":"2025-12-13T17:09:58.576458711-06:00"}
5858
{"id":"hustle-5m0.1","title":"Epic: v1.0.1 Production Stabilization","description":"Audit-driven stabilization work to make auth/login/reset reliable, restore CI/CD gates, and validate Dream Gym end-to-end.\n\nTracked using Beads: https://github.com/steveyegge/beads","status":"open","priority":0,"issue_type":"epic","created_at":"2025-12-28T19:50:12.137522928-06:00","updated_at":"2025-12-28T19:56:03.87272651-06:00","labels":["production","stability"],"dependencies":[{"issue_id":"hustle-5m0.1","depends_on_id":"hustle-5m0","type":"parent-child","created_at":"2025-12-28T19:50:12.140603327-06:00","created_by":"jeremy","metadata":"{}"}]}
59-
{"id":"hustle-5m0.1.1","title":"Epic: Auth \u0026 Account Recovery","description":"Make sign-in/session cookies/password reset resilient for real users (parents + kids).","status":"open","priority":0,"issue_type":"epic","created_at":"2025-12-28T19:50:25.892935116-06:00","updated_at":"2025-12-28T19:50:25.892935116-06:00","labels":["auth","stability"],"dependencies":[{"issue_id":"hustle-5m0.1.1","depends_on_id":"hustle-5m0.1","type":"parent-child","created_at":"2025-12-28T19:50:25.89444485-06:00","created_by":"jeremy","metadata":"{}"}]}
59+
{"id":"hustle-5m0.1.1","title":"Epic: Auth \u0026 Account Recovery","description":"Make sign-in/session cookies/password reset resilient for real users (parents + kids).","status":"closed","priority":0,"issue_type":"epic","created_at":"2025-12-28T19:50:25.892935116-06:00","updated_at":"2026-02-25T18:29:29.786142768-06:00","closed_at":"2026-02-25T18:29:29.786142768-06:00","close_reason":"All P0/P1 auth stability tasks complete. Only hustle-5m0.1.1.5 (P2 account migration) remains — deferred, not blocking production.","labels":["auth","stability"],"dependencies":[{"issue_id":"hustle-5m0.1.1","depends_on_id":"hustle-5m0.1","type":"parent-child","created_at":"2025-12-28T19:50:25.89444485-06:00","created_by":"jeremy","metadata":"{}"}]}
6060
{"id":"hustle-5m0.1.1.1","title":"Bug: Login works without email verification","description":"Unverified emails must be able to login and reach dashboard; show a non-blocking banner prompting verification.","status":"closed","priority":0,"issue_type":"bug","created_at":"2025-12-28T19:51:46.627352001-06:00","updated_at":"2025-12-28T19:52:50.887848937-06:00","closed_at":"2025-12-28T19:52:50.887848937-06:00","labels":["auth","login"],"dependencies":[{"issue_id":"hustle-5m0.1.1.1","depends_on_id":"hustle-5m0.1.1","type":"parent-child","created_at":"2025-12-28T19:51:46.62940291-06:00","created_by":"jeremy","metadata":"{}"}]}
6161
{"id":"hustle-5m0.1.1.2","title":"Bug: Password reset + verification links are reliable","description":"Forgot password and resend verification must work even when Resend is not configured (fallback to Firebase email delivery), and links must target the correct website origin.","status":"closed","priority":0,"issue_type":"bug","created_at":"2025-12-28T19:51:57.202551911-06:00","updated_at":"2025-12-28T19:52:57.905985522-06:00","closed_at":"2025-12-28T19:52:57.905985522-06:00","labels":["auth","email","password-reset"],"dependencies":[{"issue_id":"hustle-5m0.1.1.2","depends_on_id":"hustle-5m0.1.1","type":"parent-child","created_at":"2025-12-28T19:51:57.204213733-06:00","created_by":"jeremy","metadata":"{}"}]}
6262
{"id":"hustle-5m0.1.1.3","title":"Task: Friendly auth error messages","description":"Map Firebase Auth error codes to user-friendly messages on /login, /register, /forgot-password.","status":"closed","priority":1,"issue_type":"task","created_at":"2025-12-28T19:52:07.415590378-06:00","updated_at":"2026-02-16T12:16:18.365387814-06:00","closed_at":"2026-02-16T12:16:18.365387814-06:00","close_reason":"Fixed: forgot-password 504 timeout (replaced dynamic imports with static), added registration success banner on login page, added password reset success banner. Commit 3d8e4e35, deployed to production.","labels":["auth","ux"],"dependencies":[{"issue_id":"hustle-5m0.1.1.3","depends_on_id":"hustle-5m0.1.1","type":"parent-child","created_at":"2025-12-28T19:52:07.417157488-06:00","created_by":"jeremy","metadata":"{}"}]}
6363
{"id":"hustle-5m0.1.1.4","title":"Task: Add /api/auth/whoami for session debugging","description":"Add an authenticated endpoint that returns current session UID/email/emailVerified and cookie presence to speed up debugging login issues.","status":"closed","priority":1,"issue_type":"task","created_at":"2025-12-28T19:52:18.703713886-06:00","updated_at":"2025-12-30T12:53:07.240951979-06:00","closed_at":"2025-12-30T12:53:07.240951979-06:00","labels":["auth","support"],"dependencies":[{"issue_id":"hustle-5m0.1.1.4","depends_on_id":"hustle-5m0.1.1","type":"parent-child","created_at":"2025-12-28T19:52:18.705671855-06:00","created_by":"jeremy","metadata":"{}"}]}
6464
{"id":"hustle-5m0.1.1.5","title":"Task: On-demand account migration via password reset","description":"If an email exists in Firestore but has no Firebase Auth user, create a Firebase Auth user with a random password and immediately send a password reset link (email ownership required).","status":"open","priority":2,"issue_type":"task","created_at":"2025-12-28T19:52:29.566270329-06:00","updated_at":"2025-12-28T19:52:29.566270329-06:00","labels":["auth","migration"],"dependencies":[{"issue_id":"hustle-5m0.1.1.5","depends_on_id":"hustle-5m0.1.1","type":"parent-child","created_at":"2025-12-28T19:52:29.568199404-06:00","created_by":"jeremy","metadata":"{}"}]}
65+
{"id":"hustle-5m0.1.1.6","title":"Task: Server hang detection timeouts for auth flow","description":"Add timeout safety nets to prevent 300s hangs: 10s AbortController on client fetch, 10s withTimeout on verifyIdToken/createSessionCookie, 5s timeout on health check Firestore ping","status":"closed","priority":0,"issue_type":"task","owner":"noreply@localhost.local","created_at":"2026-02-25T18:28:55.714964469-06:00","created_by":"jeremylongshore","updated_at":"2026-02-25T18:28:58.890862387-06:00","closed_at":"2026-02-25T18:28:58.890862387-06:00","close_reason":"timeout safety nets shipped and pushed (cfb9c364): 10s AbortController on client set-session fetch, 10s withTimeout on verifyIdToken + createSessionCookie with 504 on timeout, 5s timeout on health check Firestore ping","labels":["auth","stability"],"dependencies":[{"issue_id":"hustle-5m0.1.1.6","depends_on_id":"hustle-5m0.1.1","type":"parent-child","created_at":"2026-02-25T18:28:55.717128344-06:00","created_by":"jeremylongshore"}]}
6566
{"id":"hustle-5m0.1.2","title":"Epic: QA + Regression Gates","description":"Keep unit + smoke tests green and add real-world coverage for critical journeys.","status":"open","priority":0,"issue_type":"epic","created_at":"2025-12-28T19:50:31.152634213-06:00","updated_at":"2025-12-28T19:50:31.152634213-06:00","labels":["qa","tests"],"dependencies":[{"issue_id":"hustle-5m0.1.2","depends_on_id":"hustle-5m0.1","type":"parent-child","created_at":"2025-12-28T19:50:31.159940576-06:00","created_by":"jeremy","metadata":"{}"},{"issue_id":"hustle-5m0.1.2","depends_on_id":"hustle-5m0.1.1","type":"blocks","created_at":"2025-12-28T19:51:13.123369163-06:00","created_by":"jeremy","metadata":"{}"}]}
6667
{"id":"hustle-5m0.1.2.1","title":"Task: Smoke tests are safe and runnable in CI","description":"Ensure CI runs a smoke subset (no real credentials) and skips real-world auth smoke unless SMOKE_TEST_EMAIL/PASSWORD provided.","status":"closed","priority":0,"issue_type":"task","created_at":"2025-12-28T19:53:22.038569822-06:00","updated_at":"2025-12-28T19:54:01.8493968-06:00","closed_at":"2025-12-28T19:54:01.8493968-06:00","labels":["playwright","qa"],"dependencies":[{"issue_id":"hustle-5m0.1.2.1","depends_on_id":"hustle-5m0.1.2","type":"parent-child","created_at":"2025-12-28T19:53:22.040086546-06:00","created_by":"jeremy","metadata":"{}"}]}
6768
{"id":"hustle-5m0.1.2.2","title":"Task: Nightly synthetic QA against staging","description":"Run Playwright smoke suite nightly and on-demand against a configured base URL (SYNTHETIC_QA_BASE_URL) with secrets.","status":"closed","priority":1,"issue_type":"task","created_at":"2025-12-28T19:53:32.965489421-06:00","updated_at":"2025-12-28T19:54:02.134222486-06:00","closed_at":"2025-12-28T19:54:02.134222486-06:00","labels":["github-actions","qa"],"dependencies":[{"issue_id":"hustle-5m0.1.2.2","depends_on_id":"hustle-5m0.1.2","type":"parent-child","created_at":"2025-12-28T19:53:32.966921036-06:00","created_by":"jeremy","metadata":"{}"}]}

src/instrumentation.ts

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,14 @@ export async function register() {
2222
traceExporter,
2323
instrumentations: [
2424
getNodeAutoInstrumentations({
25-
// Disable instrumentations that cause deadlocks with Firebase Admin SDK
26-
// HTTP instrumentation wraps outgoing REST calls (verifyIdToken, etc.)
27-
// and creates circular dependency with Cloud Trace gRPC exporter
28-
'@opentelemetry/instrumentation-fs': { enabled: false },
25+
// DISABLE ALL network/IO instrumentations to prevent deadlocks.
26+
// Auto-instrumentations wrap outgoing calls (HTTP, undici/fetch, gRPC)
27+
// creating circular dependencies with the Cloud Trace gRPC exporter.
28+
// Only keep Express/Fastify for request-level spans.
2929
'@opentelemetry/instrumentation-http': { enabled: false },
30+
'@opentelemetry/instrumentation-undici': { enabled: false },
31+
'@opentelemetry/instrumentation-grpc': { enabled: false },
32+
'@opentelemetry/instrumentation-fs': { enabled: false },
3033
'@opentelemetry/instrumentation-dns': { enabled: false },
3134
'@opentelemetry/instrumentation-net': { enabled: false },
3235
}),

0 commit comments

Comments
 (0)