diff --git a/docs/agents/ssh-tunnels.md b/docs/agents/ssh-tunnels.md index 7c5e1d91..953b8722 100644 --- a/docs/agents/ssh-tunnels.md +++ b/docs/agents/ssh-tunnels.md @@ -269,9 +269,12 @@ Remote termination is best effort with its own three-second budget; local teardown never waits for AWS. SSM cleanup reports a bounded one-time termination result through the runtime -connection attempt when an explicitly disconnected tunnel is finally dropped. -Disconnect subscribes before cancelling an in-flight connection too, so a -session whose ID has not arrived still produces an uncertainty notice. +connection attempt when its tunnel is finally dropped. The workspace subscribes +for each SSM connect and form test, independently of the form and profile +generation, so cancellation, failure, and reconnect cannot discard the result. +Background cleanup waits up to one second for a late stdout session ID, ending +early on stdout EOF; local teardown does not wait for that drain. A session +whose ID never arrives still produces an uncertainty notice. The window keeps unconfirmed results separately from other notices; profile and region are configured settings, not verified identity. No result is persisted, and the local watchdog never waits for the remote request. diff --git a/docs/ssm-tunnels.md b/docs/ssm-tunnels.md index 6bcaa641..eaf01127 100644 --- a/docs/ssm-tunnels.md +++ b/docs/ssm-tunnels.md @@ -94,7 +94,8 @@ Job Object. Local cleanup is independent of a bounded, best-effort remote remote session visible until AWS expires it. A dropped tunnel never falls back to a direct database connection or replays a query. -If AWS does not confirm the termination request after an explicit disconnect, +If AWS does not confirm termination after a connection test, failed or cancelled +connection, reconnect, or disconnect, DBDelve shows the observed session ID, if available, and the configured profile and region in a notice. Copy the ID to locate the session in the AWS Session Manager console and choose the correct account and region before ending diff --git a/src/db/ssm.rs b/src/db/ssm.rs index d5f06ea9..a4bfadba 100644 --- a/src/db/ssm.rs +++ b/src/db/ssm.rs @@ -93,7 +93,7 @@ impl ConnectAttempt { if let Some(send) = cleanup.sender.take() { let _ = send.send(result); } else { - // Startup can fail before Disconnect installs the observer. + // Preserve delivery even when cleanup beats observer registration. cleanup.result = Some(result); } } @@ -295,6 +295,7 @@ struct Output { session: Option, last_diagnostic: Option, closed: bool, + stdout_done: bool, } impl Output { fn line(&mut self, line: &str, stdout: bool) { @@ -433,9 +434,27 @@ fn drain(reader: impl Read + Send + 'static, output: Arc>, stdout: .unwrap_or_else(|p| p.into_inner()) .line(&String::from_utf8_lossy(&line), stdout); } + if stdout { + output.lock().unwrap_or_else(|p| p.into_inner()).stdout_done = true; + } }); } +// Only the background cleanup waits: closing the local tunnel never waits on +// output readers, including a descendant that keeps stdout open indefinitely. +fn cleanup_session(output: &Mutex, timeout: Duration) -> Option { + let deadline = Instant::now() + timeout; + loop { + { + let output = output.lock().unwrap_or_else(|p| p.into_inner()); + if output.session.is_some() || output.stdout_done || Instant::now() >= deadline { + return output.session.clone(); + } + } + thread::sleep(Duration::from_millis(10)); + } +} + pub(super) struct Tunnel { local: SocketAddr, process: Arc>, @@ -622,14 +641,10 @@ impl Drop for Tunnel { .lock() .unwrap_or_else(|p| p.into_inner()) .stop(); - let session = self - .output - .lock() - .unwrap_or_else(|p| p.into_inner()) - .session - .clone(); + let output = self.output.clone(); let config = self.config.clone(); thread::spawn(move || { + let session = cleanup_session(&output, Duration::from_secs(1)); let accepted = if let Some(session) = &session { let mut args = vec![ "ssm".into(), @@ -672,6 +687,41 @@ mod tests { ..Default::default() } } + #[test] + fn cleanup_waits_for_late_stdout_but_not_indefinitely() { + let output = Arc::new(Mutex::new(Output::default())); + let pending = output.clone(); + let reader = thread::spawn(move || { + thread::sleep(Duration::from_millis(30)); + drain( + std::io::Cursor::new(b"Starting session with SessionId: late-session"), + pending, + true, + ); + }); + assert_eq!( + cleanup_session(&output, Duration::from_secs(2)).as_deref(), + Some("late-session") + ); + reader.join().unwrap(); + + let unread = Mutex::new(Output::default()); + assert_eq!(cleanup_session(&unread, Duration::from_millis(20)), None); + } + + #[test] + fn cleanup_returns_on_stdout_eof_before_timeout() { + let output = Arc::new(Mutex::new(Output::default())); + drain(std::io::empty(), output.clone(), true); + let timeout = Duration::from_secs(2); + let started = Instant::now(); + assert_eq!(cleanup_session(&output, timeout), None); + assert!( + started.elapsed() < timeout, + "EOF must end the wait, not the deadline" + ); + } + #[test] fn cleanup_completed_before_subscription_is_delivered() { for accepted in [false, true] { @@ -1007,6 +1057,33 @@ mod lifecycle_tests { thread::sleep(Duration::from_millis(20)); } } + #[test] + fn cleanup_recovers_session_id_parsed_after_drop() { + let mut fixture = Fixture::new("ready"); + let attempt = ConnectAttempt::default(); + fixture.config.attempt = Some(attempt.clone()); + let mut tunnel = fixture.open(); + // Hold the reader's result until after Drop, as when stdout is buffered + // but its drain thread has not yet been scheduled. + let output = Arc::new(Mutex::new(Output::default())); + tunnel.output = output.clone(); + fs::write( + fixture.config.aws_cli.as_ref().unwrap(), + "#!/bin/sh\nprintf '{\"SessionId\":\"late-session\"}'\n", + ) + .unwrap(); + let receiver = attempt.observe_cleanup(); + drop(tunnel); + // Reaching here before providing output proves local Drop did not wait. + output + .lock() + .unwrap() + .line("Starting session with SessionId: late-session", true); + let result = cleanup_result(receiver); + assert!(result.accepted); + assert_eq!(result.session.as_deref(), Some("late-session")); + } + #[test] fn cleanup_result_waits_for_last_tunnel_owner() { let mut fixture = Fixture::new("ready"); diff --git a/src/session.rs b/src/session.rs index 7639b1ef..746f7549 100644 --- a/src/session.rs +++ b/src/session.rs @@ -72,7 +72,6 @@ pub(crate) struct Profile { pub(crate) confirmed_stale: bool, pub(crate) generation: u64, pub(crate) state: ProfileState, - pub(crate) cleanup_attempt: Option, /// Whether the server is known to hold this connection as `mode` says /// (`Connection::set_read_only`): set when a connect lands, which fails /// rather than open unheld, and by the latest hold `set_mode` sent, once diff --git a/src/workspace/e2e.rs b/src/workspace/e2e.rs index cc4aabed..b20e5921 100644 --- a/src/workspace/e2e.rs +++ b/src/workspace/e2e.rs @@ -8270,11 +8270,11 @@ fn ssm_disconnect_delivers_failed_cleanup_notice(cx: &mut TestAppContext) { let (workspace, cx) = launch_on_sqlite(cx); let attempt = crate::db::ConnectAttempt::default(); attempt.finish().unwrap(); - let profile_index = workspace.update(cx, |workspace, _| { + let profile_index = workspace.update(cx, |workspace, cx| { + workspace.observe_ssm_cleanup(&attempt, cx); let index = workspace.active.unwrap(); let profile = &mut workspace.profiles[index]; assert!(matches!(profile.state, ProfileState::Connected(_))); - profile.cleanup_attempt = Some(attempt.clone()); index }); disconnect(cx); @@ -8284,7 +8284,6 @@ fn ssm_disconnect_delivers_failed_cleanup_notice(cx: &mut TestAppContext) { workspace.profiles[profile_index].state, ProfileState::Idle )); - assert!(workspace.profiles[profile_index].cleanup_attempt.is_none()); assert!(workspace.cleanup_notices.is_empty()); }); // Deliver after disconnect returns, as the background AWS request does. @@ -8311,17 +8310,16 @@ fn ssm_disconnect_does_not_warn_after_successful_cleanup(cx: &mut TestAppContext let (workspace, cx) = launch_on_sqlite(cx); let attempt = crate::db::ConnectAttempt::default(); attempt.finish().unwrap(); - let profile_index = workspace.update(cx, |workspace, _| { + workspace.update(cx, |workspace, cx| { + workspace.observe_ssm_cleanup(&attempt, cx); let index = workspace.active.unwrap(); let profile = &mut workspace.profiles[index]; assert!(matches!(profile.state, ProfileState::Connected(_))); - profile.cleanup_attempt = Some(attempt.clone()); index }); disconnect(cx); workspace.read_with(cx, |workspace, _| { assert!(workspace.active.is_none()); - assert!(workspace.profiles[profile_index].cleanup_attempt.is_none()); }); attempt.report_cleanup(crate::db::ssm::CleanupResult { accepted: true, @@ -8344,9 +8342,9 @@ fn ssm_disconnect_while_connecting_delivers_missing_session_notice(cx: &mut Test store::tests::with_home(|| { let (workspace, cx) = launch_on_sqlite(cx); let attempt = crate::db::ConnectAttempt::default(); - workspace.update(cx, |workspace, _| { + workspace.update(cx, |workspace, cx| { + workspace.observe_ssm_cleanup(&attempt, cx); let profile = workspace.profile_mut().unwrap(); - profile.cleanup_attempt = Some(attempt.clone()); profile.state = ProfileState::Connecting { _attempt: attempt.guard(), }; @@ -8388,7 +8386,6 @@ fn ssm_disconnect_delivers_cleanup_completed_before_subscription(cx: &mut TestAp let attempt = crate::db::ConnectAttempt::default(); workspace.update(cx, |workspace, _| { let profile = workspace.profile_mut().unwrap(); - profile.cleanup_attempt = Some(attempt.clone()); profile.state = ProfileState::Connecting { _attempt: attempt.guard(), }; @@ -8401,6 +8398,9 @@ fn ssm_disconnect_delivers_cleanup_completed_before_subscription(cx: &mut TestAp region: Some("test-region".into()), }; attempt.report_cleanup(result.clone()); + workspace.update(cx, |workspace, cx| { + workspace.observe_ssm_cleanup(&attempt, cx) + }); disconnect(cx); settle(cx); workspace.read_with(cx, |workspace, _| { @@ -12099,3 +12099,243 @@ fn a_stored_selection_waits_for_a_connection_that_loads_later(cx: &mut TestAppCo quit(workspace, cx); }); } + +#[cfg(all(unix, any(target_os = "macos", target_env = "gnu")))] +#[derive(Clone, Copy, Debug)] +enum SsmCleanupScenario { + FailedConnect, + FailedFormTest, + ClosedFormTest, + Reconnect, + Disconnect, +} + +// Opt in only for lifecycle tests that must act while a real process is running. +thread_local! { + pub(super) static THREADED_BLOCKING: std::cell::Cell = const { std::cell::Cell::new(false) }; +} +pub(super) fn threaded_work( + executor: &gpui::BackgroundExecutor, + work: impl FnOnce() -> T + Send + 'static, +) -> gpui::Task { + let (send, receive) = std::sync::mpsc::sync_channel(1); + std::thread::spawn(move || { + let _ = send.send(work()); + }); + // Only the test thread wakes GPUI's deterministic scheduler. + executor.spawn(std::future::poll_fn(move |cx| match receive.try_recv() { + Ok(result) => std::task::Poll::Ready(result), + Err(std::sync::mpsc::TryRecvError::Empty) => { + cx.waker().wake_by_ref(); + std::thread::yield_now(); + std::task::Poll::Pending + } + Err(std::sync::mpsc::TryRecvError::Disconnected) => panic!("test worker panicked"), + })) +} + +// These tests run only disposable shell tools: no AWS credentials or network. +#[cfg(all(unix, any(target_os = "macos", target_env = "gnu")))] +fn ssm_cleanup_lifecycle(cx: &mut TestAppContext, scenario: SsmCleanupScenario) { + use std::os::unix::fs::PermissionsExt; + struct ThreadedBlocking; + impl ThreadedBlocking { + fn new() -> Self { + THREADED_BLOCKING.set(true); + Self + } + } + impl Drop for ThreadedBlocking { + fn drop(&mut self) { + THREADED_BLOCKING.set(false); + } + } + store::tests::with_home(|| { + let (workspace, cx) = launch_on_sqlite(cx); + cx.executor().allow_parking(); + let _threaded = ThreadedBlocking::new(); + let root = store::dbdelve_directory().unwrap().join("fake-ssm"); + std::fs::create_dir_all(&root).unwrap(); + let aws = root.join("aws"); + let plugin = root.join("session-manager-plugin"); + let started = root.join("started"); + let release = root.join("release-cleanup"); + let opening = matches!( + scenario, + SsmCleanupScenario::ClosedFormTest + | SsmCleanupScenario::Reconnect + | SsmCleanupScenario::Disconnect + ); + let startup = if opening { + "while :; do sleep 1; done" + } else { + "exit 1" + }; + std::fs::write(&aws, format!( + "#!/bin/sh\nif [ \"$1\" = --version ]; then echo aws-cli/2.0.0; exit; fi\nif [ \"$2\" = terminate-session ]; then while [ ! -e '{}' ]; do sleep 0.01; done; exit 1; fi\necho 'Starting session with SessionId: lifecycle-session'\ntouch '{}'\n{startup}\n", + release.display().to_string().replace('\'', "'\\''"), + started.display().to_string().replace('\'', "'\\''"), + )).unwrap(); + std::fs::write(&plugin, "#!/bin/sh\necho 1.2.3.4\n").unwrap(); + for tool in [&aws, &plugin] { + std::fs::set_permissions(tool, std::fs::Permissions::from_mode(0o700)).unwrap(); + } + let config = ConnectionConfig::Postgres(crate::db::PostgresConfig { + server: crate::db::ServerConfig { + host: "never-contacted.example.test".into(), + user: "reader".into(), + password: "fake".into(), + ssm: Some(crate::db::SsmTunnel { + target: "i-0123456789abcdef0".into(), + aws_cli: Some(aws), + session_manager_plugin: Some(plugin), + ..Default::default() + }), + ..Default::default() + }, + ..Default::default() + }); + let form_test = matches!( + scenario, + SsmCleanupScenario::FailedFormTest | SsmCleanupScenario::ClosedFormTest + ); + let original = workspace.update_in(cx, |workspace, window, cx| { + let index = workspace.active.unwrap(); + let original = workspace.profiles[index].config.clone(); + if form_test { + workspace.form = Some(ConnectionForm::new(Some(&config), window, cx)); + workspace.test_connection(&gpui::ClickEvent::default(), window, cx); + } else { + workspace.profiles[index].config = config; + workspace.begin_connect(index, cx); + } + cx.notify(); + original + }); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while !started.exists() { + // Drain one task at a time: run_until_parked waits for the deliberately + // running connect job, preventing the test from cancelling it. + cx.executor().tick(); + assert!( + std::time::Instant::now() < deadline, + "fake CLI did not start" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + workspace.update_in(cx, |workspace, window, cx| { + if opening { + if form_test { + assert!(matches!( + workspace.form.as_ref().unwrap().test, + Some(crate::connection_form::ConnectionTest::Running(..)) + )); + } else { + assert!(matches!( + workspace.profile().unwrap().state, + ProfileState::Connecting { .. } + )); + } + } + match scenario { + SsmCleanupScenario::ClosedFormTest => { + workspace.form = None; + } + SsmCleanupScenario::Reconnect => { + let index = workspace.active.unwrap(); + workspace.profiles[index].config = original; + workspace.reconnect(index, cx); + } + SsmCleanupScenario::Disconnect => workspace.disconnect(&Disconnect, window, cx), + SsmCleanupScenario::FailedConnect | SsmCleanupScenario::FailedFormTest => {} + } + cx.notify(); + }); + std::fs::write(release, "").unwrap(); + loop { + cx.executor().tick(); + if workspace.read_with(cx, |workspace, _| !workspace.cleanup_notices.is_empty()) { + break; + } + assert!( + std::time::Instant::now() < deadline, + "cleanup notice missing for {scenario:?}" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + settle(cx); + workspace.read_with(cx, |workspace, _| { + assert_eq!(workspace.cleanup_notices.len(), 1); + assert_eq!( + workspace.cleanup_notices[0].session.as_deref(), + Some("lifecycle-session") + ); + }); + // A later disconnect must not subscribe again or repeat the notice. + if workspace.read_with(cx, |workspace, _| workspace.active.is_some()) { + disconnect(cx); + settle(cx); + } + workspace.read_with(cx, |workspace, _| { + assert_eq!(workspace.cleanup_notices.len(), 1) + }); + quit(workspace, cx); + }); +} + +#[cfg(all(unix, any(target_os = "macos", target_env = "gnu")))] +#[gpui::test] +fn ssm_cleanup_after_failed_connect(cx: &mut TestAppContext) { + ssm_cleanup_lifecycle(cx, SsmCleanupScenario::FailedConnect); +} +#[cfg(all(unix, any(target_os = "macos", target_env = "gnu")))] +#[gpui::test] +fn ssm_cleanup_after_failed_form_test(cx: &mut TestAppContext) { + ssm_cleanup_lifecycle(cx, SsmCleanupScenario::FailedFormTest); +} +#[cfg(all(unix, any(target_os = "macos", target_env = "gnu")))] +#[gpui::test] +fn ssm_cleanup_after_closed_form_test(cx: &mut TestAppContext) { + ssm_cleanup_lifecycle(cx, SsmCleanupScenario::ClosedFormTest); +} +#[cfg(all(unix, any(target_os = "macos", target_env = "gnu")))] +#[gpui::test] +fn ssm_cleanup_after_reconnect(cx: &mut TestAppContext) { + ssm_cleanup_lifecycle(cx, SsmCleanupScenario::Reconnect); +} +#[cfg(all(unix, any(target_os = "macos", target_env = "gnu")))] +#[gpui::test] +fn ssm_cleanup_after_disconnect(cx: &mut TestAppContext) { + ssm_cleanup_lifecycle(cx, SsmCleanupScenario::Disconnect); +} + +#[gpui::test] +fn ssm_cleanup_survives_cancelling_a_running_form_test(cx: &mut TestAppContext) { + store::tests::with_home(|| { + let (workspace, cx) = launch_on_sqlite(cx); + let attempt = crate::db::ConnectAttempt::default(); + workspace.update_in(cx, |workspace, window, cx| { + workspace.observe_ssm_cleanup(&attempt, cx); + let mut form = ConnectionForm::new(None, window, cx); + let task = cx.spawn(async move |_, _| std::future::pending::<()>().await); + form.test = Some(crate::connection_form::ConnectionTest::Running( + task, + attempt.guard(), + )); + workspace.form = Some(form); + }); + workspace.update(cx, |workspace, _| workspace.form = None); + assert!(attempt.is_cancelled()); + attempt.report_cleanup(crate::db::ssm::CleanupResult { + accepted: false, + session: None, + profile: None, + region: None, + }); + settle(cx); + workspace.read_with(cx, |workspace, _| { + assert_eq!(workspace.cleanup_notices.len(), 1); + }); + quit(workspace, cx); + }); +} diff --git a/src/workspace/mod.rs b/src/workspace/mod.rs index d88293ad..8c6b1d50 100644 --- a/src/workspace/mod.rs +++ b/src/workspace/mod.rs @@ -2871,6 +2871,10 @@ pub(crate) fn spawn_blocking( // thread and fails a test woken from any other, so the end-to-end tests // get the work inline, as one blocking task at a time. if executor.is_test() { + #[cfg(test)] + if e2e::THREADED_BLOCKING.get() { + return e2e::threaded_work(executor, work); + } return executor.spawn(async move { work() }); } executor.spawn(blocking::unblock(work)) diff --git a/src/workspace/profiles.rs b/src/workspace/profiles.rs index eceee93e..9f7ca709 100644 --- a/src/workspace/profiles.rs +++ b/src/workspace/profiles.rs @@ -284,7 +284,6 @@ impl Workspace { confirmed_stale: stored.confirmed.iter().any(|slug| slug == STALE_ROWS), generation: 0, state: ProfileState::Idle, - cleanup_attempt: None, held: false, holds: Arc::default(), in_transaction: false, @@ -356,7 +355,6 @@ impl Workspace { confirmed_stale: false, generation: 0, state: ProfileState::Idle, - cleanup_attempt: None, held: false, holds: Arc::default(), in_transaction: false, @@ -851,6 +849,27 @@ impl Workspace { } } + pub(super) fn observe_ssm_cleanup( + &mut self, + attempt: &crate::db::ConnectAttempt, + cx: &mut Context, + ) { + let cleanup = attempt.observe_cleanup(); + cx.spawn(async move |workspace, cx| { + if let Ok(result) = cleanup.await + && !result.accepted + { + workspace + .update(cx, |workspace, cx| { + workspace.cleanup_notices.push(result); + cx.notify(); + }) + .ok(); + } + }) + .detach(); + } + /// Opens the form's connection and throws it away, so a profile can be /// checked before it is saved. The same open, keychain lookup and /// Read-only hold `begin_connect` does, or a pass here would not mean the @@ -883,6 +902,7 @@ impl Workspace { let attempt = crate::db::ConnectAttempt::default(); config.set_connect_attempt(attempt.clone()); let guard = attempt.guard(); + let cleanup = config.ssm().map(|_| attempt.clone()); let engine = config.target().label(); let probe = spawn_blocking(cx.background_executor(), move || { let started = std::time::Instant::now(); @@ -943,6 +963,9 @@ impl Workspace { }); }); form.test = Some(ConnectionTest::Running(task, guard)); + if let Some(attempt) = cleanup { + self.observe_ssm_cleanup(&attempt, cx); + } cx.notify(); } @@ -1370,18 +1393,6 @@ impl Workspace { let Some(profile) = self.profile_mut() else { return; }; - let cleanup = if matches!( - profile.state, - ProfileState::Connecting { .. } | ProfileState::Connected(_) - ) { - profile - .cleanup_attempt - .take() - .map(|attempt| attempt.observe_cleanup()) - } else { - profile.cleanup_attempt = None; - None - }; let rolled_back = profile.in_transaction && profile.connection().is_some(); tracing::info!( connection = crate::logging::connection_number(&profile.id), @@ -1399,21 +1410,6 @@ impl Workspace { mark_objects_stale(profile); let name = profile.name.clone(); self.active = None; - if let Some(cleanup) = cleanup { - cx.spawn(async move |workspace, cx| { - if let Ok(result) = cleanup.await - && !result.accepted - { - workspace - .update(cx, |workspace, cx| { - workspace.cleanup_notices.push(result); - cx.notify(); - }) - .ok(); - } - }) - .detach(); - } self.remember_profiles(cx); self.rebuild_connections(cx); self.refocus_front(); @@ -1906,7 +1902,6 @@ impl Workspace { profile.generation = generation; profile.drop_sessions(); let attempt = crate::db::ConnectAttempt::default(); - profile.cleanup_attempt = profile.config.ssm().map(|_| attempt.clone()); profile.state = ProfileState::Connecting { _attempt: attempt.guard(), }; @@ -1928,6 +1923,9 @@ impl Workspace { let mut config = profile.config.clone(); config.set_connect_attempt(attempt.clone()); let mode = profile.mode; + if config.ssm().is_some() { + self.observe_ssm_cleanup(&attempt, cx); + } // The sidebar draws the tree whenever there are saved queries, so the // old catalog's rows have to leave it with the catalog. self.refresh_explorer(&id, cx);