fix(jest-worker): fix hanging when workers are killed or unexpectedly exit (#13566)

Author: gluxon

CHANGELOG.md

@@ -7,6 +7,7 @@
 ### Fixes
 
 - `[jest-mock]` Treat cjs modules as objects so they can be mocked ([#13513](https://github.com/facebook/jest/pull/13513))
+- `[jest-worker]` Throw an error instead of hanging when jest workers terminate unexpectedly ([#13566](https://github.com/facebook/jest/pull/13566))
 
 ### Chore & Maintenance
 

packages/jest-worker/src/workers/ChildProcessWorker.ts

@@ -343,7 +343,7 @@ export default class ChildProcessWorker
     }
   }
 
-  private _onExit(exitCode: number | null) {
+  private _onExit(exitCode: number | null, signal: NodeJS.Signals | null) {
     this._workerReadyPromise = undefined;
     this._resolveWorkerReady = undefined;
 
@@ -372,6 +372,46 @@ export default class ChildProcessWorker
         this._child.send(this._request);
       }
     } else {
+      // At this point, it's not clear why the child process exited. There could
+      // be several reasons:
+      //
+      //  1. The child process exited successfully after finishing its work.
+      //     This is the most likely case.
+      //  2. The child process crashed in a manner that wasn't caught through
+      //     any of the heuristic-based checks above.
+      //  3. The child process was killed by another process or daemon unrelated
+      //     to Jest. For example, oom-killer on Linux may have picked the child
+      //     process to kill because overall system memory is constrained.
+      //
+      // If there's a pending request to the child process in any of those
+      // situations, the request still needs to be handled in some manner before
+      // entering the shutdown phase. Otherwise the caller expecting a response
+      // from the worker will never receive indication that something unexpected
+      // happened and hang forever.
+      //
+      // In normal operation, the request is handled and cleared before the
+      // child process exits. If it's still present, it's not clear what
+      // happened and probably best to throw an error. In practice, this usually
+      // happens when the child process is killed externally.
+      //
+      // There's a reasonable argument that the child process should be retried
+      // with request re-sent in this scenario. However, if the problem was due
+      // to situations such as oom-killer attempting to free up system
+      // resources, retrying would exacerbate the problem.
+      const isRequestStillPending = !!this._request;
+      if (isRequestStillPending) {
+        // If a signal is present, we can be reasonably confident the process
+        // was killed externally. Log this fact so it's more clear to users that
+        // something went wrong externally, rather than a bug in Jest itself.
+        const error = new Error(
+          signal != null
+            ? `A jest worker process (pid=${this._child.pid}) was terminated by another process: signal=${signal}, exitCode=${exitCode}. Operating system logs may contain more information on why this occurred.`
+            : `A jest worker process (pid=${this._child.pid}) crashed for an unknown reason: exitCode=${exitCode}`,
+        );
+
+        this._onProcessEnd(error, null);
+      }
+
       this._shutdown();
     }
   }

packages/jest-worker/src/workers/NodeThreadsWorker.ts

@@ -254,6 +254,18 @@ export default class ExperimentalWorker
         this._worker.postMessage(this._request);
       }
     } else {
+      // If the worker thread exits while a request is still pending, throw an
+      // error. This is unexpected and tests may not have run to completion.
+      const isRequestStillPending = !!this._request;
+      if (isRequestStillPending) {
+        this._onProcessEnd(
+          new Error(
+            'A Jest worker thread exited unexpectedly before finishing tests for an unknown reason. One of the ways this can happen is if process.exit() was called in testing code.',
+          ),
+          null,
+        );
+      }
+
       this._shutdown();
     }
   }

packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.ts

@@ -385,4 +385,66 @@ describe.each([
       );
     });
   });
+
+  describe('should not hang when worker is killed or unexpectedly terminated', () => {
+    let worker: ChildProcessWorker | ThreadsWorker;
+
+    beforeEach(() => {
+      const options = {
+        childWorkerPath: processChildWorkerPath,
+        maxRetries: 0,
+        silent: true,
+        workerPath: join(__dirname, '__fixtures__', 'SelfKillWorker'),
+      } as unknown as WorkerOptions;
+
+      worker = new ChildProcessWorker(options);
+    });
+
+    afterEach(async () => {
+      await new Promise<void>(resolve => {
+        setTimeout(async () => {
+          if (worker) {
+            worker.forceExit();
+            await worker.waitForExit();
+          }
+
+          resolve();
+        }, 500);
+      });
+    });
+
+    // Regression test for https://github.com/facebook/jest/issues/13183
+    test('onEnd callback is called', async () => {
+      let onEndPromiseResolve: () => void;
+      let onEndPromiseReject: (err: Error) => void;
+      const onEndPromise = new Promise<void>((resolve, reject) => {
+        onEndPromiseResolve = resolve;
+        onEndPromiseReject = reject;
+      });
+
+      const onStart = jest.fn();
+      const onEnd = jest.fn((err: Error | null) => {
+        if (err) {
+          return onEndPromiseReject(err);
+        }
+        onEndPromiseResolve();
+      });
+      const onCustom = jest.fn();
+
+      await worker.waitForWorkerReady();
+
+      // The SelfKillWorker simulates an external process calling SIGTERM on it,
+      // but just SIGTERMs itself underneath the hood to make this test easier.
+      worker.send(
+        [CHILD_MESSAGE_CALL, true, 'selfKill', []],
+        onStart,
+        onEnd,
+        onCustom,
+      );
+
+      // The onEnd callback should be called when the child process exits.
+      await expect(onEndPromise).rejects.toBeInstanceOf(Error);
+      expect(onEnd).toHaveBeenCalled();
+    });
+  });
 });

packages/jest-worker/src/workers/__tests__/__fixtures__/SelfKillWorker.js

@@ -0,0 +1,21 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+const {isMainThread} = require('worker_threads');
+
+async function selfKill() {
+  if (!isMainThread) {
+    // process.exit is documented to only stop the current thread rather than
+    // the entire process in a worker_threads environment.
+    process.exit();
+  }
+
+  process.kill(process.pid);
+}
+
+module.exports = {
+  selfKill,
+};