Explorations: Stream API

.github/workflows/ci.yml

@@ -52,6 +52,9 @@ jobs:
             - uses: actions/checkout@v3
             - uses: ./.github/actions/prepare-playground
             - run: npx nx affected --target=build --parallel=3 --verbose
+            # Make sure that both the ESM and CJS builds of the affected
+            # modules still produce usable bundles
+            - run: npx nx affected --target=test:esmcjs --parallel=3 --verbose
 
     # Deploy documentation job
     deploy_docs:

packages/docs/site/docs/13-contributing/04-packages-and-projects.md

@@ -52,7 +52,7 @@ Follow this checklist:
 -   Add a new `workspaces` in the top-level `package.json` file if needed. Otherwise TypeScript won't work correctly and Lerna won't pick up your package for publishing.
 -   Add a `typecheck` task to the `project.json` file in the new project. You can copy it from another project in this repo.
 -   Run `nx typecheck <project name>` to make sure TypeScript is set up correctly.
--   If the project needs to ship both ESM and CJS builds, add a `test:esmcjs` task to the `project.json` and add it to the `dependsOn` list of the `test` task. This way the CI will continuously verify whether your double build works. The alternative is to find out when a disappointed extender files an issue.
+-   If the project needs to ship both ESM and CJS builds, add a `test:esmcjs` task to the `project.json`. This way the CI will continuously verify whether your double build works. The alternative is to find out when a disappointed extender files an issue.
 -   Add the `package-json` task to the build pipeline. See the details below.
 
 ### Package.json

packages/nx-extensions/src/executors/assert-built-esm-and-cjs/executor.ts

@@ -19,17 +19,16 @@ export default async function runExecutor(
 	const testsPath = path.join(context.root, buildDir, 'test-esm-cjs');
 	mkdirSync(testsPath, { recursive: true });
 
+	const testESMPath = path.join(testsPath, 'test-esm.mjs');
 	writeFileSync(
-		path.join(testsPath, 'test-esm.mjs'),
+		testESMPath,
 		`import * as result from '../../${options.outputPath}/index.js';`
 	);
-	writeFileSync(
-		path.join(testsPath, 'test-cjs.cjs'),
-		`require('../../${options.outputPath}');`
-	);
-	const checkForSuccess = (scriptName) =>
+	const testCJSPath = path.join(testsPath, 'test-cjs.cjs');
+	writeFileSync(testCJSPath, `require('../../${options.outputPath}');`);
+	const checkForSuccess = (scriptPath) =>
 		new Promise((resolve, reject) => {
-			const test = spawn('sh', ['node', scriptName], {
+			const test = spawn('node', [scriptPath], {
 				cwd: testsPath,
 				stdio: 'pipe',
 			});
@@ -49,7 +48,7 @@ export default async function runExecutor(
 			});
 		});
 
-	await checkForSuccess('test-esm.mjs');
-	await checkForSuccess('test-cjs.cjs');
+	await checkForSuccess(testESMPath);
+	await checkForSuccess(testCJSPath);
 	return { success: true };
 }

packages/php-wasm/progress/src/lib/emscripten-download-monitor.ts

@@ -173,6 +173,10 @@ export interface DownloadProgress {
 	total: number;
 }
 
+export function parseContentLength(contentLength: string | null) {
+	return parseInt(contentLength || '', 10) || FALLBACK_FILE_SIZE;
+}
+
 /**
  * Clones a fetch Response object and returns a version
  * that calls the `onProgress` callback as the #progress
@@ -186,8 +190,7 @@ export function cloneResponseMonitorProgress(
 	response: Response,
 	onProgress: (event: CustomEvent<DownloadProgress>) => void
 ): Response {
-	const contentLength = response.headers.get('content-length') || '';
-	const total = parseInt(contentLength, 10) || FALLBACK_FILE_SIZE;
+	const total = parseContentLength(response.headers.get('content-length'));
 
 	function notify(loaded: number, total: number) {
 		onProgress(
@@ -200,37 +203,19 @@ export function cloneResponseMonitorProgress(
 		);
 	}
 
+	// We could just tee() the response body here, but the following
+	// Chromium issue crashes the browser tab and prevents us from doing so:
+	// https://bugs.chromium.org/p/chromium/issues/detail?id=1512548
+	let seenBytes = 0;
 	return new Response(
-		new ReadableStream({
-			async start(controller) {
-				if (!response.body) {
-					controller.close();
-					return;
-				}
-				const reader = response.body.getReader();
-				let loaded = 0;
-				for (;;) {
-					try {
-						const { done, value } = await reader.read();
-						if (value) {
-							loaded += value.byteLength;
-						}
-						if (done) {
-							notify(loaded, loaded);
-							controller.close();
-							break;
-						} else {
-							notify(loaded, total);
-							controller.enqueue(value);
-						}
-					} catch (e) {
-						console.error({ e });
-						controller.error(e);
-						break;
-					}
-				}
+		peekByobStream(
+			response.body!,
+			(bytes) => {
+				seenBytes += bytes.byteLength;
+				notify(seenBytes, total);
 			},
-		}),
+			() => notify(total, total)
+		),
 		{
 			status: response.status,
 			statusText: response.statusText,
@@ -240,3 +225,39 @@ export function cloneResponseMonitorProgress(
 }
 
 export type DownloadProgressCallback = (progress: DownloadProgress) => void;
+
+function peekByobStream(
+	stream: ReadableStream<Uint8Array>,
+	onChunk: (chunk: Uint8Array) => void,
+	onDone: () => void
+) {
+	const reader = stream.getReader({ mode: 'byob' });
+	return new ReadableStream({
+		type: 'bytes',
+		// 0.5 MB seems like a reasonable chunk size, let's adjust
+		// this if needed.
+		autoAllocateChunkSize: 512 * 1024,
+
+		/**
+		 * We could write directly to controller.byobRequest.view
+		 * here. Unfortunately, in Chrome it detaches on the first
+		 * `await` and cannot be reused once we actually have the data.
+		 */
+		async pull(controller) {
+			// Read the next chunk of data:
+			const view = controller.byobRequest!.view!;
+			const uint8array = new Uint8Array(view.byteLength);
+			const { value: chunk, done } = await reader.read(uint8array);
+			if (done) {
+				controller.close();
+				controller.byobRequest?.respond(0);
+				onDone();
+				return;
+			}
+
+			// Emit that chunk:
+			onChunk(chunk);
+			controller.byobRequest?.respondWithNewView(chunk);
+		},
+	});
+}

packages/php-wasm/scopes/package.json

@@ -25,7 +25,7 @@
 	"types": "index.d.ts",
 	"gitHead": "2f8d8f3cea548fbd75111e8659a92f601cddc593",
 	"engines": {
-		"node": ">=16.15.1",
+		"node": ">=18.18.2",
 		"npm": ">=8.11.0"
 	}
 }

packages/php-wasm/stream-compression/src/utils/stream-write-to-php.ts

@@ -0,0 +1,22 @@
+import { UniversalPHP } from '@php-wasm/universal';
+import { dirname, joinPaths } from '@php-wasm/util';
+
+/**
+ * Writes streamed files to PHP filesystem.
+ */
+export function streamWriteToPhp(php: UniversalPHP, root: string) {
+	return new WritableStream({
+		async write(file: File) {
+			const filePath = joinPaths(root, file.name);
+			if (file.type === 'directory') {
+				await php.mkdir(filePath);
+			} else {
+				await php.mkdir(dirname(filePath));
+				await php.writeFile(
+					filePath,
+					new Uint8Array(await file.arrayBuffer())
+				);
+			}
+		},
+	});
+}

packages/php-wasm/universal/src/lib/index.ts

@@ -17,6 +17,9 @@ export type {
 	SpawnHandler,
 } from './universal-php';
 
+export type { IteratePhpFilesOptions as IterateFilesOptions } from './iterate-files';
+export { iteratePhpFiles as iterateFiles } from './iterate-files';
+
 export { UnhandledRejectionsTarget } from './wasm-error-reporting';
 
 export type { IteratePhpFilesOptions as IterateFilesOptions } from './iterate-files';

packages/php-wasm/web/package.json

@@ -31,7 +31,7 @@
 	"types": "index.d.ts",
 	"gitHead": "2f8d8f3cea548fbd75111e8659a92f601cddc593",
 	"engines": {
-		"node": ">=16.15.1",
+		"node": ">=18.18.2",
 		"npm": ">=8.11.0"
 	}
 }