match next.js client side routing (buildManifest rewrites & devPagesManifest HMR)

crates/next-core/js/src/entry/config/next.js

@@ -1,4 +1,5 @@
 import loadConfig from "next/dist/server/config";
+import loadCustomRoutes from "next/dist/lib/load-custom-routes";
 import { PHASE_DEVELOPMENT_SERVER } from "next/dist/shared/lib/constants";
 import assert from "assert";
 
@@ -13,6 +14,12 @@ const loadNextConfig = async (silent) => {
 
   nextConfig.generateBuildId = await nextConfig.generateBuildId?.();
 
+  const customRoutes = await loadCustomRoutes(nextConfig);
+
+  nextConfig.headers = customRoutes.headers;
+  nextConfig.rewrites = customRoutes.rewrites;
+  nextConfig.redirects = customRoutes.redirects;
+
   // TODO: these functions takes arguments, have to be supported in a different way
   nextConfig.exportPathMap = nextConfig.exportPathMap && {};
   nextConfig.webpack = nextConfig.webpack && {};

crates/next-core/js/src/entry/manifest/buildManifest.js

@@ -0,0 +1,18 @@
+((manifest) => {
+  // adapted from https://github.com/vercel/next.js/blob/canary/packages/next/src/build/webpack/plugins/build-manifest-plugin.ts#L54-L54
+  function processRoute(rewrite) {
+    // omit external rewrite destinations since these aren't
+    // handled client-side
+    if (!rewrite.destination.startsWith("/")) {
+      delete rewrite.destination;
+    }
+    return rewrite;
+  }
+
+  manifest.__rewrites.beforeFiles.map(processRoute);
+  manifest.__rewrites.afterFiles.map(processRoute);
+  manifest.__rewrites.fallback.map(processRoute);
+
+  self.__BUILD_MANIFEST = manifest;
+  self.__BUILD_MANIFEST_CB && self.__BUILD_MANIFEST_CB();
+})($$MANIFEST$$);

crates/next-core/js/src/entry/next-hydrate.tsx

@@ -58,16 +58,6 @@ async function loadPageChunk(assetPrefix: string, chunkPath: string) {
   };
 
   const pagePath = window.__NEXT_DATA__.page;
-  window.__BUILD_MANIFEST = {
-    [pagePath]: [],
-    __rewrites: {
-      beforeFiles: [],
-      afterFiles: [],
-      fallback: [],
-    } as any,
-    sortedPages: [pagePath, "/_app"],
-  };
-
   window.__NEXT_P.push(["/_app", () => _app]);
   window.__NEXT_P.push([pagePath, () => page]);
 
@@ -79,10 +69,34 @@ async function loadPageChunk(assetPrefix: string, chunkPath: string) {
   // during hydration. To make this dependency clearer, we pass `router` as an
   // explicit argument instead of relying on the `router` import binding.
   subscribeToCurrentPageData({ assetPrefix, router });
+  subscribeToPageManifest({ assetPrefix });
 
   console.debug("The page has been hydrated");
 })().catch((err) => console.error(err));
 
+function subscribeToPageManifest({ assetPrefix }: { assetPrefix: string }) {
+  // adapted from https://github.com/vercel/next.js/blob/836ac9cc7f290e95b564a61341fa95a5f4f0327e/packages/next/src/client/next-dev.ts#L57
+  subscribeToUpdate(
+    {
+      path: "_next/static/development/_devPagesManifest.json",
+    },
+    (update) => {
+      if (["restart", "partial"].includes(update.type)) {
+        return;
+      }
+
+      fetch(`${assetPrefix}/_next/static/development/_devPagesManifest.json`)
+        .then((res) => res.json())
+        .then((manifest) => {
+          window.__DEV_PAGES_MANIFEST = manifest;
+        })
+        .catch((err) => {
+          console.log(`Failed to fetch devPagesManifest`, err);
+        });
+    }
+  );
+}
+
 /**
  * Subscribes to the current page's data updates from the HMR server.
  *

crates/next-core/js/src/entry/router.ts

@@ -2,8 +2,9 @@ import type { Ipc } from "@vercel/turbopack-next/ipc/index";
 import type { IncomingMessage, ServerResponse } from "node:http";
 import { Buffer } from "node:buffer";
 import { createServer, makeRequest } from "@vercel/turbopack-next/ipc/server";
-import loadNextConfig from "@vercel/turbopack-next/entry/config/next";
 import { makeResolver } from "next/dist/server/router.js";
+import loadConfig from "next/dist/server/config";
+import { PHASE_DEVELOPMENT_SERVER } from "next/dist/shared/lib/constants";
 
 import "next/dist/server/node-polyfill-fetch.js";
 
@@ -48,10 +49,22 @@ type MiddlewareHeadersResponse = {
 };
 
 let resolveRouteMemo: Promise<
-  (req: IncomingMessage, res: ServerResponse) => Promise<unknown>
+  (req: IncomingMessage, res: ServerResponse) => Promise<void>
 >;
-async function getResolveRoute(dir: string) {
-  const nextConfig = await loadNextConfig(true);
+
+async function getResolveRoute(
+  dir: string
+): ReturnType<
+  typeof import("next/dist/server/lib/route-resolver").makeResolver
+> {
+  const nextConfig = await loadConfig(
+    PHASE_DEVELOPMENT_SERVER,
+    process.cwd(),
+    undefined,
+    undefined,
+    true
+  );
+
   return await makeResolver(dir, nextConfig);
 }
 
@@ -113,7 +126,7 @@ export default async function route(
 async function handleClientResponse(
   _ipc: Ipc<RouterRequest, IpcOutgoingMessage>,
   clientResponse: IncomingMessage
-): Promise<MessageData | void> {
+): Promise<MessageData> {
   if (clientResponse.headers["x-nextjs-route-result"] === "1") {
     clientResponse.setEncoding("utf8");
     // We're either a redirect or a rewrite

crates/next-core/js/src/internal/page-server-handler.tsx

@@ -119,7 +119,7 @@ export default function startHandler({
       devFiles: [],
       ampDevFiles: [],
       polyfillFiles: [],
-      lowPriorityFiles: [],
+      lowPriorityFiles: ["static/development/_buildManifest.js"],
       rootMainFiles: [],
       ampFirstPages: [],
     };

crates/next-core/src/manifest.rs

@@ -1,6 +1,11 @@
-use anyhow::Result;
-use mime::APPLICATION_JSON;
-use turbo_tasks::{primitives::StringsVc, TryFlatMapRecursiveJoinIterExt, TryJoinIterExt};
+use anyhow::{Context, Result};
+use indexmap::IndexMap;
+use mime::{APPLICATION_JAVASCRIPT_UTF_8, APPLICATION_JSON};
+use serde::Serialize;
+use turbo_tasks::{
+    primitives::{StringVc, StringsVc},
+    TryFlatMapRecursiveJoinIterExt, TryJoinIterExt,
+};
 use turbo_tasks_fs::File;
 use turbopack_core::asset::AssetContentVc;
 use turbopack_dev_server::source::{
@@ -11,15 +16,23 @@ use turbopack_node::render::{
     node_api_source::NodeApiContentSourceVc, rendered_source::NodeRenderContentSourceVc,
 };
 
+use crate::{
+    embed_js::next_js_file,
+    next_config::{NextConfigVc, RewritesReadRef},
+    util::get_asset_path_from_route,
+};
+
 /// A content source which creates the next.js `_devPagesManifest.json` and
 /// `_devMiddlewareManifest.json` which are used for client side navigation.
 #[turbo_tasks::value(shared)]
 pub struct DevManifestContentSource {
     pub page_roots: Vec<ContentSourceVc>,
+    pub next_config: NextConfigVc,
 }
 
 #[turbo_tasks::value_impl]
 impl DevManifestContentSourceVc {
+    /// Recursively find all routes in the `page_roots` content sources.
     #[turbo_tasks::function]
     async fn find_routes(self) -> Result<StringsVc> {
         let this = &*self.await?;
@@ -61,10 +74,73 @@ impl DevManifestContentSourceVc {
             .flatten()
             .collect::<Vec<_>>();
 
-        routes.sort();
+        routes.sort_by_cached_key(|s| s.split('/').map(PageSortKey::from).collect::<Vec<_>>());
 
         Ok(StringsVc::cell(routes))
     }
+
+    /// Recursively find all pages in the `page_roots` content sources
+    /// (excluding api routes).
+    #[turbo_tasks::function]
+    async fn find_pages(self) -> Result<StringsVc> {
+        let routes = &*self.find_routes().await?;
+
+        // we don't need to sort as it's already sorted by `find_routes`
+        let pages = routes
+            .iter()
+            .filter(|s| !s.starts_with("/api"))
+            .cloned()
+            .collect();
+
+        Ok(StringsVc::cell(pages))
+    }
+
+    /// Create a build manifest with all pages.
+    #[turbo_tasks::function]
+    async fn create_build_manifest(self) -> Result<StringVc> {
+        let this = &*self.await?;
+
+        let sorted_pages = &*self.find_pages().await?;
+        let routes = sorted_pages
+            .iter()
+            .map(|p| {
+                (
+                    p,
+                    vec![format!(
+                        "_next/static/chunks/pages/{}",
+                        get_asset_path_from_route(p.split_at(1).1, ".js")
+                    )],
+                )
+            })
+            .collect();
+
+        let manifest = BuildManifest {
+            rewrites: this.next_config.rewrites().await?,
+            sorted_pages,
+            routes,
+        };
+
+        let manifest = next_js_file("entry/manifest/buildManifest.js")
+            .await?
+            .as_content()
+            .context("embedded buildManifest file missing")?
+            .content()
+            .to_str()?
+            .replace("$$MANIFEST$$", &serde_json::to_string(&manifest)?);
+
+        Ok(StringVc::cell(manifest))
+    }
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+struct BuildManifest<'a> {
+    #[serde(rename = "__rewrites")]
+    rewrites: RewritesReadRef,
+    sorted_pages: &'a Vec<String>,
+
+    #[serde(flatten)]
+    routes: IndexMap<&'a String, Vec<String>>,
 }
 
 #[turbo_tasks::value_impl]
@@ -75,25 +151,53 @@ impl ContentSource for DevManifestContentSource {
         path: &str,
         _data: turbo_tasks::Value<ContentSourceData>,
     ) -> Result<ContentSourceResultVc> {
-        let manifest_content = match path {
+        let manifest_file = match path {
             "_next/static/development/_devPagesManifest.json" => {
                 let pages = &*self_vc.find_routes().await?;
 
-                serde_json::to_string(&serde_json::json!({
+                File::from(serde_json::to_string(&serde_json::json!({
                     "pages": pages,
-                }))?
+                }))?)
+                .with_content_type(APPLICATION_JSON)
+            }
+            "_next/static/development/_buildManifest.js" => {
+                let build_manifest = &*self_vc.create_build_manifest().await?;
+
+                File::from(build_manifest.as_str()).with_content_type(APPLICATION_JAVASCRIPT_UTF_8)
             }
             "_next/static/development/_devMiddlewareManifest.json" => {
                 // empty middleware manifest
-                "[]".to_string()
+                File::from("[]").with_content_type(APPLICATION_JSON)
             }
             _ => return Ok(ContentSourceResultVc::not_found()),
         };
 
-        let file = File::from(manifest_content).with_content_type(APPLICATION_JSON);
-
         Ok(ContentSourceResultVc::exact(
-            ContentSourceContentVc::static_content(AssetContentVc::from(file).into()).into(),
+            ContentSourceContentVc::static_content(AssetContentVc::from(manifest_file).into())
+                .into(),
         ))
     }
 }
+
+/// PageSortKey is necessary because the next.js client code looks for matches
+/// in the order the pages are sent in the manifest,if they're sorted
+/// alphabetically this means \[slug] and \[\[catchall]] routes are prioritized
+/// over fixed paths, so we have to override the ordering with this.
+#[derive(Ord, PartialOrd, Eq, PartialEq)]
+enum PageSortKey {
+    Static(String),
+    Slug,
+    CatchAll,
+}
+
+impl From<&str> for PageSortKey {
+    fn from(value: &str) -> Self {
+        if value.starts_with("[[") && value.ends_with("]]") {
+            PageSortKey::CatchAll
+        } else if value.starts_with('[') && value.ends_with(']') {
+            PageSortKey::Slug
+        } else {
+            PageSortKey::Static(value.to_string())
+        }
+    }
+}