Add experimental support for history.pushState and history.replaceState (#58335)

## What?

This PR introduces support for manually calling `history.pushState` and `history.replaceState`.

It's currently under an experimental flag:

```js
/**
 * @type {import('next').NextConfig}
 */
const nextConfig = {
  experimental: {
    windowHistorySupport: true,
  },
}

module.exports = nextConfig
```

Going forward I'll refer to `history.pushState` as `replaceState` is interchangable.
 
When the flag is enabled you're able to call the web platform `history.pushState` in the usual way:

```js
const data = {
	foo: 'bar'
}
const url = '/my-new-url?search=tim'
window.history.pushState(data, '', url)
```

Let's start by explaining what would happen without the flag:

When a new history entry is pushed outside of the Next.js router any back navigation to that history entry will cause a browser reload as it can no longer be used by Next.js as the required metadata for the router is missing. In practice this makes it so that pushState/replaceState is not feasible to be used. Any pathname / searchParams added can't be observed by `usePathname` / `useSearchParams` either.

With the flag enabled the pushState/replaceState calls are instrumented and is synced into the Next.js router. This way the Next.js router's internal metadata is preserved, making back navigations apply still, and pathname / searchParams is synced as well, making sure that you can observe it using `usePathname` and `useSearchParams`.

## How?

- Added a new experimental flag `windowHistorySupport`
- Instruments `history.pushState` and `history.replaceState`
	- Triggers the same action as popstate (ACTION_RESTORE) to sync the provided url (if provided) into the Next.js router
	- Copies the Next.js values kept in history.state so that they are not lost
	- Calls the original pushState/replaceState

~~Something to figure out is how we handle additional pushes/replaces in Next.js as that should override the history state that was previously set.~~
Went with this after discussing with @sebmarkbage:
- When you open a page it preserves the custom history state
	- This is to solve this case: when you manually `window.history.pushState` / `window.history.replaceState` and then do an mpa navigation (i.e. `<a>` or `window.location.href`) and the navigate backwards the custom history state is preserved
- When you navigate back and forward (popstate) it preserves the custom history state
- When you navigate client-side (i.e. `router.push()` / `<Link>`) the custom history state is not preserved

Author: timneutkens

packages/next/src/build/webpack/plugins/define-env-plugin.ts

@@ -95,6 +95,9 @@ export function getDefineEnv({
       isEdgeServer ? 'edge' : isNodeServer ? 'nodejs' : ''
     ),
     'process.env.NEXT_MINIMAL': JSON.stringify(''),
+    'process.env.__NEXT_WINDOW_HISTORY_SUPPORT': JSON.stringify(
+      config.experimental.windowHistorySupport
+    ),
     'process.env.__NEXT_ACTIONS_DEPLOYMENT_ID': JSON.stringify(
       config.experimental.useDeploymentIdServerActions
     ),

packages/next/src/client/components/app-router.tsx

@@ -35,6 +35,7 @@ import {
   PrefetchKind,
 } from './router-reducer/router-reducer-types'
 import type {
+  PushRef,
   ReducerActions,
   RouterChangeByServerResponse,
   RouterNavigate,
@@ -108,24 +109,44 @@ function isExternalURL(url: URL) {
   return url.origin !== window.location.origin
 }
 
-function HistoryUpdater({ tree, pushRef, canonicalUrl, sync }: any) {
+function HistoryUpdater({
+  tree,
+  pushRef,
+  canonicalUrl,
+  sync,
+}: {
+  tree: FlightRouterState
+  pushRef: PushRef
+  canonicalUrl: string
+  sync: () => void
+}) {
   useInsertionEffect(() => {
-    // Identifier is shortened intentionally.
-    // __NA is used to identify if the history entry can be handled by the app-router.
-    // __N is used to identify if the history entry can be handled by the old router.
     const historyState = {
+      ...(process.env.__NEXT_WINDOW_HISTORY_SUPPORT &&
+      pushRef.preserveCustomHistoryState
+        ? window.history.state
+        : {}),
+      // Identifier is shortened intentionally.
+      // __NA is used to identify if the history entry can be handled by the app-router.
+      // __N is used to identify if the history entry can be handled by the old router.
       __NA: true,
-      tree,
+      __PRIVATE_NEXTJS_INTERNALS_TREE: tree,
     }
     if (
       pushRef.pendingPush &&
+      // Skip pushing an additional history entry if the canonicalUrl is the same as the current url.
+      // This mirrors the browser behavior for normal navigation.
       createHrefFromUrl(new URL(window.location.href)) !== canonicalUrl
     ) {
       // This intentionally mutates React state, pushRef is overwritten to ensure additional push/replace calls do not trigger an additional history entry.
       pushRef.pendingPush = false
-      window.history.pushState(historyState, '', canonicalUrl)
+      if (originalPushState) {
+        originalPushState(historyState, '', canonicalUrl)
+      }
     } else {
-      window.history.replaceState(historyState, '', canonicalUrl)
+      if (originalReplaceState) {
+        originalReplaceState(historyState, '', canonicalUrl)
+      }
     }
     sync()
   }, [tree, pushRef, canonicalUrl, sync])
@@ -204,6 +225,28 @@ function useNavigate(dispatch: React.Dispatch<ReducerActions>): RouterNavigate {
   )
 }
 
+const originalPushState =
+  typeof window !== 'undefined'
+    ? window.history.pushState.bind(window.history)
+    : null
+const originalReplaceState =
+  typeof window !== 'undefined'
+    ? window.history.replaceState.bind(window.history)
+    : null
+
+function copyNextJsInternalHistoryState(data: any) {
+  const currentState = window.history.state
+  const __NA = currentState?.__NA
+  if (__NA) {
+    data.__NA = __NA
+  }
+  const __PRIVATE_NEXTJS_INTERNALS_TREE =
+    currentState?.__PRIVATE_NEXTJS_INTERNALS_TREE
+  if (__PRIVATE_NEXTJS_INTERNALS_TREE) {
+    data.__PRIVATE_NEXTJS_INTERNALS_TREE = __PRIVATE_NEXTJS_INTERNALS_TREE
+  }
+}
+
 /**
  * The global router that wraps the application components.
  */
@@ -371,12 +414,16 @@ function Router({
     // would trigger the mpa navigation logic again from the lines below.
     // This will restore the router to the initial state in the event that the app is restored from bfcache.
     function handlePageShow(event: PageTransitionEvent) {
-      if (!event.persisted || !window.history.state?.tree) return
+      if (
+        !event.persisted ||
+        !window.history.state?.__PRIVATE_NEXTJS_INTERNALS_TREE
+      )
+        return
 
       dispatch({
         type: ACTION_RESTORE,
         url: new URL(window.location.href),
-        tree: window.history.state.tree,
+        tree: window.history.state.__PRIVATE_NEXTJS_INTERNALS_TREE,
       })
     }
 
@@ -416,13 +463,66 @@ function Router({
     use(createInfinitePromise())
   }
 
-  /**
-   * Handle popstate event, this is used to handle back/forward in the browser.
-   * By default dispatches ACTION_RESTORE, however if the history entry was not pushed/replaced by app-router it will reload the page.
-   * That case can happen when the old router injected the history entry.
-   */
-  const onPopState = useCallback(
-    ({ state }: PopStateEvent) => {
+  useEffect(() => {
+    if (process.env.__NEXT_WINDOW_HISTORY_SUPPORT) {
+      // Ensure the canonical URL in the Next.js Router is updated when the URL is changed so that `usePathname` and `useSearchParams` hold the pushed values.
+      const applyUrlFromHistoryPushReplace = (
+        url: string | URL | null | undefined
+      ) => {
+        startTransition(() => {
+          dispatch({
+            type: ACTION_RESTORE,
+            url: new URL(url ?? window.location.href),
+            tree: window.history.state.__PRIVATE_NEXTJS_INTERNALS_TREE,
+          })
+        })
+      }
+
+      if (originalPushState) {
+        /**
+         * Patch pushState to ensure external changes to the history are reflected in the Next.js Router.
+         * Ensures Next.js internal history state is copied to the new history entry.
+         * Ensures usePathname and useSearchParams hold the newly provided url.
+         */
+        window.history.pushState = function pushState(
+          data: any,
+          _unused: string,
+          url?: string | URL | null
+        ): void {
+          copyNextJsInternalHistoryState(data)
+
+          applyUrlFromHistoryPushReplace(url)
+
+          return originalPushState(data, _unused, url)
+        }
+      }
+      if (originalReplaceState) {
+        /**
+         * Patch replaceState to ensure external changes to the history are reflected in the Next.js Router.
+         * Ensures Next.js internal history state is copied to the new history entry.
+         * Ensures usePathname and useSearchParams hold the newly provided url.
+         */
+        window.history.replaceState = function replaceState(
+          data: any,
+          _unused: string,
+          url?: string | URL | null
+        ): void {
+          copyNextJsInternalHistoryState(data)
+
+          if (url) {
+            applyUrlFromHistoryPushReplace(url)
+          }
+          return originalReplaceState(data, _unused, url)
+        }
+      }
+    }
+
+    /**
+     * Handle popstate event, this is used to handle back/forward in the browser.
+     * By default dispatches ACTION_RESTORE, however if the history entry was not pushed/replaced by app-router it will reload the page.
+     * That case can happen when the old router injected the history entry.
+     */
+    const onPopState = ({ state }: PopStateEvent) => {
       if (!state) {
         // TODO-APP: this case only happens when pushState/replaceState was called outside of Next.js. It should probably reload the page in this case.
         return
@@ -441,20 +541,23 @@ function Router({
         dispatch({
           type: ACTION_RESTORE,
           url: new URL(window.location.href),
-          tree: state.tree,
+          tree: state.__PRIVATE_NEXTJS_INTERNALS_TREE,
         })
       })
-    },
-    [dispatch]
-  )
+    }
 
-  // Register popstate event to call onPopstate.
-  useEffect(() => {
+    // Register popstate event to call onPopstate.
     window.addEventListener('popstate', onPopState)
     return () => {
+      if (originalPushState) {
+        window.history.pushState = originalPushState
+      }
+      if (originalReplaceState) {
+        window.history.replaceState = originalReplaceState
+      }
       window.removeEventListener('popstate', onPopState)
     }
-  }, [onPopState])
+  }, [dispatch])
 
   const { cache, tree, nextUrl, focusAndScrollRef } =
     useUnwrapState(reducerState)

packages/next/src/client/components/router-reducer/create-initial-router-state.test.tsx

@@ -98,7 +98,11 @@ describe('createInitialRouterState', () => {
       tree: initialTree,
       canonicalUrl: initialCanonicalUrl,
       prefetchCache: new Map(),
-      pushRef: { pendingPush: false, mpaNavigation: false },
+      pushRef: {
+        pendingPush: false,
+        mpaNavigation: false,
+        preserveCustomHistoryState: true,
+      },
       focusAndScrollRef: {
         apply: false,
         onlyHashChange: false,

packages/next/src/client/components/router-reducer/create-initial-router-state.ts

@@ -46,7 +46,13 @@ export function createInitialRouterState({
     tree: initialTree,
     cache,
     prefetchCache: new Map(),
-    pushRef: { pendingPush: false, mpaNavigation: false },
+    pushRef: {
+      pendingPush: false,
+      mpaNavigation: false,
+      // First render needs to preserve the previous window.history.state
+      // to avoid it being overwritten on navigation back/forward with MPA Navigation.
+      preserveCustomHistoryState: true,
+    },
     focusAndScrollRef: {
       apply: false,
       onlyHashChange: false,

packages/next/src/client/components/router-reducer/handle-mutable.ts

@@ -5,6 +5,10 @@ import type {
   ReducerState,
 } from './router-reducer-types'
 
+function isNotUndefined<T>(value: T): value is Exclude<T, undefined> {
+  return typeof value !== 'undefined'
+}
+
 export function handleMutable(
   state: ReadonlyReducerState,
   mutable: Mutable
@@ -15,26 +19,28 @@ export function handleMutable(
   return {
     buildId: state.buildId,
     // Set href.
-    canonicalUrl:
-      mutable.canonicalUrl != null
-        ? mutable.canonicalUrl === state.canonicalUrl
-          ? state.canonicalUrl
-          : mutable.canonicalUrl
-        : state.canonicalUrl,
+    canonicalUrl: isNotUndefined(mutable.canonicalUrl)
+      ? mutable.canonicalUrl === state.canonicalUrl
+        ? state.canonicalUrl
+        : mutable.canonicalUrl
+      : state.canonicalUrl,
     pushRef: {
-      pendingPush:
-        mutable.pendingPush != null
-          ? mutable.pendingPush
-          : state.pushRef.pendingPush,
-      mpaNavigation:
-        mutable.mpaNavigation != null
-          ? mutable.mpaNavigation
-          : state.pushRef.mpaNavigation,
+      pendingPush: isNotUndefined(mutable.pendingPush)
+        ? mutable.pendingPush
+        : state.pushRef.pendingPush,
+      mpaNavigation: isNotUndefined(mutable.mpaNavigation)
+        ? mutable.mpaNavigation
+        : state.pushRef.mpaNavigation,
+      preserveCustomHistoryState: isNotUndefined(
+        mutable.preserveCustomHistoryState
+      )
+        ? mutable.preserveCustomHistoryState
+        : state.pushRef.preserveCustomHistoryState,
     },
     // All navigation requires scroll and focus management to trigger.
     focusAndScrollRef: {
       apply: shouldScroll
-        ? mutable?.scrollableSegments !== undefined
+        ? isNotUndefined(mutable?.scrollableSegments)
           ? true
           : state.focusAndScrollRef.apply
         : // If shouldScroll is false then we should not apply scroll and focus management.
@@ -63,11 +69,12 @@ export function handleMutable(
       ? mutable.prefetchCache
       : state.prefetchCache,
     // Apply patched router state.
-    tree: mutable.patchedTree !== undefined ? mutable.patchedTree : state.tree,
-    nextUrl:
-      mutable.patchedTree !== undefined
-        ? computeChangedPath(state.tree, mutable.patchedTree) ??
-          state.canonicalUrl
-        : state.nextUrl,
+    tree: isNotUndefined(mutable.patchedTree)
+      ? mutable.patchedTree
+      : state.tree,
+    nextUrl: isNotUndefined(mutable.patchedTree)
+      ? computeChangedPath(state.tree, mutable.patchedTree) ??
+        state.canonicalUrl
+      : state.nextUrl,
   }
 }

packages/next/src/client/components/router-reducer/reducers/fast-refresh-reducer.ts

@@ -26,6 +26,8 @@ function fastRefreshReducerImpl(
     return handleMutable(state, mutable)
   }
 
+  mutable.preserveCustomHistoryState = false
+
   if (!cache.data) {
     // TODO-APP: verify that `href` is not an external url.
     // Fetch data from the root of the tree.