avoid sending modules in parent for dynamic imports (vercel/turborepo#4056)

### Description

Keeps track of available modules when importing new chunk groups. Omits
assets that are already available in the parent chunk group(s) when
determining assets to be included in chunks.

### Testing Instructions

Dynamic imports should not include modules that are already in the
parent chunks

<!--
  When the below is checked (default) our PR bot will automatically
  assign labels to your PR based on the content to help the team
  organize and review it faster.
-->

- [x] Auto label
fixes WEB-599

Author: sokra

crates/next-core/js/src/entry/app/server-to-client-ssr.tsx

@@ -1,6 +1,6 @@
 import { createProxy } from "next/dist/build/webpack/loaders/next-flight-loader/module-proxy";
 
-("TURBOPACK { chunking-type: parallel }");
+("TURBOPACK { chunking-type: isolatedParallel }");
 // @ts-expect-error CLIENT_MODULE is provided by rust
 import { __turbopack_module_id__ as id } from "CLIENT_MODULE";
 

crates/next-core/src/app_source.rs

@@ -593,7 +593,7 @@ impl AppRendererVc {
 
         for (_, imports) in segments.iter() {
             for (p, identifier, chunks_identifier) in imports.values() {
-                result += r#"("TURBOPACK { transition: next-layout-entry; chunking-type: parallel }");
+                result += r#"("TURBOPACK { transition: next-layout-entry; chunking-type: isolatedParallel }");
 "#;
                 writeln!(
                     result,

crates/next-core/src/next_client_chunks/in_chunking_context_asset.rs

@@ -1,12 +1,15 @@
 use anyhow::Result;
-use turbo_tasks::primitives::StringVc;
+use turbo_tasks::{primitives::StringVc, Value};
 use turbopack::ecmascript::chunk::{
     EcmascriptChunkItemVc, EcmascriptChunkPlaceable, EcmascriptChunkPlaceableVc, EcmascriptChunkVc,
     EcmascriptExportsVc,
 };
 use turbopack_core::{
     asset::{Asset, AssetContentVc, AssetVc},
-    chunk::{ChunkVc, ChunkableAsset, ChunkableAssetVc, ChunkingContextVc},
+    chunk::{
+        availability_info::AvailabilityInfo, ChunkVc, ChunkableAsset, ChunkableAssetVc,
+        ChunkingContextVc,
+    },
     ident::AssetIdentVc,
     reference::AssetReferencesVc,
 };
@@ -43,8 +46,12 @@ impl Asset for InChunkingContextAsset {
 #[turbo_tasks::value_impl]
 impl ChunkableAsset for InChunkingContextAsset {
     #[turbo_tasks::function]
-    fn as_chunk(&self, _context: ChunkingContextVc) -> ChunkVc {
-        EcmascriptChunkVc::new(self.chunking_context, self.asset).into()
+    fn as_chunk(
+        &self,
+        _context: ChunkingContextVc,
+        availability_info: Value<AvailabilityInfo>,
+    ) -> ChunkVc {
+        EcmascriptChunkVc::new(self.chunking_context, self.asset, availability_info).into()
     }
 }
 

crates/next-core/src/next_client_chunks/with_chunks.rs

@@ -1,7 +1,6 @@
 use anyhow::{bail, Result};
 use indoc::formatdoc;
-use serde_json::Value;
-use turbo_tasks::{primitives::StringVc, TryJoinIterExt, ValueToString, ValueToStringVc};
+use turbo_tasks::{primitives::StringVc, TryJoinIterExt, Value};
 use turbo_tasks_fs::FileSystemPathVc;
 use turbopack::ecmascript::{
     chunk::{
@@ -14,18 +13,15 @@ use turbopack::ecmascript::{
 use turbopack_core::{
     asset::{Asset, AssetContentVc, AssetVc},
     chunk::{
-        Chunk, ChunkGroupVc, ChunkItem, ChunkItemVc, ChunkListReferenceVc, ChunkVc, ChunkableAsset,
-        ChunkableAssetReference, ChunkableAssetReferenceVc, ChunkableAssetVc, ChunkingContext,
-        ChunkingContextVc, ChunkingType, ChunkingTypeOptionVc,
+        availability_info::AvailabilityInfo, Chunk, ChunkGroupReferenceVc, ChunkGroupVc, ChunkItem,
+        ChunkItemVc, ChunkListReferenceVc, ChunkVc, ChunkableAsset, ChunkableAssetVc,
+        ChunkingContextVc,
     },
     ident::AssetIdentVc,
-    reference::{AssetReference, AssetReferenceVc, AssetReferencesVc},
-    resolve::{ResolveResult, ResolveResultVc},
+    reference::AssetReferencesVc,
 };
 use turbopack_ecmascript::utils::stringify_js_pretty;
 
-use super::in_chunking_context_asset::InChunkingContextAsset;
-
 #[turbo_tasks::function]
 fn modifier() -> StringVc {
     StringVc::cell("chunks".to_string())
@@ -51,16 +47,30 @@ impl Asset for WithChunksAsset {
     }
 
     #[turbo_tasks::function]
-    fn references(&self) -> AssetReferencesVc {
-        unimplemented!()
+    async fn references(self_vc: WithChunksAssetVc) -> Result<AssetReferencesVc> {
+        let this = self_vc.await?;
+        let chunk_group = self_vc.chunk_group();
+        Ok(AssetReferencesVc::cell(vec![
+            ChunkGroupReferenceVc::new(chunk_group).into(),
+            ChunkListReferenceVc::new(this.server_root, chunk_group).into(),
+        ]))
     }
 }
 
 #[turbo_tasks::value_impl]
 impl ChunkableAsset for WithChunksAsset {
     #[turbo_tasks::function]
-    fn as_chunk(self_vc: WithChunksAssetVc, context: ChunkingContextVc) -> ChunkVc {
-        EcmascriptChunkVc::new(context, self_vc.as_ecmascript_chunk_placeable()).into()
+    fn as_chunk(
+        self_vc: WithChunksAssetVc,
+        context: ChunkingContextVc,
+        availability_info: Value<AvailabilityInfo>,
+    ) -> ChunkVc {
+        EcmascriptChunkVc::new(
+            context,
+            self_vc.as_ecmascript_chunk_placeable(),
+            availability_info,
+        )
+        .into()
     }
 }
 
@@ -71,10 +81,8 @@ impl EcmascriptChunkPlaceable for WithChunksAsset {
         self_vc: WithChunksAssetVc,
         context: ChunkingContextVc,
     ) -> Result<EcmascriptChunkItemVc> {
-        let this = self_vc.await?;
         Ok(WithChunksChunkItem {
             context,
-            inner_context: this.chunking_context,
             inner: self_vc,
         }
         .cell()
@@ -88,32 +96,27 @@ impl EcmascriptChunkPlaceable for WithChunksAsset {
     }
 }
 
-#[turbo_tasks::value]
-struct WithChunksChunkItem {
-    context: ChunkingContextVc,
-    inner_context: ChunkingContextVc,
-    inner: WithChunksAssetVc,
-}
-
 #[turbo_tasks::value_impl]
-impl WithChunksChunkItemVc {
-    #[turbo_tasks::function]
-    async fn chunk_list_path(self) -> Result<FileSystemPathVc> {
-        let this = self.await?;
-        Ok(this.inner_context.chunk_list_path(this.inner.ident()))
-    }
-
+impl WithChunksAssetVc {
     #[turbo_tasks::function]
     async fn chunk_group(self) -> Result<ChunkGroupVc> {
         let this = self.await?;
-        let inner = this.inner.await?;
         Ok(ChunkGroupVc::from_asset(
-            inner.asset.into(),
-            this.inner_context,
+            this.asset.into(),
+            this.chunking_context,
+            Value::new(AvailabilityInfo::Root {
+                current_availability_root: this.asset.into(),
+            }),
         ))
     }
 }
 
+#[turbo_tasks::value]
+struct WithChunksChunkItem {
+    context: ChunkingContextVc,
+    inner: WithChunksAssetVc,
+}
+
 #[turbo_tasks::value_impl]
 impl EcmascriptChunkItem for WithChunksChunkItem {
     #[turbo_tasks::function]
@@ -122,15 +125,14 @@ impl EcmascriptChunkItem for WithChunksChunkItem {
     }
 
     #[turbo_tasks::function]
-    async fn content(self_vc: WithChunksChunkItemVc) -> Result<EcmascriptChunkItemContentVc> {
-        let this = self_vc.await?;
-        let inner = this.inner.await?;
-        let group = self_vc.chunk_group();
+    async fn content(&self) -> Result<EcmascriptChunkItemContentVc> {
+        let inner = self.inner.await?;
+        let group = self.inner.chunk_group();
         let chunks = group.chunks().await?;
         let server_root = inner.server_root.await?;
         let mut client_chunks = Vec::new();
 
-        let chunk_list_path = self_vc.chunk_list_path().await?;
+        let chunk_list_path = group.chunk_list_path().await?;
         let chunk_list_path = if let Some(path) = server_root.get_path_to(&chunk_list_path) {
             path
         } else {
@@ -139,10 +141,16 @@ impl EcmascriptChunkItem for WithChunksChunkItem {
 
         for chunk_path in chunks.iter().map(|c| c.path()).try_join().await? {
             if let Some(path) = server_root.get_path_to(&chunk_path) {
-                client_chunks.push(Value::String(path.to_string()));
+                client_chunks.push(serde_json::Value::String(path.to_string()));
             }
         }
-        let module_id = stringify_js(&*inner.asset.as_chunk_item(this.inner_context).id().await?);
+        let module_id = stringify_js(
+            &*inner
+                .asset
+                .as_chunk_item(inner.chunking_context)
+                .id()
+                .await?,
+        );
         Ok(EcmascriptChunkItemContent {
             inner_code: formatdoc! {
                 r#"
@@ -171,58 +179,7 @@ impl ChunkItem for WithChunksChunkItem {
     }
 
     #[turbo_tasks::function]
-    async fn references(self_vc: WithChunksChunkItemVc) -> Result<AssetReferencesVc> {
-        let this = self_vc.await?;
-        let inner = this.inner.await?;
-        Ok(AssetReferencesVc::cell(vec![
-            WithChunksAssetReference {
-                asset: InChunkingContextAsset {
-                    asset: inner.asset,
-                    chunking_context: this.inner_context,
-                }
-                .cell()
-                .into(),
-            }
-            .cell()
-            .into(),
-            ChunkListReferenceVc::new(
-                inner.server_root,
-                self_vc.chunk_group(),
-                self_vc.chunk_list_path(),
-            )
-            .into(),
-        ]))
-    }
-}
-
-#[turbo_tasks::value]
-struct WithChunksAssetReference {
-    asset: AssetVc,
-}
-
-#[turbo_tasks::value_impl]
-impl ValueToString for WithChunksAssetReference {
-    #[turbo_tasks::function]
-    async fn to_string(&self) -> Result<StringVc> {
-        Ok(StringVc::cell(format!(
-            "referenced asset {}",
-            self.asset.ident().to_string().await?
-        )))
-    }
-}
-
-#[turbo_tasks::value_impl]
-impl AssetReference for WithChunksAssetReference {
-    #[turbo_tasks::function]
-    fn resolve_reference(&self) -> ResolveResultVc {
-        ResolveResult::asset(self.asset).cell()
-    }
-}
-
-#[turbo_tasks::value_impl]
-impl ChunkableAssetReference for WithChunksAssetReference {
-    #[turbo_tasks::function]
-    fn chunking_type(&self) -> ChunkingTypeOptionVc {
-        ChunkingTypeOptionVc::cell(Some(ChunkingType::Separate))
+    fn references(&self) -> AssetReferencesVc {
+        self.inner.references()
     }
 }

crates/next-core/src/next_client_component/with_chunking_context_scope_asset.rs

@@ -1,12 +1,15 @@
 use anyhow::Result;
-use turbo_tasks::primitives::StringVc;
+use turbo_tasks::{primitives::StringVc, Value};
 use turbopack::ecmascript::chunk::{
     EcmascriptChunkItemVc, EcmascriptChunkPlaceable, EcmascriptChunkPlaceableVc, EcmascriptChunkVc,
     EcmascriptExportsVc,
 };
 use turbopack_core::{
     asset::{Asset, AssetContentVc, AssetVc},
-    chunk::{ChunkVc, ChunkableAsset, ChunkableAssetVc, ChunkingContext, ChunkingContextVc},
+    chunk::{
+        availability_info::AvailabilityInfo, ChunkVc, ChunkableAsset, ChunkableAssetVc,
+        ChunkingContext, ChunkingContextVc,
+    },
     ident::AssetIdentVc,
     reference::AssetReferencesVc,
 };
@@ -43,8 +46,17 @@ impl Asset for WithChunkingContextScopeAsset {
 #[turbo_tasks::value_impl]
 impl ChunkableAsset for WithChunkingContextScopeAsset {
     #[turbo_tasks::function]
-    fn as_chunk(&self, context: ChunkingContextVc) -> ChunkVc {
-        EcmascriptChunkVc::new(context.with_layer(&self.layer), self.asset).into()
+    fn as_chunk(
+        &self,
+        context: ChunkingContextVc,
+        availability_info: Value<AvailabilityInfo>,
+    ) -> ChunkVc {
+        EcmascriptChunkVc::new(
+            context.with_layer(&self.layer),
+            self.asset,
+            availability_info,
+        )
+        .into()
     }
 }