Improve Queue Phase parallelization and other small optimizations

crates/bevy_core_pipeline/src/core_2d/main_pass_2d_node.rs

@@ -84,7 +84,7 @@ impl Node for MainPass2dNode {
         if let Some(viewport) = camera.viewport.as_ref() {
             tracked_pass.set_camera_viewport(viewport);
         }
-        for item in &transparent_phase.items {
+        for item in &transparent_phase.sorted {
             let draw_function = draw_functions.get_mut(item.draw_function).unwrap();
             draw_function.draw(world, &mut tracked_pass, view_entity, item);
         }

crates/bevy_core_pipeline/src/core_2d/mod.rs

@@ -79,6 +79,9 @@ pub struct Transparent2d {
 }
 
 impl PhaseItem for Transparent2d {
+    /// Transparent sprite rendering is reliant on stable sorting for proper
+    /// batching as the sort key does not include the Entity, and only the z-coordinate.
+    const ALLOWS_UNSTABLE_SORT: bool = false;
     type SortKey = FloatOrd;
 
     #[inline]

crates/bevy_core_pipeline/src/core_3d/main_pass_3d_node.rs

@@ -105,13 +105,13 @@ impl Node for MainPass3dNode {
             if let Some(viewport) = camera.viewport.as_ref() {
                 tracked_pass.set_camera_viewport(viewport);
             }
-            for item in &opaque_phase.items {
+            for item in &opaque_phase.sorted {
                 let draw_function = draw_functions.get_mut(item.draw_function).unwrap();
                 draw_function.draw(world, &mut tracked_pass, view_entity, item);
             }
         }
 
-        if !alpha_mask_phase.items.is_empty() {
+        if !alpha_mask_phase.sorted.is_empty() {
             // Run the alpha mask pass, sorted front-to-back
             // NOTE: Scoped to drop the mutable borrow of render_context
             #[cfg(feature = "trace")]
@@ -144,13 +144,13 @@ impl Node for MainPass3dNode {
             if let Some(viewport) = camera.viewport.as_ref() {
                 tracked_pass.set_camera_viewport(viewport);
             }
-            for item in &alpha_mask_phase.items {
+            for item in &alpha_mask_phase.sorted {
                 let draw_function = draw_functions.get_mut(item.draw_function).unwrap();
                 draw_function.draw(world, &mut tracked_pass, view_entity, item);
             }
         }
 
-        if !transparent_phase.items.is_empty() {
+        if !transparent_phase.sorted.is_empty() {
             // Run the transparent pass, sorted back-to-front
             // NOTE: Scoped to drop the mutable borrow of render_context
             #[cfg(feature = "trace")]
@@ -188,7 +188,7 @@ impl Node for MainPass3dNode {
             if let Some(viewport) = camera.viewport.as_ref() {
                 tracked_pass.set_camera_viewport(viewport);
             }
-            for item in &transparent_phase.items {
+            for item in &transparent_phase.sorted {
                 let draw_function = draw_functions.get_mut(item.draw_function).unwrap();
                 draw_function.draw(world, &mut tracked_pass, view_entity, item);
             }

crates/bevy_core_pipeline/src/core_3d/mod.rs

@@ -87,6 +87,8 @@ pub struct Opaque3d {
 }
 
 impl PhaseItem for Opaque3d {
+    /// Opaque rendering is not reliant on the stable sort order.
+    const ALLOWS_UNSTABLE_SORT: bool = true;
     type SortKey = FloatOrd;
 
     #[inline]
@@ -122,6 +124,8 @@ pub struct AlphaMask3d {
 }
 
 impl PhaseItem for AlphaMask3d {
+    /// Alpha mask rendering is not reliant on the stable sort order.
+    const ALLOWS_UNSTABLE_SORT: bool = true;
     type SortKey = FloatOrd;
 
     #[inline]
@@ -157,6 +161,8 @@ pub struct Transparent3d {
 }
 
 impl PhaseItem for Transparent3d {
+    /// 3D transparent rendering is not reliant on the stable sort order.
+    const ALLOWS_UNSTABLE_SORT: bool = true;
     type SortKey = FloatOrd;
 
     #[inline]

crates/bevy_pbr/Cargo.toml

@@ -28,3 +28,4 @@ bevy_window = { path = "../bevy_window", version = "0.8.0-dev" }
 bitflags = "1.2"
 # direct dependency required for derive macro
 bytemuck = { version = "1", features = ["derive"] }
+copyless = "0.1"

crates/bevy_pbr/src/material.rs

@@ -23,14 +23,15 @@ use bevy_render::{
         SetItemPipeline, TrackedRenderPass,
     },
     render_resource::{
-        BindGroup, BindGroupLayout, PipelineCache, RenderPipelineDescriptor, Shader,
+        BindGroup, BindGroupLayout, LockablePipelineCache, RenderPipelineDescriptor, Shader,
         SpecializedMeshPipeline, SpecializedMeshPipelineError, SpecializedMeshPipelines,
     },
     renderer::RenderDevice,
     view::{ExtractedView, Msaa, VisibleEntities},
     RenderApp, RenderStage,
 };
 use bevy_utils::tracing::error;
+use copyless::VecHelper;
 use std::hash::Hash;
 use std::marker::PhantomData;
 
@@ -330,21 +331,20 @@ pub fn queue_material_meshes<M: SpecializedMaterial>(
     transparent_draw_functions: Res<DrawFunctions<Transparent3d>>,
     material_pipeline: Res<MaterialPipeline<M>>,
     mut pipelines: ResMut<SpecializedMeshPipelines<MaterialPipeline<M>>>,
-    mut pipeline_cache: ResMut<PipelineCache>,
+    pipeline_cache: Res<LockablePipelineCache>,
     msaa: Res<Msaa>,
     render_meshes: Res<RenderAssets<Mesh>>,
     render_materials: Res<RenderAssets<M>>,
     material_meshes: Query<(&Handle<M>, &Handle<Mesh>, &MeshUniform)>,
-    mut views: Query<(
+    views: Query<(
         &ExtractedView,
         &VisibleEntities,
-        &mut RenderPhase<Opaque3d>,
-        &mut RenderPhase<AlphaMask3d>,
-        &mut RenderPhase<Transparent3d>,
+        &RenderPhase<Opaque3d>,
+        &RenderPhase<AlphaMask3d>,
+        &RenderPhase<Transparent3d>,
     )>,
 ) {
-    for (view, visible_entities, mut opaque_phase, mut alpha_mask_phase, mut transparent_phase) in
-        views.iter_mut()
+    for (view, visible_entities, opaque_phase, alpha_mask_phase, transparent_phase) in views.iter()
     {
         let draw_opaque_pbr = opaque_draw_functions
             .read()
@@ -363,6 +363,14 @@ pub fn queue_material_meshes<M: SpecializedMaterial>(
         let inverse_view_row_2 = inverse_view_matrix.row(2);
         let msaa_key = MeshPipelineKey::from_msaa_samples(msaa.samples);
 
+        let opaque_phase_cell = opaque_phase.get();
+        let alpha_mask_phase_cell = alpha_mask_phase.get();
+        let transparent_phase_cell = transparent_phase.get();
+
+        let mut opaque_phase_queue = opaque_phase_cell.take();
+        let mut alpha_mask_phase_queue = alpha_mask_phase_cell.take();
+        let mut transparent_phase_queue = transparent_phase_cell.take();
+
         for visible_entity in &visible_entities.entities {
             if let Ok((material_handle, mesh_handle, mesh_uniform)) =
                 material_meshes.get(*visible_entity)
@@ -380,7 +388,7 @@ pub fn queue_material_meshes<M: SpecializedMaterial>(
                         let material_key = M::key(material);
 
                         let pipeline_id = pipelines.specialize(
-                            &mut pipeline_cache,
+                            &pipeline_cache,
                             &material_pipeline,
                             MaterialPipelineKey {
                                 mesh_key,
@@ -402,7 +410,7 @@ pub fn queue_material_meshes<M: SpecializedMaterial>(
                         let mesh_z = inverse_view_row_2.dot(mesh_uniform.transform.col(3)) + bias;
                         match alpha_mode {
                             AlphaMode::Opaque => {
-                                opaque_phase.add(Opaque3d {
+                                opaque_phase_queue.alloc().init(Opaque3d {
                                     entity: *visible_entity,
                                     draw_function: draw_opaque_pbr,
                                     pipeline: pipeline_id,
@@ -414,7 +422,7 @@ pub fn queue_material_meshes<M: SpecializedMaterial>(
                                 });
                             }
                             AlphaMode::Mask(_) => {
-                                alpha_mask_phase.add(AlphaMask3d {
+                                alpha_mask_phase_queue.alloc().init(AlphaMask3d {
                                     entity: *visible_entity,
                                     draw_function: draw_alpha_mask_pbr,
                                     pipeline: pipeline_id,
@@ -426,7 +434,7 @@ pub fn queue_material_meshes<M: SpecializedMaterial>(
                                 });
                             }
                             AlphaMode::Blend => {
-                                transparent_phase.add(Transparent3d {
+                                transparent_phase_queue.alloc().init(Transparent3d {
                                     entity: *visible_entity,
                                     draw_function: draw_transparent_pbr,
                                     pipeline: pipeline_id,
@@ -442,5 +450,9 @@ pub fn queue_material_meshes<M: SpecializedMaterial>(
                 }
             }
         }
+
+        opaque_phase_cell.set(opaque_phase_queue);
+        alpha_mask_phase_cell.set(alpha_mask_phase_queue);
+        transparent_phase_cell.set(transparent_phase_queue);
     }
 }

crates/bevy_pbr/src/render/light.rs

@@ -1353,9 +1353,9 @@ pub fn queue_shadows(
     casting_meshes: Query<&Handle<Mesh>, Without<NotShadowCaster>>,
     render_meshes: Res<RenderAssets<Mesh>>,
     mut pipelines: ResMut<SpecializedMeshPipelines<ShadowPipeline>>,
-    mut pipeline_cache: ResMut<PipelineCache>,
+    pipeline_cache: Res<LockablePipelineCache>,
     view_lights: Query<&ViewLightEntities>,
-    mut view_light_shadow_phases: Query<(&LightEntity, &mut RenderPhase<Shadow>)>,
+    view_light_shadow_phases: Query<(&LightEntity, &RenderPhase<Shadow>)>,
     point_light_entities: Query<&CubemapVisibleEntities, With<ExtractedPointLight>>,
     directional_light_entities: Query<&VisibleEntities, With<ExtractedDirectionalLight>>,
 ) {
@@ -1365,8 +1365,8 @@ pub fn queue_shadows(
             .get_id::<DrawShadowMesh>()
             .unwrap();
         for view_light_entity in view_lights.lights.iter().copied() {
-            let (light_entity, mut shadow_phase) =
-                view_light_shadow_phases.get_mut(view_light_entity).unwrap();
+            let (light_entity, shadow_phase) =
+                view_light_shadow_phases.get(view_light_entity).unwrap();
             let visible_entities = match light_entity {
                 LightEntity::Directional { light_entity } => directional_light_entities
                     .get(*light_entity)
@@ -1379,37 +1379,39 @@ pub fn queue_shadows(
                     .expect("Failed to get point light visible entities")
                     .get(*face_index),
             };
-            // NOTE: Lights with shadow mapping disabled will have no visible entities
-            // so no meshes will be queued
-            for entity in visible_entities.iter().copied() {
-                if let Ok(mesh_handle) = casting_meshes.get(entity) {
-                    if let Some(mesh) = render_meshes.get(mesh_handle) {
-                        let key =
-                            ShadowPipelineKey::from_primitive_topology(mesh.primitive_topology);
-                        let pipeline_id = pipelines.specialize(
-                            &mut pipeline_cache,
-                            &shadow_pipeline,
-                            key,
-                            &mesh.layout,
-                        );
-
-                        let pipeline_id = match pipeline_id {
-                            Ok(id) => id,
-                            Err(err) => {
-                                error!("{}", err);
-                                continue;
-                            }
-                        };
-
-                        shadow_phase.add(Shadow {
-                            draw_function: draw_shadow_mesh,
-                            pipeline: pipeline_id,
-                            entity,
-                            distance: 0.0, // TODO: sort back-to-front
-                        });
+            shadow_phase.phase_scope(|mut phase| {
+                // NOTE: Lights with shadow mapping disabled will have no visible entities
+                // so no meshes will be queued
+                for entity in visible_entities.iter().copied() {
+                    if let Ok(mesh_handle) = casting_meshes.get(entity) {
+                        if let Some(mesh) = render_meshes.get(mesh_handle) {
+                            let key =
+                                ShadowPipelineKey::from_primitive_topology(mesh.primitive_topology);
+                            let pipeline_id = pipelines.specialize(
+                                &pipeline_cache,
+                                &shadow_pipeline,
+                                key,
+                                &mesh.layout,
+                            );
+
+                            let pipeline_id = match pipeline_id {
+                                Ok(id) => id,
+                                Err(err) => {
+                                    error!("{}", err);
+                                    continue;
+                                }
+                            };
+
+                            phase.add(Shadow {
+                                draw_function: draw_shadow_mesh,
+                                pipeline: pipeline_id,
+                                entity,
+                                distance: 0.0, // TODO: sort back-to-front
+                            });
+                        }
                     }
                 }
-            }
+            });
         }
     }
 }
@@ -1422,6 +1424,9 @@ pub struct Shadow {
 }
 
 impl PhaseItem for Shadow {
+    /// Shadow rendering is not batched, thus not reliant on a stable sort
+    /// during the sort phase.
+    const ALLOWS_UNSTABLE_SORT: bool = true;
     type SortKey = FloatOrd;
 
     #[inline]
@@ -1488,7 +1493,7 @@ impl Node for ShadowPassNode {
                     .get_manual(world, view_light_entity)
                     .unwrap();
 
-                if shadow_phase.items.is_empty() {
+                if shadow_phase.sorted.is_empty() {
                     continue;
                 }
 
@@ -1511,7 +1516,7 @@ impl Node for ShadowPassNode {
                     .begin_render_pass(&pass_descriptor);
                 let mut draw_functions = draw_functions.write();
                 let mut tracked_pass = TrackedRenderPass::new(render_pass);
-                for item in &shadow_phase.items {
+                for item in &shadow_phase.sorted {
                     let draw_function = draw_functions.get_mut(item.draw_function).unwrap();
                     draw_function.draw(world, &mut tracked_pass, view_light_entity, item);
                 }