Fix handling of components that only vary by descriptors

Cargo.lock

@@ -8549,7 +8549,7 @@ dependencies = [
  "re_chunk_store",
  "re_format",
  "re_log_types",
- "re_types",
+ "re_types_core",
  "re_ui",
  "re_viewer_context",
 ]
@@ -9676,7 +9676,6 @@ dependencies = [
  "re_test_context",
  "re_tracing",
  "re_types",
- "re_types_core",
  "re_ui",
  "re_viewer_context",
  "re_viewport_blueprint",

crates/store/re_chunk/examples/chunk_latest_at.rs

@@ -11,13 +11,13 @@ fn main() -> anyhow::Result<()> {
     let query = LatestAtQuery::new(TimelineName::new("frame"), 4);
 
     // Find all relevant data for a query:
-    let chunk = chunk.latest_at(&query, &MyPoints::descriptor_points());
+    let chunk = chunk.latest_at(&query, MyPoints::descriptor_points().component);
     eprintln!("{:?} @ {query:?}:\n{chunk}", MyPoints::descriptor_points());
 
     // And then slice it as appropriate:
     let chunk = chunk
         .timeline_sliced(TimelineName::log_time())
-        .component_sliced(&MyPoints::descriptor_points());
+        .component_sliced(MyPoints::descriptor_points().component);
     eprintln!("Sliced down to specific timeline and component:\n{chunk}");
 
     Ok(())

crates/store/re_chunk/examples/chunk_range.rs

@@ -14,13 +14,13 @@ fn main() -> anyhow::Result<()> {
     let query = RangeQuery::new(TimelineName::new("frame"), AbsoluteTimeRange::EVERYTHING);
 
     // Find all relevant data for a query:
-    let chunk = chunk.range(&query, &MyPoints::descriptor_points());
+    let chunk = chunk.range(&query, MyPoints::descriptor_points().component);
     eprintln!("{:?} @ {query:?}:\n{chunk}", MyPoints::descriptor_points());
 
     // And then slice it as appropriate:
     let chunk = chunk
         .timeline_sliced(TimelineName::log_time())
-        .component_sliced(&MyPoints::descriptor_points());
+        .component_sliced(MyPoints::descriptor_points().component);
     eprintln!("Sliced down to specific timeline and component:\n{chunk}");
 
     Ok(())

crates/store/re_chunk/src/batcher.rs

@@ -12,7 +12,7 @@ use nohash_hasher::IntMap;
 use re_arrow_util::arrays_to_list_array_opt;
 use re_byte_size::SizeBytes as _;
 use re_log_types::{AbsoluteTimeRange, EntityPath, TimeInt, TimePoint, Timeline, TimelineName};
-use re_types_core::ComponentDescriptor;
+use re_types_core::{ComponentDescriptor, ComponentIdentifier};
 
 use crate::{Chunk, ChunkId, ChunkResult, RowId, TimeColumn, chunk::ChunkComponents};
 
@@ -784,18 +784,35 @@ pub struct PendingRow {
     /// The component data.
     ///
     /// Each array is a single component, i.e. _not_ a list array.
-    pub components: IntMap<ComponentDescriptor, ArrayRef>,
+    pub components: IntMap<ComponentIdentifier, (ComponentDescriptor, ArrayRef)>,
 }
 
 impl PendingRow {
     #[inline]
-    pub fn new(timepoint: TimePoint, components: IntMap<ComponentDescriptor, ArrayRef>) -> Self {
+    pub fn new(
+        timepoint: TimePoint,
+        components: IntMap<ComponentIdentifier, (ComponentDescriptor, ArrayRef)>,
+    ) -> Self {
         Self {
             row_id: RowId::new(),
             timepoint,
             components,
         }
     }
+
+    #[inline]
+    pub fn from_iter(
+        timepoint: TimePoint,
+        components: impl IntoIterator<Item = (ComponentDescriptor, ArrayRef)>,
+    ) -> Self {
+        Self::new(
+            timepoint,
+            components
+                .into_iter()
+                .map(|(desc, array)| (desc.component, (desc, array)))
+                .collect(),
+        )
+    }
 }
 
 impl re_byte_size::SizeBytes for PendingRow {
@@ -836,10 +853,10 @@ impl PendingRow {
             .collect();
 
         let mut per_desc = ChunkComponents::default();
-        for (component_desc, array) in components {
+        for (component, (desc, array)) in components {
             let list_array = arrays_to_list_array_opt(&[Some(&*array as _)]);
             if let Some(list_array) = list_array {
-                per_desc.insert(component_desc, list_array);
+                per_desc.insert(component, (desc, list_array));
             }
         }
 
@@ -925,7 +942,7 @@ impl PendingRow {
                     let mut hasher = ahash::AHasher::default();
                     row.components
                         .values()
-                        .for_each(|array| array.data_type().hash(&mut hasher));
+                        .for_each(|(_, array)| array.data_type().hash(&mut hasher));
                     per_datatype_set
                         .entry(hasher.finish())
                         .or_default()
@@ -943,11 +960,12 @@ impl PendingRow {
 
                 // Create all the logical list arrays that we're going to need, accounting for the
                 // possibility of sparse components in the data.
-                let mut all_components: IntMap<ComponentDescriptor, Vec<Option<&dyn ArrowArray>>> =
-                    IntMap::default();
+                let mut all_components: IntMap<ComponentIdentifier, _> = IntMap::default();
                 for row in &rows {
-                    for component_desc in row.components.keys() {
-                        all_components.entry(component_desc.clone()).or_default();
+                    for (component, (desc, _)) in &row.components {
+                        all_components
+                            .entry(*component)
+                            .or_insert_with(|| (desc.clone(), Vec::new()));
                     }
                 }
 
@@ -983,15 +1001,16 @@ impl PendingRow {
                                     .map(|(name, time_column)| (name, time_column.finish()))
                                     .collect(),
                                 {
-                                    let mut per_desc = ChunkComponents::default();
-                                    for (component_desc, arrays) in std::mem::take(&mut components)
+                                    let mut per_component = ChunkComponents::default();
+                                    for (component, (desc, arrays)) in
+                                        std::mem::take(&mut components)
                                     {
                                         let list_array = arrays_to_list_array_opt(&arrays);
                                         if let Some(list_array) = list_array {
-                                            per_desc.insert(component_desc, list_array);
+                                            per_component.insert(component, (desc, list_array));
                                         }
                                     }
-                                    per_desc
+                                    per_component
                                 },
                             ));
 
@@ -1008,13 +1027,13 @@ impl PendingRow {
                         time_column.push(cell.into());
                     }
 
-                    for (component_desc, arrays) in &mut components {
+                    for (component, (_desc, arrays)) in &mut components {
                         // NOTE: This will push `None` if the row doesn't actually hold a value for this
                         // component -- these are sparse list arrays!
                         arrays.push(
                             row_components
-                                .get(component_desc)
-                                .map(|array| &**array as &dyn ArrowArray),
+                                .get(component)
+                                .map(|(_, array)| &**array as &dyn ArrowArray),
                         );
                     }
                 }
@@ -1030,10 +1049,10 @@ impl PendingRow {
                         .collect(),
                     {
                         let mut per_desc = ChunkComponents::default();
-                        for (component_desc, arrays) in components {
+                        for (component, (desc, arrays)) in components {
                             let list_array = arrays_to_list_array_opt(&arrays);
                             if let Some(list_array) = list_array {
-                                per_desc.insert(component_desc, list_array);
+                                per_desc.insert(component, (desc, list_array));
                             }
                         }
                         per_desc
@@ -1152,9 +1171,9 @@ mod tests {
             (MyIndex::partial_descriptor(), indices3.clone()),
         ];
 
-        let row1 = PendingRow::new(timepoint1.clone(), components1.into_iter().collect());
-        let row2 = PendingRow::new(timepoint2.clone(), components2.into_iter().collect());
-        let row3 = PendingRow::new(timepoint3.clone(), components3.into_iter().collect());
+        let row1 = PendingRow::from_iter(timepoint1.clone(), components1);
+        let row2 = PendingRow::from_iter(timepoint2.clone(), components2);
+        let row3 = PendingRow::from_iter(timepoint3.clone(), components3);
 
         let entity_path1: EntityPath = "a/b/c".into();
         batcher.push_row(entity_path1.clone(), row1.clone());
@@ -1261,9 +1280,9 @@ mod tests {
             (MyPoints::descriptor_points(), points3.clone()),
         ];
 
-        let row1 = PendingRow::new(timepoint1.clone(), components1.into_iter().collect());
-        let row2 = PendingRow::new(timepoint2.clone(), components2.into_iter().collect());
-        let row3 = PendingRow::new(timepoint3.clone(), components3.into_iter().collect());
+        let row1 = PendingRow::from_iter(timepoint1.clone(), components1);
+        let row2 = PendingRow::from_iter(timepoint2.clone(), components2);
+        let row3 = PendingRow::from_iter(timepoint3.clone(), components3);
 
         let entity_path1: EntityPath = "a/b/c".into();
         batcher.push_row(entity_path1.clone(), row1.clone());
@@ -1349,9 +1368,9 @@ mod tests {
         let components2 = [(MyPoints::descriptor_points(), points2.clone())];
         let components3 = [(MyPoints::descriptor_points(), points3.clone())];
 
-        let row1 = PendingRow::new(static_.clone(), components1.into_iter().collect());
-        let row2 = PendingRow::new(static_.clone(), components2.into_iter().collect());
-        let row3 = PendingRow::new(static_.clone(), components3.into_iter().collect());
+        let row1 = PendingRow::from_iter(static_.clone(), components1);
+        let row2 = PendingRow::from_iter(static_.clone(), components2);
+        let row3 = PendingRow::from_iter(static_.clone(), components3);
 
         let entity_path1: EntityPath = "a/b/c".into();
         batcher.push_row(entity_path1.clone(), row1.clone());
@@ -1424,9 +1443,9 @@ mod tests {
         let components2 = [(MyPoints::descriptor_points(), points2.clone())];
         let components3 = [(MyPoints::descriptor_points(), points3.clone())];
 
-        let row1 = PendingRow::new(timepoint1.clone(), components1.into_iter().collect());
-        let row2 = PendingRow::new(timepoint2.clone(), components2.into_iter().collect());
-        let row3 = PendingRow::new(timepoint3.clone(), components3.into_iter().collect());
+        let row1 = PendingRow::from_iter(timepoint1.clone(), components1);
+        let row2 = PendingRow::from_iter(timepoint2.clone(), components2);
+        let row3 = PendingRow::from_iter(timepoint3.clone(), components3);
 
         let entity_path1: EntityPath = "ent1".into();
         let entity_path2: EntityPath = "ent2".into();
@@ -1532,9 +1551,9 @@ mod tests {
         let components2 = [(MyPoints::descriptor_points(), points2.clone())];
         let components3 = [(MyPoints::descriptor_points(), points3.clone())];
 
-        let row1 = PendingRow::new(timepoint1.clone(), components1.into_iter().collect());
-        let row2 = PendingRow::new(timepoint2.clone(), components2.into_iter().collect());
-        let row3 = PendingRow::new(timepoint3.clone(), components3.into_iter().collect());
+        let row1 = PendingRow::from_iter(timepoint1.clone(), components1);
+        let row2 = PendingRow::from_iter(timepoint2.clone(), components2);
+        let row3 = PendingRow::from_iter(timepoint3.clone(), components3);
 
         let entity_path1: EntityPath = "a/b/c".into();
         batcher.push_row(entity_path1.clone(), row1.clone());
@@ -1641,9 +1660,9 @@ mod tests {
         let components2 = [(MyPoints::descriptor_points(), points2.clone())]; // same name, different datatype
         let components3 = [(MyPoints::descriptor_points(), points3.clone())];
 
-        let row1 = PendingRow::new(timepoint1.clone(), components1.into_iter().collect());
-        let row2 = PendingRow::new(timepoint2.clone(), components2.into_iter().collect());
-        let row3 = PendingRow::new(timepoint3.clone(), components3.into_iter().collect());
+        let row1 = PendingRow::from_iter(timepoint1.clone(), components1);
+        let row2 = PendingRow::from_iter(timepoint2.clone(), components2);
+        let row3 = PendingRow::from_iter(timepoint3.clone(), components3);
 
         let entity_path1: EntityPath = "a/b/c".into();
         batcher.push_row(entity_path1.clone(), row1.clone());
@@ -1763,10 +1782,10 @@ mod tests {
         let components3 = [(MyPoints::descriptor_points(), points3.clone())];
         let components4 = [(MyPoints::descriptor_points(), points4.clone())];
 
-        let row1 = PendingRow::new(timepoint4.clone(), components1.into_iter().collect());
-        let row2 = PendingRow::new(timepoint1.clone(), components2.into_iter().collect());
-        let row3 = PendingRow::new(timepoint2.clone(), components3.into_iter().collect());
-        let row4 = PendingRow::new(timepoint3.clone(), components4.into_iter().collect());
+        let row1 = PendingRow::from_iter(timepoint4.clone(), components1);
+        let row2 = PendingRow::from_iter(timepoint1.clone(), components2);
+        let row3 = PendingRow::from_iter(timepoint2.clone(), components3);
+        let row4 = PendingRow::from_iter(timepoint3.clone(), components4);
 
         let entity_path1: EntityPath = "a/b/c".into();
         batcher.push_row(entity_path1.clone(), row1.clone());
@@ -1870,10 +1889,10 @@ mod tests {
         let components3 = [(MyPoints::descriptor_points(), points3.clone())];
         let components4 = [(MyPoints::descriptor_points(), points4.clone())];
 
-        let row1 = PendingRow::new(timepoint4.clone(), components1.into_iter().collect());
-        let row2 = PendingRow::new(timepoint1.clone(), components2.into_iter().collect());
-        let row3 = PendingRow::new(timepoint2.clone(), components3.into_iter().collect());
-        let row4 = PendingRow::new(timepoint3.clone(), components4.into_iter().collect());
+        let row1 = PendingRow::from_iter(timepoint4.clone(), components1);
+        let row2 = PendingRow::from_iter(timepoint1.clone(), components2);
+        let row3 = PendingRow::from_iter(timepoint2.clone(), components3);
+        let row4 = PendingRow::from_iter(timepoint3.clone(), components4);
 
         let entity_path1: EntityPath = "a/b/c".into();
         batcher.push_row(entity_path1.clone(), row1.clone());

crates/store/re_chunk/src/builder.rs

@@ -5,7 +5,7 @@ use nohash_hasher::IntMap;
 use re_log_types::{EntityPath, NonMinI64, TimePoint, Timeline, TimelineName};
 use re_types_core::{AsComponents, ComponentBatch, ComponentDescriptor, SerializedComponentBatch};
 
-use crate::{Chunk, ChunkId, ChunkResult, RowId, TimeColumn, chunk::ChunkComponents};
+use crate::{Chunk, ChunkId, ChunkResult, RowId, TimeColumn};
 
 // ---
 
@@ -297,16 +297,14 @@ impl ChunkBuilder {
 
         let components = {
             re_tracing::profile_scope!("components");
-            ChunkComponents(
-                components
-                    .into_iter()
-                    .filter_map(|(component_desc, arrays)| {
-                        let arrays = arrays.iter().map(|array| array.as_deref()).collect_vec();
-                        re_arrow_util::arrays_to_list_array_opt(&arrays)
-                            .map(|list_array| (component_desc, list_array))
-                    })
-                    .collect(),
-            )
+            components
+                .into_iter()
+                .filter_map(|(component_desc, arrays)| {
+                    let arrays = arrays.iter().map(|array| array.as_deref()).collect_vec();
+                    re_arrow_util::arrays_to_list_array_opt(&arrays)
+                        .map(|list_array| (component_desc, list_array))
+                })
+                .collect()
         };
 
         Chunk::from_native_row_ids(id, entity_path, None, &row_ids, timelines, components)
@@ -350,23 +348,21 @@ impl ChunkBuilder {
                 .map(|(timeline, time_column)| (timeline, time_column.build()))
                 .collect(),
             {
-                ChunkComponents(
-                    components
-                        .into_iter()
-                        .filter_map(|(component_desc, arrays)| {
-                            let arrays = arrays.iter().map(|array| array.as_deref()).collect_vec();
-                            // If we know the datatype in advance, we're able to keep even fully sparse
-                            // columns around.
-                            if let Some(datatype) = datatypes.get(&component_desc) {
-                                re_arrow_util::arrays_to_list_array(datatype.clone(), &arrays)
-                                    .map(|list_array| (component_desc, list_array))
-                            } else {
-                                re_arrow_util::arrays_to_list_array_opt(&arrays)
-                                    .map(|list_array| (component_desc, list_array))
-                            }
-                        })
-                        .collect(),
-                )
+                components
+                    .into_iter()
+                    .filter_map(|(component_desc, arrays)| {
+                        let arrays = arrays.iter().map(|array| array.as_deref()).collect_vec();
+                        // If we know the datatype in advance, we're able to keep even fully sparse
+                        // columns around.
+                        if let Some(datatype) = datatypes.get(&component_desc) {
+                            re_arrow_util::arrays_to_list_array(datatype.clone(), &arrays)
+                                .map(|list_array| (component_desc, list_array))
+                        } else {
+                            re_arrow_util::arrays_to_list_array_opt(&arrays)
+                                .map(|list_array| (component_desc, list_array))
+                        }
+                    })
+                    .collect()
             },
         )
     }