POC improve transfers

Cargo.toml

@@ -55,6 +55,7 @@ num-traits = "0.2"
 pretty_assertions = "1"
 proj = { version = "0.30", optional = true } # libproj version used by 'proj' crate must be propagated to CI and makefile
 relational_types = { git = "https://github.com/hove-io/relational_types", tag = "v2"}
+rstar = "0.12"
 rust_decimal = "1"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

gtfs2ntfs/src/main.rs

@@ -137,13 +137,14 @@ fn run(opt: Opt) -> Result<()> {
     let model = if opt.ignore_transfers {
         model
     } else {
-        generates_transfers(
+        let collections = generates_transfers(
             model,
             opt.max_distance,
             opt.walking_speed,
             opt.waiting_time,
             None,
-        )?
+        )?;
+        transit_model::Model::new(collections)?
     };
 
     match opt.output.extension() {

ntfs2ntfs/src/main.rs

@@ -94,7 +94,7 @@ fn run(opt: Opt) -> Result<()> {
 
     let model = transit_model::ntfs::read(opt.input)?;
     let model = if opt.ignore_transfers {
-        model
+        model.into_collections()
     } else {
         generates_transfers(
             model,

src/ntfs/mod.rs

@@ -352,7 +352,7 @@ where
 /// [NTFS](https://github.com/hove-io/ntfs-specification/blob/master/ntfs_fr.md)
 /// files in the given directory.
 pub fn write<P: AsRef<path::Path>>(
-    model: &Model,
+    model: &Collections,
     path: P,
     current_datetime: DateTime<FixedOffset>,
 ) -> Result<()> {
@@ -437,7 +437,7 @@ pub fn write<P: AsRef<path::Path>>(
 /// [NTFS](https://github.com/hove-io/ntfs-specification/blob/master/ntfs_fr.md)
 /// ZIP archive at the given full path.
 pub fn write_to_zip<P: AsRef<path::Path>>(
-    model: &Model,
+    model: &Collections,
     path: P,
     current_datetime: DateTime<FixedOffset>,
 ) -> Result<()> {

src/transfers.rs

@@ -15,11 +15,13 @@
 //! See function generates_transfers
 
 use crate::{
-    model::Model,
+    model::{Collections, Model},
     objects::{Coord, StopPoint, Transfer},
     Result,
 };
+use rstar::{RTree, RTreeObject, AABB};
 use std::collections::HashMap;
+use std::time::Instant;
 use tracing::info;
 use typed_index_collection::{Collection, CollectionWithId, Idx};
 
@@ -49,7 +51,33 @@ pub fn get_available_transfers(
         .collect()
 }
 
+/// Wrapper for stop point with its index for use in R-tree
+#[derive(Debug, Clone)]
+struct StopPointLocation {
+    idx: Idx<StopPoint>,
+    coord: Coord,
+}
+
+impl RTreeObject for StopPointLocation {
+    type Envelope = AABB<[f64; 2]>;
+
+    fn envelope(&self) -> Self::Envelope {
+        AABB::from_point([self.coord.lon, self.coord.lat])
+    }
+}
+
+impl StopPointLocation {
+    fn new(idx: Idx<StopPoint>, coord: Coord) -> Self {
+        Self { idx, coord }
+    }
+}
 /// Generate missing transfers from stop points within the required distance
+///
+/// This function uses an R-tree spatial index for efficient proximity queries.
+/// R-trees are optimized for spatial data and provide O(log n) query performance.
+///
+/// Complexity: O(n × log n) for building the tree + O(n × k × log n) for queries
+/// where k is the average number of nearby points within max_distance
 pub fn generate_missing_transfers_from_sp(
     transfers_map: &TransferMap,
     model: &Model,
@@ -58,18 +86,60 @@ pub fn generate_missing_transfers_from_sp(
     waiting_time: u32,
     need_transfer: Option<NeedTransfer>,
 ) -> TransferMap {
+    let total_start = Instant::now();
     info!("Adding missing transfers from stop points.");
     let mut new_transfers_map = TransferMap::new();
     let sq_max_distance = max_distance * max_distance;
+
+    // Build R-tree for efficient spatial queries
+    let rtree_start = Instant::now();
+    let stop_locations: Vec<StopPointLocation> = model
+        .stop_points
+        .iter()
+        .filter(|(_, sp)| sp.coord != Coord::default())
+        .map(|(idx, sp)| StopPointLocation::new(idx, sp.coord.clone()))
+        .collect();
+
+    let valid_stop_count = stop_locations.len();
+    let rtree = RTree::bulk_load(stop_locations);
+    let rtree_duration = rtree_start.elapsed();
+    info!(
+        "Built R-tree for {} valid stop points in {:.2?}",
+        valid_stop_count, rtree_duration
+    );
+
+    let compute_start = Instant::now();
+    let mut total_comparisons = 0_u64;
+    let mut total_transfers_created = 0_u64;
+    let mut total_distance_checks = 0_u64;
+
+    // For each stop point, query nearby points from the R-tree
     for (idx1, sp1) in model.stop_points.iter() {
         if sp1.coord == Coord::default() {
             continue;
         }
+
+        // Pre-calculate the approximation (cosinus of latitude) once for this stop
+        // This optimizes distance calculations for all nearby points
         let approx = sp1.coord.approx();
-        for (idx2, sp2) in model.stop_points.iter() {
-            if sp2.coord == Coord::default() {
-                continue;
-            }
+
+        // Query R-tree for points within a bounding box
+        // Note: locate_within_distance cannot be used here because it requires
+        // Euclidean distance, but we need geographic distance calculation with approx()
+        // Convert max_distance from meters to degrees (approximate: 1 degree ≈ 111km at equator)
+        let search_distance_degrees = max_distance / 111_000.0;
+        let min_lon = sp1.coord.lon - search_distance_degrees;
+        let max_lon = sp1.coord.lon + search_distance_degrees;
+        let min_lat = sp1.coord.lat - search_distance_degrees;
+        let max_lat = sp1.coord.lat + search_distance_degrees;
+
+        let search_box = AABB::from_corners([min_lon, min_lat], [max_lon, max_lat]);
+
+        // Get all points within the bounding box and filter by actual distance
+        for nearby_location in rtree.locate_in_envelope(&search_box) {
+            total_comparisons += 1;
+            let idx2 = nearby_location.idx;
+
             if transfers_map.contains_key(&(idx1, idx2)) {
                 continue;
             }
@@ -78,11 +148,16 @@ pub fn generate_missing_transfers_from_sp(
                     continue;
                 }
             }
-            let sq_distance = approx.sq_distance_to(&sp2.coord);
+
+            total_distance_checks += 1;
+            // Use the pre-calculated approximation for efficient distance calculation
+            let sq_distance = approx.sq_distance_to(&nearby_location.coord);
             if sq_distance > sq_max_distance {
                 continue;
             }
+            total_transfers_created += 1;
             let transfer_time = (sq_distance.sqrt() / walking_speed) as u32;
+            let sp2 = &model.stop_points[idx2];
             new_transfers_map.insert(
                 (idx1, idx2),
                 Transfer {
@@ -95,6 +170,22 @@ pub fn generate_missing_transfers_from_sp(
             );
         }
     }
+
+    let compute_duration = compute_start.elapsed();
+    let total_duration = total_start.elapsed();
+
+    info!(
+        "Transfer computation stats: {} comparisons, {} distance checks, {} transfers created in {:.2?}",
+        total_comparisons,
+        total_distance_checks,
+        total_transfers_created,
+        compute_duration
+    );
+    info!(
+        "Total time for generate_missing_transfers_from_sp: {:.2?}",
+        total_duration
+    );
+
     new_transfers_map
 }
 
@@ -130,9 +221,19 @@ pub fn generates_transfers(
     walking_speed: f64,
     waiting_time: u32,
     need_transfer: Option<NeedTransfer>,
-) -> Result<Model> {
+) -> Result<Collections> {
+    let total_start = Instant::now();
     info!("Generating transfers...");
+
+    let get_transfers_start = Instant::now();
     let mut transfers_map = get_available_transfers(model.transfers.clone(), &model.stop_points);
+    info!(
+        "get_available_transfers: {} existing transfers in {:.2?}",
+        transfers_map.len(),
+        get_transfers_start.elapsed()
+    );
+
+    let gen_transfers_start = Instant::now();
     let new_transfers_map = generate_missing_transfers_from_sp(
         &transfers_map,
         &model,
@@ -141,16 +242,39 @@ pub fn generates_transfers(
         waiting_time,
         need_transfer,
     );
+    info!(
+        "generate_missing_transfers_from_sp returned {} new transfers in {:.2?}",
+        new_transfers_map.len(),
+        gen_transfers_start.elapsed()
+    );
 
+    let merge_start = Instant::now();
     transfers_map.extend(new_transfers_map);
     let mut new_transfers: Vec<_> = transfers_map.into_values().collect();
+    info!("Merged transfers in {:.2?}", merge_start.elapsed());
+
+    let sort_start = Instant::now();
     new_transfers.sort_unstable_by(|t1, t2| {
         (&t1.from_stop_id, &t1.to_stop_id).cmp(&(&t2.from_stop_id, &t2.to_stop_id))
     });
+    info!(
+        "Sorted {} transfers in {:.2?}",
+        new_transfers.len(),
+        sort_start.elapsed()
+    );
 
+    let rebuild_start = Instant::now();
     let mut collections = model.into_collections();
     collections.transfers = Collection::new(new_transfers);
-    Model::new(collections)
+    // let result = Model::new(collections);
+    info!("Rebuilt model in {:.2?}", rebuild_start.elapsed());
+
+    info!(
+        "generates_transfers TOTAL TIME: {:.2?}",
+        total_start.elapsed()
+    );
+
+    Ok(collections)
 }
 
 #[cfg(test)]
@@ -361,8 +485,8 @@ mod tests {
     #[test]
     fn test_generates_transfers() {
         let model = base_model();
-        let new_model = generates_transfers(model, 100.0, 0.7, 2, None).expect("an error occured");
-        let mut collections = new_model.into_collections();
+        let mut collections =
+            generates_transfers(model, 100.0, 0.7, 2, None).expect("an error occured");
 
         let mut transfers = Collection::new(vec![
             Transfer {
@@ -441,4 +565,61 @@ mod tests {
 
         assert_eq!(transfers, transfers_expected);
     }
+
+    #[test]
+    fn test_spatial_grid_performance() {
+        // Create a model with many stop points to demonstrate the performance improvement
+        let mut model_builder = ModelBuilder::default();
+
+        // Create a grid of stop points (e.g., 100 x 100 = 10,000 points)
+        // In a real scenario with 10k points:
+        // - O(n²) = 100,000,000 comparisons
+        // - O(n×log n) with R-tree for spatial queries
+        // This is significantly faster!
+
+        let grid_size = 10; // Use 10x10 = 100 points for the test (to keep it fast)
+        for i in 0..grid_size {
+            for j in 0..grid_size {
+                let stop_id = format!("SP_{i}_{j}");
+                // Create stops in a 0.01° x 0.01° grid (roughly 1km x 1km)
+
+                model_builder = model_builder.vj(&format!("vj_{i}_{j}"), |vj_builder| {
+                    vj_builder
+                        .route(&format!("route_{i}_{j}"))
+                        .st(&stop_id, "10:00:00");
+                });
+            }
+        }
+
+        let transit_model = model_builder.build();
+        let mut collections = transit_model.into_collections();
+
+        // Set coordinates for all stop points
+        for i in 0..grid_size {
+            for j in 0..grid_size {
+                let stop_id = format!("SP_{i}_{j}");
+                collections.stop_points.get_mut(&stop_id).unwrap().coord = Coord {
+                    lon: 2.39 + (i as f64) * 0.001,
+                    lat: 48.85 + (j as f64) * 0.001,
+                };
+            }
+        }
+
+        let model = Model::new(collections).unwrap();
+
+        // Generate transfers with a reasonable distance (500m)
+        let result = generates_transfers(model, 500.0, 0.7, 2, None);
+        assert!(result.is_ok());
+
+        // With R-tree, this should complete quickly even with 100+ points
+        // The number of transfers should be reasonable (not n²)
+        let collections = result.unwrap();
+        let transfer_count = collections.transfers.len();
+
+        // Each point should have transfers to nearby points (not all points)
+        // With 100 points in a 10x10 grid and 500m max distance,
+        // each point should connect to roughly 4-9 neighbors
+        assert!(transfer_count < grid_size * grid_size * grid_size * grid_size);
+        assert!(transfer_count > 0);
+    }
 }