Update geojson metadata and data cols, change counting method (#365)

update geojson metadata and data cols, change counting method

Author: norlandrhagen

ocr/pipeline/fire_wind_risk_regional_aggregator.py

@@ -75,7 +75,7 @@ def custom_histogram_query(
     summary_stats_path: UPath,
     hist_bins: list[int] | None = [0.01, 0.02, 0.035, 0.06, 0.1, 0.2, 0.5, 1, 3, 100],
 ):
-    join_clause = f'JOIN {geo_table_name} b ON ST_Intersects(a.geometry, b.geometry)'
+    join_clause = f'JOIN {geo_table_name} b ON ST_Intersects(b.geometry, ST_Centroid(a.geometry))'
 
     if geo_table_name == 'county':
         name_column = 'b.NAME as NAME,'
@@ -149,7 +149,7 @@ def custom_histogram_query(
 
         b.geometry as geometry
     FROM buildings a
-    JOIN {geo_table_name} b ON ST_Intersects(a.geometry, b.geometry)
+        {join_clause}
     GROUP BY GEOID, b.geometry{name_group_by}
     """
     con.execute(nonzero_hist_query)

ocr/pipeline/write_aggregated_region_analysis_files.py

@@ -56,44 +56,61 @@ def write_stats_table(
         FROM read_parquet('{stats_parquet_path}')
     """)
 
-    # create geojson with metadata at top level
-    con.execute(f"""
-        COPY (
-            SELECT json_object(
-                'type', 'FeatureCollection',
-                'metadata', '{metadata_json}'::JSON,
-                'features', (
-                    SELECT json_group_array(
-                        json_object(
-                            'type', 'Feature',
-                            'properties', json_object(
-                                'GEOID', GEOID,
-                                {"'STUSPS', STUSPS, 'NAME', NAME," if stats_table_name == 'states' else "'NAME', NAME," if stats_table_name == 'counties' else ''}
-                                'building_count', building_count,
-                                'rps_2011_mean', rps_2011_mean,
-                                'rps_2047_mean', rps_2047_mean,
-                                'bp_2011_mean', bp_2011_mean,
-                                'bp_2047_mean', bp_2047_mean,
-                                'crps_scott_mean', crps_scott_mean,
-                                'bp_2011_riley_mean', bp_2011_riley_mean,
-                                'bp_2047_riley_mean', bp_2047_riley_mean,
-                                'rps_2011_median', rps_2011_median,
-                                'rps_2047_median', rps_2047_median,
-                                'bp_2011_median', bp_2011_median,
-                                'bp_2047_median', bp_2047_median,
-                                'crps_scott_median', crps_scott_median,
-                                'bp_2011_riley_median', bp_2011_riley_median,
-                                'bp_2047_riley_median', bp_2047_riley_median
-                            ),
-                            'geometry', json(ST_AsGeoJSON(geometry))
-                        )
-                    )
-                    FROM {stats_table_name}
-                )
-            )::VARCHAR as content
-        ) TO '{region_stats_path}/stats.geojson' (FORMAT csv, HEADER false, QUOTE '');
-    """)
+    # Stream features row-by-row via fetchone() to avoid json_group_array overflow
+    name_props = (
+        "'STUSPS', STUSPS, 'NAME', NAME,"
+        if stats_table_name == 'states'
+        else "'NAME', NAME,"
+        if stats_table_name == 'counties'
+        else ''
+    )
 
+    feature_query = f"""
+        SELECT json_object(
+            'type', 'Feature',
+            'properties', json_object(
+                'GEOID', GEOID,
+                {name_props}
+                'building_count', building_count,
+                'rps_2011_mean', rps_2011_mean,
+                'rps_2047_mean', rps_2047_mean,
+                'bp_2011_mean', bp_2011_mean,
+                'bp_2047_mean', bp_2047_mean,
+                'crps_scott_mean', crps_scott_mean,
+                'bp_2011_riley_mean', bp_2011_riley_mean,
+                'bp_2047_riley_mean', bp_2047_riley_mean,
+                'rps_2011_median', rps_2011_median,
+                'rps_2047_median', rps_2047_median,
+                'bp_2011_median', bp_2011_median,
+                'bp_2047_median', bp_2047_median,
+                'crps_scott_median', crps_scott_median,
+                'bp_2011_riley_median', bp_2011_riley_median,
+                'bp_2047_riley_median', bp_2047_riley_median,
+                'risk_score_2011_hist', risk_score_2011_hist,
+                'risk_score_2047_hist', risk_score_2047_hist
+            ),
+            'geometry', json(ST_AsGeoJSON(geometry))
+        )::VARCHAR as feature
+        FROM {stats_table_name}
+    """
+
+    geojson_path = region_stats_path / 'stats.geojson'
+    with geojson_path.open('w') as out:
+        out.write('{"type":"FeatureCollection",')
+        out.write(f'"metadata":{metadata_json},')
+        out.write('"features":[')
+
+        result = con.execute(feature_query)
+        first = True
+        while row := result.fetchone():
+            if not first:
+                out.write(',')
+            out.write(row[0])
+            first = False
+
+        out.write(']}')
+
+    # CSV output
     csv_path = region_stats_path / 'stats.csv'
 
     temp_csv_path = region_stats_path / 'stats_temp.csv'