Follow-on work to get map-packer flow working completely

package.json

@@ -24,8 +24,7 @@
     "path": "^0.12.7",
     "pg": "^8.11.5",
     "pmtiles": "3.0.3",
-    "sharp": "^0.33.2",
-    "xmldom": "^0.6.0"
+    "sharp": "^0.33.2"
   },
   "devDependencies": {
     "@babel/core": "^7.23.9",

src/azure_queue_service.js

@@ -1,5 +1,5 @@
 import { QueueServiceClient } from "@azure/storage-queue";
-import { Client } from "pg";
+import pg from "pg";
 
 import {
   parseListToFloat,
@@ -10,7 +10,8 @@ import { initiateRendering } from "./initiate.js";
 
 // Make db connection
 const connectionString = process.env["CONNECTION_STRING"];
-const client = new Client({
+
+const client = new pg.Client({
   connectionString: connectionString,
 });
 client.connect();
@@ -58,6 +59,7 @@ const processQueueMessages = async () => {
       outputDir,
       outputFilename;
     let boundsArray = [];
+    let requestId;
 
     // Decode, parse, and validate the message
     try {
@@ -84,6 +86,8 @@ const processQueueMessages = async () => {
         outputFilename = "output",
       } = options);
 
+      requestId = options.requestId;
+
       boundsArray = parseListToFloat(bounds);
 
       validateInputOptions(
@@ -101,7 +105,7 @@ const processQueueMessages = async () => {
       );
     } catch (error) {
       renderResult = handleError(error, "badRequest");
-      await writeRenderResult(renderResult, message);
+      await writeRenderResult(renderResult, message, requestId);
       continue;
     }
 
@@ -127,13 +131,19 @@ const processQueueMessages = async () => {
     } catch (error) {
       renderResult = handleError(error, "internalServerError");
     } finally {
-      await writeRenderResult(renderResult, message);
+      await writeRenderResult(renderResult, message, requestId);
     }
   }
 };
 
-const writeRenderResult = async (renderResult, message) => {
+function camelToSnakeCase(str) {
+  return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
+}
+
+const writeRenderResult = async (renderResult, message, requestId) => {
   if (renderResult) {
+    console.log(`Writing render result to database..`);
+
     let updateDbRenderRequest = `UPDATE ${db_table} SET `;
     let params = [];
     let count = 1;
@@ -143,8 +153,9 @@ const writeRenderResult = async (renderResult, message) => {
       // add a clause to the SQL query to update the corresponding column
       // and add the property value to the parameters array
       if (renderResult[key] !== null && renderResult[key] !== undefined) {
-        // All keys converted to lowercase match the db column names
-        updateDbRenderRequest += `${key.toLowerCase()} = $${count}, `;
+        // Convert camelCase to snake_case for db column names
+        let snakeCaseKey = camelToSnakeCase(key);
+        updateDbRenderRequest += `${snakeCaseKey} = $${count}, `;
         params.push(renderResult[key]);
         count++;
       }
@@ -153,9 +164,13 @@ const writeRenderResult = async (renderResult, message) => {
     // Remove space and comma from last column
     updateDbRenderRequest = updateDbRenderRequest.slice(0, -2);
     updateDbRenderRequest += ` WHERE id = $${count}`;
-    params.push(message.requestId);
+    params.push(requestId);
 
     await client.query(updateDbRenderRequest, params);
+
+    console.log(
+      `Render result has successfully been successfully to database!`,
+    );
   }
   // Delete message from queue
   await sourceQueueClient.deleteMessage(message.messageId, message.popReceipt);

src/download_resources.js

@@ -3,7 +3,6 @@ import path from "path";
 import axios from "axios";
 import pLimit from "p-limit";
 import osmtogeojson from "osmtogeojson";
-import xmldom from "xmldom";
 
 import {
   convertCoordinatesToTiles,
@@ -331,30 +330,34 @@ export const requestOpenStreetMapData = async (bounds, tempDir) => {
     console.log(`OpenStreetMap data already exists: ${osmFile}`);
   }
 
-  console.log(`OpenStreetMap data downloaded!`);
+  console.log(`OpenStreetMap data downloaded! Converting to GeoJSON...`);
 
-  // Convert OSM XML data to OSM JSON using xmldom
-  // In the future, we might want to use a more robust OSM parser like
-  // https://github.com/tyrasd/osmtogeojson/blob/gh-pages/parse_osmxml.js
-  const parser = new xmldom.DOMParser();
-  const osmData = parser.parseFromString(
-    fs.readFileSync(`${outputDir}/data.osm`, "utf-8"),
-  );
-
-  // Convert OSM JSON to GeoJSON
-  const geojson = osmtogeojson(osmData);
-
-  // Filter out lines and points only
-  geojson.features = geojson.features.filter(
-    (feature) =>
-      feature.geometry.type === "LineString" ||
-      feature.geometry.type === "Point",
-  );
+  try {
+    // Convert OSM XML data to OSM JSON using parse_osmxml
+    const parse_osmxml = (await import("osmtogeojson/parse_osmxml.js")).default;
+    const osmData = fs.readFileSync(`${outputDir}/data.osm`, "utf-8");
+
+    // Use the parser's method to parse the XML string
+    parse_osmxml.parseFromString(osmData);
+    const osmJson = parse_osmxml.getJSON();
+
+    // Convert OSM JSON to GeoJSON
+    const geojson = osmtogeojson(osmJson);
+
+    // Filter out lines and points only
+    geojson.features = geojson.features.filter(
+      (feature) =>
+        feature.geometry.type === "LineString" ||
+        feature.geometry.type === "Point",
+    );
 
-  fs.writeFileSync(
-    `${outputDir}/openstreetmap.geojson`,
-    JSON.stringify(geojson, null, 4),
-  );
+    fs.writeFileSync(
+      `${outputDir}/openstreetmap.geojson`,
+      JSON.stringify(geojson, null, 4),
+    );
+  } catch (error) {
+    throw new Error(`Error converting OpenStreetMap data to GeoJSON: ${error}`);
+  }
   console.log(
     `\x1b[32mOpenStreetMap data successfully downloaded and converted to GeoJSON!\x1b[0m`,
   );

src/generate_resources.js

@@ -100,7 +100,8 @@ export const generateMBTiles = async (
   outputDir,
   outputFilename,
 ) => {
-  const tempPath = `${tempDir}/${outputFilename}.mbtiles`;
+  const outputMBTiles = `${outputFilename}.mbtiles`;
+  const tempPath = `${tempDir}/${outputMBTiles}`;
   console.log(`Generating MBTiles file...`);
 
   let numberOfTiles = 0;
@@ -211,7 +212,7 @@ export const generateMBTiles = async (
   if (!fs.existsSync(outputDir)) {
     fs.mkdirSync(outputDir, { recursive: true });
   }
-  const outputPath = `${outputDir}/${outputFilename}.mbtiles`;
+  const outputPath = `${outputDir}/${outputMBTiles}`;
 
   try {
     const readStream = fs.createReadStream(tempPath);
@@ -240,7 +241,8 @@ export const generateMBTiles = async (
   // Return with success status
   return {
     errorMessage: null,
-    fileLocation: outputPath,
+    fileLocation: outputDir,
+    filename: outputMBTiles,
     fileSize: fileSize,
     numberOfTiles,
   };

src/initiate.js

@@ -182,6 +182,7 @@ export const initiateRendering = async (
     status: "SUCCEEDED",
     errorMessage: generateResult.errorMessage,
     fileLocation: generateResult.fileLocation,
+    filename: generateResult.filename,
     fileSize: generateResult.fileSize,
     numberOfTiles: generateResult.numberOfTiles,
     workBegun,