fix: add new implementation of read and code gen phases based on parse package (#413)

Fixes #412

Author: chrispcampbell

.gitignore

@@ -1,6 +1,7 @@
 # Project files
 models/**/build*
 models/**/output*
+models/**/compare
 app_spec.json
 removals.txt
 .DS_Store

packages/compile/package.json

@@ -16,6 +16,7 @@
     "ci:build": "run-s lint prettier:check type-check test:ci"
   },
   "dependencies": {
+    "@sdeverywhere/parse": "^0.1.0",
     "antlr4": "4.12.0",
     "antlr4-vensim": "0.6.2",
     "bufx": "^1.0.5",

packages/compile/src/_tests/test-support.ts

@@ -1,10 +1,25 @@
+import { readFileSync } from 'fs'
 import { resolve } from 'path'
 import { fileURLToPath } from 'url'
 
+import type { Model } from '@sdeverywhere/parse'
+
 import type { VensimModelParseTree } from '../parse/parser'
-import { parseModel } from '../parse/parser'
 import { preprocessModel } from '../preprocess/preprocessor'
 import { canonicalName } from '../_shared/helpers'
+import { parseModel } from '../parse-and-generate'
+
+export interface ParsedVensimModel {
+  kind: 'vensim'
+  root: Model
+}
+
+export interface LegacyParsedVensimModel {
+  kind: 'vensim-legacy'
+  parseTree: VensimModelParseTree
+}
+
+export type ParsedModel = ParsedVensimModel | LegacyParsedVensimModel
 
 export type DimModelName = string
 export type DimCName = string
@@ -49,6 +64,7 @@ export type VariableType = 'const' | 'aux' | 'level' | 'initial' | 'lookup' | 'd
 export interface Variable {
   modelLHS: string // 'Target Capacity'
   modelFormula: string // 'ACTIVE INITIAL(Capacity*Utilization Adjustment,Initial Target Capacity)'
+  origModelFormula?: string // 'IF THEN ELSE(cond, x, y)'
   varName: string // '_target_capacity'
   subscripts: string[] // TODO: sub type
   exceptSubscripts: string[] // TODO: This is only used during parsing, doesn't need to be exposed
@@ -140,14 +156,31 @@ export function sampleModelDir(modelName: string): string {
   return resolve(__dirname, '..', '..', '..', '..', 'models', modelName)
 }
 
-export function parseVensimModel(modelName: string): VensimModelParseTree {
-  const modelFile = resolve(sampleModelDir(modelName), `${modelName}.mdl`)
-  const preprocessed = preprocessModel(modelFile, undefined, 'genc', false)
-  return parseModel(preprocessed)
+export function parseVensimModel(modelName: string): ParsedModel {
+  const modelDir = sampleModelDir(modelName)
+  const modelFile = resolve(modelDir, `${modelName}.mdl`)
+  let mdlContent: string
+  if (process.env.SDE_NONPUBLIC_USE_NEW_PARSE === '1') {
+    // Note that the new parser implicitly runs the preprocessor on the input model text,
+    // so we don't need to do that here.  (We should make it configurable so that we can
+    // skip the preprocess step in `parse-and-generate.js` when the input model text has
+    // already been run through a preprocessor.)
+    mdlContent = readFileSync(modelFile, 'utf8')
+  } else {
+    mdlContent = preprocessModel(modelFile, undefined, 'genc', false)
+  }
+  // We currently sort the preprocessed definitions alphabetically for
+  // compatibility with the legacy preprocessor.  Once we drop the legacy code
+  // we could remove this step and update the tests to use the original order.
+  return parseModel(mdlContent, modelDir, /*sort=*/ true)
 }
 
-export function parseInlineVensimModel(mdl: string): VensimModelParseTree {
-  return parseModel(mdl)
+export function parseInlineVensimModel(mdlContent: string, modelDir?: string): ParsedModel {
+  // For tests that parse inline model text, in the case of the legacy parser, don't run
+  // the preprocess step, and in the case of the new parser (which implicitly runs the
+  // preprocess step), don't sort the definitions.  This makes it easier to do apples
+  // to apples comparisons on the outputs from the two parser implementations.
+  return parseModel(mdlContent, modelDir, /*sort=*/ false)
 }
 
 function prettyVar(variable: Variable): string {

packages/compile/src/generate/code-gen.js

@@ -4,14 +4,15 @@ import { asort, lines, strlist, abend, mapIndexed } from '../_shared/helpers.js'
 import { sub, allDimensions, allMappings, subscriptFamilies } from '../_shared/subscript.js'
 import Model from '../model/model.js'
 
+import { generateEquation } from './gen-equation.js'
 import EquationGen from './equation-gen.js'
 import ModelLHSReader from './model-lhs-reader.js'
 
-export function generateCode(parseTree, opts) {
-  return codeGenerator(parseTree, opts).generate()
+export function generateCode(parsedModel, opts) {
+  return codeGenerator(parsedModel, opts).generate()
 }
 
-let codeGenerator = (parseTree, opts) => {
+let codeGenerator = (parsedModel, opts) => {
   const { spec, operation, extData, directData, modelDirname } = opts
   // Set to 'decl', 'init-lookups', 'eval', etc depending on the section being generated.
   let mode = ''
@@ -25,13 +26,19 @@ let codeGenerator = (parseTree, opts) => {
     outputAllVars = true
   }
   // Function to generate a section of the code
-  let generateSection = R.map(v => new EquationGen(v, extData, directData, mode, modelDirname).generate())
+  let generateSection = R.map(v => {
+    if (parsedModel.kind === 'vensim-legacy') {
+      return new EquationGen(v, extData, directData, mode, modelDirname).generate()
+    } else {
+      return generateEquation(v, mode, extData, directData, modelDirname)
+    }
+  })
   let section = R.pipe(generateSection, R.flatten, lines)
   function generate() {
     // Read variables and subscript ranges from the model parse tree.
     // This is the main entry point for code generation and is called just once.
     try {
-      Model.read(parseTree, spec, extData, directData, modelDirname)
+      Model.read(parsedModel, spec, extData, directData, modelDirname)
       // In list mode, print variables to the console instead of generating code.
       if (operation === 'printRefIdTest') {
         Model.printRefIdTest()
@@ -198,10 +205,21 @@ void ${name}${idx}() {
     }
     let funcCalls = R.pipe(mapIndexed(funcCall), lines)
 
-    // Break the vars into chunks of 30; this number was empirically
-    // determined by looking at runtime performance and memory usage
-    // of the En-ROADS model on various devices
-    let chunks = R.splitEvery(30, vars)
+    // Break the vars into chunks.  The default value of 30 was empirically
+    // determined by looking at runtime performance and memory usage of the
+    // En-ROADS model on various devices.
+    let chunkSize
+    if (process.env.SDE_CODE_GEN_CHUNK_SIZE) {
+      chunkSize = parseInt(process.env.SDE_CODE_GEN_CHUNK_SIZE)
+    } else {
+      chunkSize = 30
+    }
+    let chunks
+    if (chunkSize > 0) {
+      chunks = R.splitEvery(chunkSize, vars)
+    } else {
+      chunks = [vars]
+    }
 
     if (!preStep) {
       preStep = ''

packages/compile/src/generate/direct-data-helpers.js

@@ -0,0 +1,86 @@
+import path from 'node:path'
+
+import XLSX from 'xlsx'
+
+import { cdbl, readCsv, readXlsx } from '../_shared/helpers.js'
+
+/**
+ * Return a `getCellValue` function that reads the CSV or XLS[X] content.
+ *
+ * @param {string} fileOrTag The filename (e.g., 'data.xlsx') or tag name (e.g., '?data').
+ * @param {string} tabOrDelimiter
+ * @param {'data' | 'constants'} dataKind The kind of `GET DIRECT ...` being used.
+ * @param {Map<string, any>} directData The mapping of dataset name used in a `GET DIRECT DATA` call (e.g.,
+ * `?data`) to the tabular data contained in the loaded data file.
+ * @param {string} modelDir The path to the directory containing the model (used for resolving data files).
+ * @returns A `getCellValue` function.
+ */
+export function handleExcelOrCsvFile(fileOrTag, tabOrDelimiter, dataKind, directData, modelDir) {
+  if (fileOrTag.startsWith('?')) {
+    // The file is a tag for an Excel file with data in the directData map.
+    const workbook = directData.get(fileOrTag)
+    return handleExcelWorkbook(fileOrTag, workbook, tabOrDelimiter, dataKind, 'tagged')
+  } else {
+    // The file is a CSV or XLS[X] pathname. Read it now.
+    const dataPathname = path.resolve(modelDir, fileOrTag)
+    if (dataPathname.toLowerCase().endsWith('csv')) {
+      return handleCsvFile(fileOrTag, dataPathname, tabOrDelimiter, dataKind)
+    } else {
+      const workbook = readXlsx(dataPathname)
+      return handleExcelWorkbook(fileOrTag, workbook, tabOrDelimiter, dataKind, 'file')
+    }
+  }
+}
+
+/**
+ * Return a `getCellValue` function for the given Excel workbook parsed from an XLS[X] file.
+ *
+ * @param {string} fileOrTag The filename (e.g., 'data.xlsx') or tag name (e.g., '?data').
+ * @param {*} workbook The workbook data loaded from the file.
+ * @param {string} tab The name of the tab within the workbook.
+ * @param {'data' | 'constants'} dataKind The kind of `GET DIRECT ...` being used.
+ * @param {'file' | 'tagged'} dataSource The reference kind, either 'file' or 'tagged'.
+ * @returns A `getCellValue` function.
+ */
+function handleExcelWorkbook(fileOrTag, workbook, tab, dataKind, dataSource) {
+  if (workbook) {
+    let sheet = workbook.Sheets[tab]
+    if (sheet) {
+      return (c, r) => {
+        let cell = sheet[XLSX.utils.encode_cell({ c, r })]
+        return cell != null ? cdbl(cell.v) : null
+      }
+    } else {
+      throw new Error(`Direct ${dataKind} worksheet ${tab} in ${dataSource} ${fileOrTag} not found`)
+    }
+  } else {
+    throw new Error(`Direct ${dataKind} workbook ${dataSource} ${fileOrTag} not found`)
+  }
+}
+
+/**
+ * Return a `getCellValue` function for the given CSV file.
+ *
+ * @param {string} file The filename of the data file.
+ * @param {string} dataFilename The full path to the data file.
+ * @param {string} delimiter The delimiter for the tabular data.
+ * @param {'data' | 'constants'} dataKind The kind of `GET DIRECT ...` being used.
+ * @returns A `getCellValue` function.
+ */
+function handleCsvFile(file, dataPathname, delimiter, dataKind) {
+  // Return a `getCellValue` function for the given CSV file.
+  let data = readCsv(dataPathname, delimiter)
+  if (data) {
+    return (c, r) => {
+      let value = '0.0'
+      try {
+        value = data[r] != null && data[r][c] != null ? cdbl(data[r][c]) : null
+      } catch (error) {
+        console.error(`${error.message} in ${dataPathname}`)
+      }
+      return value
+    }
+  } else {
+    throw new Error(`Direct ${dataKind} file ${file} could not be read`)
+  }
+}

packages/compile/src/generate/gen-const-list.js

@@ -0,0 +1,89 @@
+import { cartesianProductOf, cdbl } from '../_shared/helpers.js'
+import { isDimension, normalizeSubscripts, sub } from '../_shared/subscript.js'
+
+/**
+ * Generate code for a single element in a const list definition.
+ *
+ * @param {*} variable The `Variable` instance to process.
+ * @param {*} parsedEqn The parsed equation.
+ * @return {string[]} An array of strings containing the generated C code for the variable,
+ * one string per line of code.
+ */
+export function generateConstListElement(variable, parsedEqn) {
+  // In the "read variables" phase, const lists are expanded into separated variable
+  // definitions, so `variable` here will have `subscripts` that represent specific
+  // subscript indices in normalized order (alphabetized by parent dimension/family
+  // name).  However, we need to consult the LHS subscripts/dimensions, which will
+  // be in the original order from the model equation.
+  //
+  // In the following example,
+  // we have a 2D variable whose original dimensions are not in normal order:
+  //   DimA: A1, A2 ~~|
+  //   DimB: B1, B2, B3 ~~|
+  //   x[DimB, DimA] = 1, 2; 3, 4; 5, 6; ~~|
+  //
+  // The variable `x` will have been separated into:
+  //   x[B1,A1]
+  //   x[B1,A2]
+  //   x[B2,A1]
+  //   ...
+  //
+  // Each one will refer to a single element from the original const list.  To determine
+  // which element in the const list goes with which variable instance, we build an array
+  // of all subscript combinations and then find the index of the one that matches the
+  // combination used for the separated variable instance.
+  const lhsSubRefs = variable.parsedEqn.lhs.varDef.subscriptRefs
+  const lhsSubIds = lhsSubRefs.map(subRef => subRef.subId)
+  const subIdArrays = lhsSubIds.map(subOrDimId => {
+    if (isDimension(subOrDimId)) {
+      // Use the full array of subscripts (indexes) for the dimension at this position
+      return sub(subOrDimId).value
+    } else {
+      // This is a single subscript (index), so use an array with a single element
+      return [subOrDimId]
+    }
+  })
+
+  // Continuing with the above example, at this point we will have a 2D array:
+  //   [
+  //     [_b1,_b2,_b3],
+  //     [_a1,_a2]
+  //   ]
+  // We expand these into the set of all combinations of subscripts in the original
+  // order of the dimensions from the equation LHS.
+  const origCombos = cartesianProductOf(subIdArrays)
+
+  // Now we have the combinations in original order:
+  //   [_b1,_a1]
+  //   [_b1,_a2]
+  //   [_b2,_a1]
+  //   ...
+  // But we need to put them into normalized order so that we can find the index of
+  // `variable.subscripts` (which is already in normalized order).
+  const normalizedCombos = origCombos.map(normalizeSubscripts)
+
+  // Convert to strings to make matching easier.  Now we have the strings in normalized order:
+  //   [_a1,_b1]
+  //   [_a2,_b1]
+  //   [_a1,_b2]
+  //   ...
+  const comboStrings = normalizedCombos.map(combo => combo.map(subId => `[${subId}]`).join(''))
+
+  // Convert `variable.subscripts` into the same format so that we can do an array lookup,
+  // for example if this separated variable instance is x[_a2,_b1], this will be:
+  //   [_a2,_b1]
+  const lhsComboString = variable.subscripts.map(subId => `[${subId}]`).join('')
+
+  // Find the index of the combination that matches `variable.subscripts`
+  const constIndex = comboStrings.indexOf(lhsComboString)
+  if (constIndex < 0) {
+    throw new Error(`Failed to determine index of const list element for ${variable.refId}`)
+  }
+
+  // Determine the LHS and RHS of the const assignment
+  const lhsVarId = variable.varName
+  const lhsIndicesString = variable.subscripts.map(subId => `[${sub(subId).value}]`).join('')
+  const lhsRef = `${lhsVarId}${lhsIndicesString}`
+  const rhsConstValue = cdbl(parsedEqn.rhs.constants[constIndex].value)
+  return `  ${lhsRef} = ${rhsConstValue};`
+}