diff --git a/packages/build/src/build/build.ts b/packages/build/src/build/build.ts index e37526d6..8cd33709 100644 --- a/packages/build/src/build/build.ts +++ b/packages/build/src/build/build.ts @@ -73,16 +73,14 @@ export async function build(mode: BuildMode, options: BuildOptions): Promise input.varName), - outputVarNames: modelSpec.outputs.map(output => output.varName), - externalDatfiles: modelSpec.datFiles, - ...modelSpec.options - } - const specPath = joinPath(config.prepDir, 'spec.json') - await writeFile(specPath, JSON.stringify(specJson, null, 2)) + // Write the spec file + const specJson = { + inputVarNames: modelSpec.inputs.map(input => input.varName), + outputVarNames: modelSpec.outputs.map(output => output.varName), + externalDatfiles: modelSpec.datFiles, + ...modelSpec.options + } + const specPath = joinPath(config.prepDir, 'spec.json') + await writeFile(specPath, JSON.stringify(specJson, null, 2)) - // Read the hash from the last successful model build, if available - const modelHashPath = joinPath(config.prepDir, 'model-hash.txt') - let previousModelHash: string - if (existsSync(modelHashPath)) { - previousModelHash = readFileSync(modelHashPath, 'utf8') - } else { - previousModelHash = 'NONE' - } + // Read the hash from the last successful model build, if available + let previousModelHash: string + if (existsSync(modelHashPath)) { + previousModelHash = readFileSync(modelHashPath, 'utf8') + } else { + previousModelHash = 'NONE' + } - // The code gen and Wasm build steps are time consuming, so we avoid rebuilding - // it if the build input files are unchanged since the last successful build - const inputFilesHash = await computeInputFilesHash(config) - let needModelGen: boolean - if (options.forceModelGen === true) { - needModelGen = true - } else { - const hashMismatch = inputFilesHash !== previousModelHash - needModelGen = hashMismatch - } + // The code gen and Wasm build steps are time consuming, so we avoid rebuilding + // it if the build input files are unchanged since the last successful build + const inputFilesHash = await computeInputFilesHash(config) + let needModelGen: boolean + if (options.forceModelGen === true) { + needModelGen = true + } else { + const hashMismatch = inputFilesHash !== previousModelHash + needModelGen = hashMismatch + } - let succeeded = true - try { if (needModelGen) { // Generate the model await generateModel(context, plugins) diff --git a/packages/build/src/build/impl/gen-model.ts b/packages/build/src/build/impl/gen-model.ts index bb9a23d1..fadfeca2 100644 --- a/packages/build/src/build/impl/gen-model.ts +++ b/packages/build/src/build/impl/gen-model.ts @@ -96,7 +96,14 @@ async function preprocessMdl( // Use SDE to preprocess the model to strip anything that's not needed to build it const command = sdeCmdPath const args = ['generate', '--preprocess', 'processed.mdl'] - await context.spawnChild(prepDir, command, args) + const ppOutput = await context.spawnChild(prepDir, command, args, { + // The default error message from `spawnChild` is not very informative, so the + // following allows us to throw our own error + ignoreError: true + }) + if (ppOutput.exitCode !== 0) { + throw new Error(`Failed to preprocess mdl file: 'sde generate' command failed (code=${ppOutput.exitCode})`) + } // Copy the processed file back to the prep directory await copyFile(joinPath(prepDir, 'build', 'processed.mdl'), joinPath(prepDir, 'processed.mdl')) @@ -148,9 +155,9 @@ async function flattenMdls( log('error', ` ${line}`) } } - throw new Error(`Flatten command failed (code=${output.exitCode})`) + throw new Error(`Failed to flatten mdl files: 'sde flatten' command failed (code=${output.exitCode})`) } else if (output.exitCode !== 0) { - throw new Error(`Flatten command failed (code=${output.exitCode})`) + throw new Error(`Failed to flatten mdl files: 'sde flatten' command failed (code=${output.exitCode})`) } // Copy the processed file back to the prep directory @@ -167,11 +174,17 @@ async function generateC(context: BuildContext, sdeDir: string, sdeCmdPath: stri // dimensions and variables (`--list`) const command = sdeCmdPath const gencArgs = ['generate', '--genc', '--list', '--spec', 'spec.json', 'processed'] - await context.spawnChild(prepDir, command, gencArgs, { + const gencOutput = await context.spawnChild(prepDir, command, gencArgs, { // By default, ignore lines that start with "WARNING: Data for" since these are often harmless // TODO: Don't filter by default, but make it configurable // ignoredMessageFilter: 'WARNING: Data for' + // The default error message from `spawnChild` is not very informative, so the + // following allows us to throw our own error + ignoreError: true }) + if (gencOutput.exitCode !== 0) { + throw new Error(`Failed to generate C code: 'sde generate' command failed (code=${gencOutput.exitCode})`) + } // Copy SDE's supporting C files into the build directory const buildDir = joinPath(prepDir, 'build') diff --git a/packages/build/tests/_shared/submodel1.mdl b/packages/build/tests/_shared/submodel1.mdl new file mode 100644 index 00000000..070daabf --- /dev/null +++ b/packages/build/tests/_shared/submodel1.mdl @@ -0,0 +1,20 @@ +{UTF-8} + +X = TIME ~~| + +Y = 0 + ~ [-10,10,0.1] + ~ + | + +Z = X + Y + ~ + ~ This is used for the "flatten" test in `build-prod.spec.ts`. \ + It intentionally has a different definition than the Z equation \ + in `submodel2.mdl` to trigger a flatten error. + | + +INITIAL TIME = 2000 ~~| +FINAL TIME = 2100 ~~| +TIME STEP = 1 ~~| +SAVEPER = TIME STEP ~~| diff --git a/packages/build/tests/_shared/submodel2.mdl b/packages/build/tests/_shared/submodel2.mdl new file mode 100644 index 00000000..1235d8ec --- /dev/null +++ b/packages/build/tests/_shared/submodel2.mdl @@ -0,0 +1,13 @@ +{UTF-8} + +Z = 10 + ~ + ~ This is used for the "flatten" test in `build-prod.spec.ts`. \ + It intentionally has a different definition than the Z equation \ + in `submodel1.mdl` to trigger a flatten error. + | + +INITIAL TIME = 2000 ~~| +FINAL TIME = 2100 ~~| +TIME STEP = 1 ~~| +SAVEPER = TIME STEP ~~| diff --git a/packages/build/tests/build/build-prod.spec.ts b/packages/build/tests/build/build-prod.spec.ts index 6d5c5f7b..67e25577 100644 --- a/packages/build/tests/build/build-prod.spec.ts +++ b/packages/build/tests/build/build-prod.spec.ts @@ -129,4 +129,131 @@ describe('build in production mode', () => { 'plugin 2: postBuild' ]) }) + + describe('should fail if plugin throws error', () => { + async function buildSample(plugin: Plugin): Promise { + const userConfig: UserConfig = { + rootDir: resolvePath(__dirname, '..'), + prepDir: resolvePath(__dirname, 'sde-prep'), + modelFiles: [resolvePath(__dirname, '..', '_shared', 'sample.mdl')], + modelSpec: async () => { + return modelSpec + }, + plugins: [plugin] + } + + const result = await build('production', buildOptions(userConfig)) + if (result.isOk()) { + throw new Error('Expected err result but got: ' + result.value) + } + + return result.error.message + } + + async function verify(pluginFunc: keyof Plugin): Promise { + const plugin = {} as Plugin + plugin[pluginFunc] = async () => { + throw new Error(`${pluginFunc} error`) + } + const msg = await buildSample(plugin) + expect(msg).toBe(`${pluginFunc} error`) + } + + it('in init', async () => verify('init')) + it('in preGenerate', async () => verify('preGenerate')) + it('in preProcessMdl', async () => verify('preProcessMdl')) + it('in postProcessMdl', async () => verify('postProcessMdl')) + it('in preGenerateC', async () => verify('preGenerateC')) + it('in postGenerateC', async () => verify('postGenerateC')) + it('in postGenerate', async () => verify('postGenerate')) + it('in postBuild', async () => verify('postBuild')) + }) + + // TODO: Not sure how to cause the preprocessor to fail, so this test is + // skipped for now + it.skip('should fail if preprocess step throws an error', async () => { + const modelSpec: ModelSpec = { + inputs: [{ varName: 'Y', defaultValue: 0, minValue: -10, maxValue: 10 }], + outputs: [{ varName: 'Z' }], + datFiles: [] + } + + const mdlDir = resolvePath(__dirname, '..', '_shared') + const userConfig: UserConfig = { + rootDir: resolvePath(__dirname, '..'), + prepDir: resolvePath(__dirname, 'sde-prep'), + modelFiles: [resolvePath(mdlDir, 'sample.mdl')], + modelSpec: async () => { + return modelSpec + } + } + + const result = await build('production', buildOptions(userConfig)) + if (result.isOk()) { + throw new Error('Expected err result but got: ' + result.value) + } + + // TODO: This error message isn't helpful, but it's due to the fact that + // the `preprocess` function spawns an `sde` process rather than calling + // into the compiler directly. Once we improve it to call into the + // compiler, we can improve the error message. + expect(result.error.message).toBe(`Failed to flatten mdl files: 'sde flatten' command failed (code=1)`) + }) + + it('should fail if flatten step throws an error', async () => { + const modelSpec: ModelSpec = { + inputs: [{ varName: 'Y', defaultValue: 0, minValue: -10, maxValue: 10 }], + outputs: [{ varName: 'Z' }], + datFiles: [] + } + + const mdlDir = resolvePath(__dirname, '..', '_shared') + const userConfig: UserConfig = { + rootDir: resolvePath(__dirname, '..'), + prepDir: resolvePath(__dirname, 'sde-prep'), + modelFiles: [resolvePath(mdlDir, 'submodel1.mdl'), resolvePath(mdlDir, 'submodel2.mdl')], + modelSpec: async () => { + return modelSpec + } + } + + const result = await build('production', buildOptions(userConfig)) + if (result.isOk()) { + throw new Error('Expected err result but got: ' + result.value) + } + + // TODO: This error message isn't helpful, but it's due to the fact that + // the `flatten` function spawns an `sde` process rather than calling + // into the compiler directly. Once we improve it to call into the + // compiler, we can improve the error message. + expect(result.error.message).toBe(`Failed to flatten mdl files: 'sde flatten' command failed (code=1)`) + }) + + it('should fail if generate step throws an error when dat file cannot be read', async () => { + const modelSpec: ModelSpec = { + inputs: [{ varName: 'Y', defaultValue: 0, minValue: -10, maxValue: 10 }], + outputs: [{ varName: 'Z' }], + datFiles: ['unknown.dat'] + } + + const userConfig: UserConfig = { + rootDir: resolvePath(__dirname, '..'), + prepDir: resolvePath(__dirname, 'sde-prep'), + modelFiles: [resolvePath(__dirname, '..', '_shared', 'sample.mdl')], + modelSpec: async () => { + return modelSpec + } + } + + const result = await build('production', buildOptions(userConfig)) + if (result.isOk()) { + throw new Error('Expected err result but got: ' + result.value) + } + + // TODO: This error message isn't helpful, but it's due to the fact that + // the `generateC` function spawns an `sde` process rather than calling + // into the compiler directly. Once we improve it to call into the + // compiler, the error message here should be the one from `readDat`. + expect(result.error.message).toBe(`Failed to generate C code: 'sde generate' command failed (code=1)`) + }) }) diff --git a/packages/cli/src/sde-bundle.js b/packages/cli/src/sde-bundle.js index 3ea7ee1e..05141a0d 100644 --- a/packages/cli/src/sde-bundle.js +++ b/packages/cli/src/sde-bundle.js @@ -42,7 +42,8 @@ export let bundle = async (configPath, verbose) => { process.exit(result.exitCode) } else { // Exit with a non-zero code if any step failed - console.error(`ERROR: ${result.error.message}\n`) + console.error(result.error) + console.error() process.exit(1) } } diff --git a/packages/cli/src/sde-dev.js b/packages/cli/src/sde-dev.js index 2c731452..5ace94e2 100644 --- a/packages/cli/src/sde-dev.js +++ b/packages/cli/src/sde-dev.js @@ -40,7 +40,8 @@ export let dev = async (configPath, verbose) => { // Exit with a non-zero code if any step failed, otherwise keep the // builder process alive if (!result.isOk()) { - console.error(`ERROR: ${result.error.message}\n`) + console.error(result.error) + console.error() process.exit(1) } } diff --git a/packages/cli/src/sde-generate.js b/packages/cli/src/sde-generate.js index cdbd68e5..d272dd76 100644 --- a/packages/cli/src/sde-generate.js +++ b/packages/cli/src/sde-generate.js @@ -43,8 +43,14 @@ export let builder = { alias: 'r' } } -export let handler = argv => { - generate(argv.model, argv) +export let handler = async argv => { + try { + await generate(argv.model, argv) + } catch (e) { + console.error(e) + console.error() + process.exit(1) + } } export let generate = async (model, opts) => { diff --git a/packages/compile/src/_shared/helpers.js b/packages/compile/src/_shared/helpers.js index 5b867a03..38bdc194 100644 --- a/packages/compile/src/_shared/helpers.js +++ b/packages/compile/src/_shared/helpers.js @@ -375,7 +375,3 @@ export let vlog = (title, value, depth = 1) => { console.trace() } } -export let abend = error => { - console.error(error) - process.exit(1) -} diff --git a/packages/compile/src/_shared/read-dat.js b/packages/compile/src/_shared/read-dat.js index 64e0bd72..abdd7e9e 100644 --- a/packages/compile/src/_shared/read-dat.js +++ b/packages/compile/src/_shared/read-dat.js @@ -35,8 +35,16 @@ export async function readDat(pathname, prefix = '') { } } - return new Promise(resolve => { - let stream = byline(fs.createReadStream(pathname, 'utf8')) + return new Promise((resolve, reject) => { + // Errors from the read stream aren't propagated by the byline package + // so we attach the error handler to `readStream` rather than to `stream` + let readStream = fs.createReadStream(pathname, 'utf8') + let stream = byline(readStream) + readStream.on('error', e => { + stream.destroy() + reject(new Error(`Failed to read dat file: ${e.message}`)) + }) + stream.on('data', line => { let values = splitDatLine(line) if (values.length === 1) { @@ -63,6 +71,7 @@ export async function readDat(pathname, prefix = '') { lineNum++ // if (lineNum % 1e5 === 0) console.log(num(lineNum).format('0,0')) }) + stream.on('end', () => { addValues() resolve(log) diff --git a/packages/compile/src/_shared/read-dat.spec.ts b/packages/compile/src/_shared/read-dat.spec.ts new file mode 100644 index 00000000..0280bc4d --- /dev/null +++ b/packages/compile/src/_shared/read-dat.spec.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from 'vitest' + +import { readDat } from './read-dat' + +describe('readDat', () => { + it('should fail if file does not exist or cannot be read', async () => { + await expect(() => readDat('unknown_file.dat')).rejects.toThrow( + `Failed to read dat file: ENOENT: no such file or directory, open 'unknown_file.dat'` + ) + }) +}) diff --git a/packages/compile/src/generate/code-gen.js b/packages/compile/src/generate/code-gen.js index 615693b1..e27c79aa 100644 --- a/packages/compile/src/generate/code-gen.js +++ b/packages/compile/src/generate/code-gen.js @@ -1,6 +1,6 @@ import * as R from 'ramda' -import { asort, lines, strlist, abend, mapIndexed } from '../_shared/helpers.js' +import { asort, lines, strlist, mapIndexed } from '../_shared/helpers.js' import { sub, allDimensions, allMappings, subscriptFamilies } from '../_shared/subscript.js' import Model from '../model/model.js' @@ -37,30 +37,26 @@ let codeGenerator = (parsedModel, opts) => { function generate() { // Read variables and subscript ranges from the model parse tree. // This is the main entry point for code generation and is called just once. - try { - Model.read(parsedModel, spec, extData, directData, modelDirname) - // In list mode, print variables to the console instead of generating code. - if (operations.includes('printRefIdTest')) { - Model.printRefIdTest() - } - if (operations.includes('printRefGraph')) { - Model.printRefGraph(opts.varname) - } - if (operations.includes('convertNames')) { - // Do not generate output, but leave the results of model analysis. - } - if (operations.includes('generateC')) { - // Generate code for each variable in the proper order. - let code = emitDeclCode() - code += emitInitLookupsCode() - code += emitInitConstantsCode() - code += emitInitLevelsCode() - code += emitEvalCode() - code += emitIOCode() - return code - } - } catch (e) { - abend(e) + Model.read(parsedModel, spec, extData, directData, modelDirname) + // In list mode, print variables to the console instead of generating code. + if (operations.includes('printRefIdTest')) { + Model.printRefIdTest() + } + if (operations.includes('printRefGraph')) { + Model.printRefGraph(opts.varname) + } + if (operations.includes('convertNames')) { + // Do not generate output, but leave the results of model analysis. + } + if (operations.includes('generateC')) { + // Generate code for each variable in the proper order. + let code = emitDeclCode() + code += emitInitLookupsCode() + code += emitInitConstantsCode() + code += emitInitLevelsCode() + code += emitEvalCode() + code += emitIOCode() + return code } } diff --git a/packages/compile/src/generate/gen-code-c.spec.ts b/packages/compile/src/generate/gen-code-c.spec.ts new file mode 100644 index 00000000..88c330e9 --- /dev/null +++ b/packages/compile/src/generate/gen-code-c.spec.ts @@ -0,0 +1,100 @@ +import path from 'node:path' + +import { describe, expect, it } from 'vitest' + +import { readXlsx, resetHelperState } from '../_shared/helpers' +import { resetSubscriptsAndDimensions } from '../_shared/subscript' + +import Model from '../model/model' + +import { parseInlineVensimModel } from '../_tests/test-support' +import { generateCode } from './code-gen' + +type ExtData = Map> +type DirectDataSpec = Map + +function readInlineModelAndGenerateC( + mdlContent: string, + opts?: { + modelDir?: string + extData?: ExtData + directDataSpec?: DirectDataSpec + inputVarNames?: string[] + outputVarNames?: string[] + } +): string { + // XXX: These steps are needed due to subs/dims and variables being in module-level storage + resetHelperState() + resetSubscriptsAndDimensions() + Model.resetModelState() + + let spec + if (opts?.inputVarNames || opts?.outputVarNames) { + spec = { + inputVarNames: opts?.inputVarNames || [], + outputVarNames: opts?.outputVarNames || [] + } + } else { + spec = {} + } + + const directData = new Map() + if (opts?.modelDir && opts?.directDataSpec) { + for (const [file, xlsxFilename] of opts.directDataSpec.entries()) { + const xlsxPath = path.join(opts.modelDir, xlsxFilename) + directData.set(file, readXlsx(xlsxPath)) + } + } + + const parsedModel = parseInlineVensimModel(mdlContent, opts?.modelDir) + return generateCode(parsedModel, { + spec, + operations: ['generateC'], + extData: opts?.extData, + directData, + modelDirname: opts?.modelDir + }) +} + +describe('generateCode (Vensim -> C)', () => { + it('should work for simple model', () => { + const mdl = ` + x = 1 ~~| + y = :NOT: x ~~| + ` + const code = readInlineModelAndGenerateC(mdl) + expect(code).toMatch('#include "sde.h"') + }) + + it('should throw error when unknown input variable name is provided in spec file', () => { + const mdl = ` + DimA: A1, A2 ~~| + A[DimA] = 10, 20 ~~| + B = 30 ~~| + ` + expect(() => + readInlineModelAndGenerateC(mdl, { + inputVarNames: ['C'], + outputVarNames: ['A[A1]'] + }) + ).toThrow( + 'The input variable _c was declared in spec.json, but no matching variable was found in the model or external data sources' + ) + }) + + it('should throw error when cyclic dependency is detected for aux variable', () => { + const mdl = ` + X = Y ~~| + Y = X + 1 ~~| + ` + expect(() => readInlineModelAndGenerateC(mdl)).toThrow('Found cyclic dependency during toposort:\n_y →\n_x\n_y') + }) + + it('should throw error when cyclic dependency is detected for init variable', () => { + const mdl = ` + X = INITIAL(Y) ~~| + Y = X + 1 ~~| + ` + expect(() => readInlineModelAndGenerateC(mdl)).toThrow('Found cyclic dependency during toposort:\n_y →\n_x\n_y') + }) +}) diff --git a/packages/compile/src/model/model.js b/packages/compile/src/model/model.js index ba9e1a57..885ba8fe 100644 --- a/packages/compile/src/model/model.js +++ b/packages/compile/src/model/model.js @@ -454,9 +454,7 @@ function removeUnusedVariables(spec) { recordUsedVariable(refVar) recordRefsOfVariable(refVar) } else { - console.error(`No var found for ${refId}`) - console.error(v) - process.exit(1) + throw new Error(`No var found for ${refId} when recording references for ${v.varName}`) } } } @@ -877,13 +875,7 @@ function sortVarsOfType(varType) { // Sort into an lhs dependency list. if (PRINT_AUX_GRAPH) printDepsGraph(graph, 'AUX') if (PRINT_LEVEL_GRAPH) printDepsGraph(graph, 'LEVEL') - let deps - try { - deps = toposort(graph).reverse() - } catch (e) { - console.error(e.message) - process.exit(1) - } + let deps = toposort(graph).reverse() // Turn the dependency-sorted var name list into a var list. let sortedVars = varsOfType( @@ -978,13 +970,7 @@ function sortInitVars() { if (PRINT_INIT_GRAPH) printDepsGraph(graph, 'INIT') // Sort into a reference id dependency list. - let deps - try { - deps = toposort(graph).reverse() - } catch (e) { - console.error(e.message) - process.exit(1) - } + let deps = toposort(graph).reverse() // Turn the reference id list into a var list. let sortedVars = R.map(refId => varWithRefId(refId), deps) diff --git a/packages/compile/src/model/toposort.js b/packages/compile/src/model/toposort.js index 52ca88e1..b3e3c95b 100644 --- a/packages/compile/src/model/toposort.js +++ b/packages/compile/src/model/toposort.js @@ -42,7 +42,7 @@ function toposort(nodes, edges) { } catch (e) { nodeRep = '' } - throw new Error('toposort cyclic dependency:\n' + [...predecessors].join(' →\n') + nodeRep) + throw new Error('Found cyclic dependency during toposort:\n' + [...predecessors].join(' →\n') + nodeRep) } if (!nodesHash.has(node)) { diff --git a/packages/plugin-wasm/src/plugin.ts b/packages/plugin-wasm/src/plugin.ts index d38d2f6f..731d0503 100644 --- a/packages/plugin-wasm/src/plugin.ts +++ b/packages/plugin-wasm/src/plugin.ts @@ -135,8 +135,14 @@ async function buildWasm( // context.log('verbose', ` emcc args: ${args}`) - await context.spawnChild(prepDir, command, args, { + const emccOutput = await context.spawnChild(prepDir, command, args, { // Ignore unhelpful Emscripten SDK cache messages - ignoredMessageFilter: 'cache:INFO' + ignoredMessageFilter: 'cache:INFO', + // The default error message from `spawnChild` is not very informative, so the + // following allows us to throw our own error + ignoreError: true }) + if (emccOutput.exitCode !== 0) { + throw new Error(`Failed to compile C model to WebAssembly: emcc command failed (code=${emccOutput.exitCode})`) + } }