diff --git a/crates/parser/src/ast.rs b/crates/parser/src/ast.rs index d6b31b7cf..e0d153cc0 100644 --- a/crates/parser/src/ast.rs +++ b/crates/parser/src/ast.rs @@ -147,7 +147,7 @@ pub struct Assertion { #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct TestDefinition { pub description: Option, - pub network: String, + pub networks: Vec, pub creds: String, pub assertions: Vec, } diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index 148c1c40d..c4206aabc 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -212,7 +212,7 @@ pub fn parse(unparsed_file: &str) -> Result return Err(errors::ParserError::ParseError(e.to_string())), }; - let mut network: Option = None; + let mut networks: Vec = vec![]; let mut creds: Option = None; let mut description: Option = None; let mut assertions: Vec = vec![]; @@ -233,7 +233,10 @@ pub fn parse(unparsed_file: &str) -> Result { - network = Some(record.into_inner().as_str().to_owned()); + let mut pairs = record.into_inner(); + let file_path = get_pair(&mut pairs, "file_path")?.as_str(); + + networks.push(file_path.to_owned()); } Rule::creds => { let mut pairs = record.into_inner(); @@ -604,7 +607,7 @@ pub fn parse(unparsed_file: &str) -> Result Result", "ZNDSL file (.zndsl) describing the tests") .argument( - "[runningNetworkSpec]", + "[runningNetworksSpec...]", "Path to the network spec json, for using a running network for running the test", ) .action(test); @@ -488,8 +488,7 @@ async function spawn( */ async function test( testFile: string, - runningNetworkSpec: string | undefined, - _opts: any, + runningNetworksSpec: string[], ) { const opts = program.opts(); @@ -513,8 +512,8 @@ async function test( const configBasePath = path.dirname(testFile); const env = new Environment(new RelativeLoader([configBasePath])); - const temmplateContent = fs.readFileSync(testFile).toString(); - const content = env.renderString(temmplateContent, process.env); + const templateContent = fs.readFileSync(testFile).toString(); + const content = env.renderString(templateContent, process.env); const testName = getTestNameFromFileName(testFile); diff --git a/javascript/packages/orchestrator/src/network.ts b/javascript/packages/orchestrator/src/network.ts index e29cca48b..9ddcba12b 100644 --- a/javascript/packages/orchestrator/src/network.ts +++ b/javascript/packages/orchestrator/src/network.ts @@ -244,7 +244,7 @@ export class Network { const nodes = this.groups[nodeOrGroupName]; if (!nodes) - throw new Error(`Noode or Group: ${nodeOrGroupName} not present`); + throw new Error(`Node or Group: ${nodeOrGroupName} not present`); return nodes; } @@ -275,6 +275,35 @@ export class Network { } } + getNetworkInfo() { + return { + tmpDir: this.tmpDir, + chainSpecPath: this.chainSpecFullPath, + relay: this.relay.map((node: any) => { + const {name, wsUri, prometheusUri, userDefinedTypes} = node; + return {name, wsUri, prometheusUri, userDefinedTypes}; + }), + paras: Object.keys(this.paras).reduce((memo: any, paraId: any) => { + const {chainSpecPath, wasmPath, statePath} = this.paras[paraId]; + memo[paraId] = {chainSpecPath, wasmPath, statePath}; + memo[paraId].nodes = this.paras[paraId].nodes.map((node) => { + return {...node}; + }); + return memo; + }, {}), + nodesByName: Object.keys(this.nodesByName).reduce( + (memo: any, nodeName) => { + const {name, wsUri, prometheusUri, userDefinedTypes, parachainId} = + this.nodesByName[nodeName]; + memo[nodeName] = {name, wsUri, prometheusUri, userDefinedTypes}; + if (parachainId) memo[nodeName].parachainId = parachainId; + return memo; + }, + {}, + ), + } + } + // show links for access and debug showNetworkInfo(provider: string) { const logTable = new CreateLogTable({ diff --git a/javascript/packages/orchestrator/src/providers/client.ts b/javascript/packages/orchestrator/src/providers/client.ts index 722e96b14..ecabf131f 100644 --- a/javascript/packages/orchestrator/src/providers/client.ts +++ b/javascript/packages/orchestrator/src/providers/client.ts @@ -100,6 +100,5 @@ export function getClient(): Client { } export function setClient(c: Client) { - if (client) throw new Error("Client already initialized"); client = c; } diff --git a/javascript/packages/orchestrator/src/test-runner/index.ts b/javascript/packages/orchestrator/src/test-runner/index.ts index 3339846a7..3c0fb13cc 100644 --- a/javascript/packages/orchestrator/src/test-runner/index.ts +++ b/javascript/packages/orchestrator/src/test-runner/index.ts @@ -1,3 +1,5 @@ +import {setClient} from "../providers/client"; + const chai = require("chai"); import { decorators, @@ -15,6 +17,7 @@ import { Providers } from "../providers"; import { LaunchConfig, TestDefinition } from "../types"; import assertions from "./assertions"; import commands from "./commands"; +import {NetworkNode} from "../networkNode"; const DEFAULT_GLOBAL_TIMEOUT = 1200; // 20 mins @@ -44,16 +47,20 @@ export async function run( let suiteName: string = testName; if (testDef.description) suiteName += `( ${testDef.description} )`; - // read network file - let networkConfigFilePath = fs.existsSync(testDef.network) - ? testDef.network - : path.resolve(configBasePath, testDef.network); - const config: LaunchConfig = readNetworkConfig(networkConfigFilePath); + let networkConfigs: LaunchConfig[] = []; + for (let networkConfigFilePath of testDef.networks) { + // read network file + if (!fs.existsSync(networkConfigFilePath)) + networkConfigFilePath = path.resolve(configBasePath, networkConfigFilePath); + let networkConfig = readNetworkConfig(networkConfigFilePath); + + // set the provider + if (!networkConfig.settings) + networkConfig.settings = { provider, timeout: DEFAULT_GLOBAL_TIMEOUT }; + else networkConfig.settings.provider = provider; - // set the provider - if (!config.settings) - config.settings = { provider, timeout: DEFAULT_GLOBAL_TIMEOUT }; - else config.settings.provider = provider; + networkConfigs.push(networkConfig); + } // find creds file let credsFile = inCI ? "config" : testDef.creds; @@ -75,129 +82,123 @@ export async function run( if (credsFileExistInPath) creds = credsFileExistInPath + "/" + credsFile; } - if (!creds && config.settings.provider === "kubernetes") - throw new Error(`Invalid credential file path: ${credsFile}`); + for (let networkConfig of networkConfigs) { + if (!creds && networkConfig.settings.provider === "kubernetes") + throw new Error(`Invalid credential file path: ${credsFile}`); + } // create suite const suite = Suite.create(mocha.suite, suiteName); suite.beforeAll("launching", async function () { - const launchTimeout = config.settings?.timeout || 500; - this.timeout(launchTimeout * 1000); - try { - if (runningNetworkSpecPath) - console.log("runningNetworkSpecPath", runningNetworkSpecPath); - if (!runningNetworkSpecPath) { - console.log(`\t Launching network... this can take a while.`); - network = await start(creds!, config, { - spawnConcurrency: concurrency, - inCI, - silent, - }); - } else { - const runningNetworkSpec: any = require(runningNetworkSpecPath); - if (provider !== runningNetworkSpec.client.providerName) - throw new Error( - `Invalid provider, the provider set doesn't match with the running network definition`, - ); + for (let [networkConfigIdx, networkConfig] of networkConfigs.entries()) { + const launchTimeout = config.settings?.timeout || 500; + this.timeout(launchTimeout * 1000); + + let runningNetworkSpecPath = runningNetworksSpec[networkConfigIdx]; + try { + if (runningNetworkSpecPath) + console.log("runningNetworkSpecPath", runningNetworkSpecPath); + + let network: Network; + if (!runningNetworkSpecPath) { + console.log(`\t Launching network... this can take a while.`); + network = await start(creds!, networkConfig, { + spawnConcurrency: concurrency, + inCI, + silent, + }); + } else { + const runningNetworkSpec: any = require(runningNetworkSpecPath); + if (provider !== runningNetworkSpec.client.providerName) + throw new Error( + `Invalid provider, the provider set doesn't match with the running network definition`, + ); + + const { client, namespace, tmpDir } = runningNetworkSpec; + // initialize the Client + const initClient = Providers.get( + runningNetworkSpec.client.providerName, + ).initClient(client.configPath, namespace, tmpDir); + // initialize the network + network = rebuildNetwork(initClient, runningNetworkSpec); + } - const { client, namespace, tmpDir } = runningNetworkSpec; - // initialize the Client - const initClient = Providers.get( - runningNetworkSpec.client.providerName, - ).initClient(client.configPath, namespace, tmpDir); - // initialize the network - network = rebuildNetwork(initClient, runningNetworkSpec); + networks.push(network); + network.showNetworkInfo(networkConfig.settings.provider); + } catch (err) { + console.log( + `\n${decorators.red( + "Error launching the network!", + )} \t ${decorators.bright(err)}`, + ); + exitMocha(100); } - - network.showNetworkInfo(config.settings.provider); - - await sleep(5 * 1000); - return; - } catch (err) { - console.log( - `\n${decorators.red( - "Error launching the network!", - )} \t ${decorators.bright(err)}`, - ); - exitMocha(100); } + + await sleep(5 * 1000); + return; }); suite.afterAll("teardown", async function () { - this.timeout(180 * 1000); - if (network && !network.wasRunning) { - const logsPath = await network.dumpLogs(false); - const tests = this.test?.parent?.tests; - - if (tests) { - const failed = tests.filter((test) => { - return test.state !== "passed"; - }); - if (failed.length) { + for (let network of networks) { + this.timeout(180 * 1000); + if (network && !network.wasRunning) { + const logsPath = await network.dumpLogs(false); + const tests = this.test?.parent?.tests; + + if (tests) { + const failed = tests.filter((test) => { + return test.state !== "passed"; + }); + if (failed.length) { + console.log( + `\n\n\t${decorators.red("❌ One or more of your test failed...")}`, + ); + + switch (network.client.providerName) { + case "podman": + case "native": + console.log(`\n\t ${decorators.green("Deleting network")}`); + await network.stop(); + break; + case "kubernetes": + if (inCI) { + // keep pods running for 30 mins. + console.log( + `\n\t${decorators.red( + "One or more test failed, we will keep the namespace up for 30 more minutes", + )}`, + ); + await network.upsertCronJob(30); + } else { + console.log(`\n\t ${decorators.green("Deleting network")}`); + await network.stop(); + } + break; + } + } else { + // All test passed, just remove the network + console.log(`\n\t ${decorators.green("Deleting network")}`); + await network.stop(); + } + + // show logs console.log( - `\n\n\t${decorators.red("❌ One or more of your test failed...")}`, + `\n\n\t${decorators.magenta( + "📓 To see the full logs of the nodes please go to:", + )}`, ); - switch (network.client.providerName) { case "podman": case "native": - console.log(`\n\t ${decorators.green("Deleting network")}`); - await network.stop(); + console.log(`\n\t${decorators.magenta(logsPath)}`); break; case "kubernetes": if (inCI) { - // keep pods running for 30 mins. - console.log( - `\n\t${decorators.red( - "One or more test failed, we will keep the namespace up for 30 more minutes", - )}`, - ); - await network.upsertCronJob(30); - } else { - console.log(`\n\t ${decorators.green("Deleting network")}`); - await network.stop(); - } - break; - } - } else { - // All test passed, just remove the network - console.log(`\n\t ${decorators.green("Deleting network")}`); - await network.stop(); - } - - // show logs - console.log( - `\n\n\t${decorators.magenta( - "📓 To see the full logs of the nodes please go to:", - )}`, - ); - switch (network.client.providerName) { - case "podman": - case "native": - console.log(`\n\t${decorators.magenta(logsPath)}`); - break; - case "kubernetes": - if (inCI) { - // show links to grafana and also we need to move the logs to artifacts - const networkEndtime = new Date().getTime(); - for (const node of network.relay) { - const loki_url = getLokiUrl( - network.namespace, - node.name, - network.networkStartTime!, - networkEndtime, - ); - console.log( - `\t${decorators.magenta(node.name)}: ${decorators.green( - loki_url, - )}`, - ); - } - - for (const [paraId, parachain] of Object.entries(network.paras)) { - console.log(`\n\tParaId: ${decorators.magenta(paraId)}`); - for (const node of parachain?.nodes) { + // show links to grafana and also we need to move the logs to artifacts + const networkEndtime = new Date().getTime(); + for (const node of network.relay) { const loki_url = getLokiUrl( network.namespace, node.name, @@ -205,23 +206,40 @@ export async function run( networkEndtime, ); console.log( - `\t\t${decorators.magenta(node.name)}: ${decorators.green( + `\t${decorators.magenta(node.name)}: ${decorators.green( loki_url, )}`, ); } - } - // logs are also collaected as artifacts - console.log( - `\n\n\t ${decorators.yellow( - "📓 Logs are also available in the artifacts' pipeline in gitlab", - )}`, - ); - } else { - console.log(`\n\t${decorators.magenta(logsPath)}`); - } - break; + for (const [paraId, parachain] of Object.entries(network.paras)) { + console.log(`\n\tParaId: ${decorators.magenta(paraId)}`); + for (const node of parachain?.nodes) { + const loki_url = getLokiUrl( + network.namespace, + node.name, + network.networkStartTime!, + networkEndtime, + ); + console.log( + `\t\t${decorators.magenta(node.name)}: ${decorators.green( + loki_url, + )}`, + ); + } + } + + // logs are also collaected as artifacts + console.log( + `\n\n\t ${decorators.yellow( + "📓 Logs are also available in the artifacts' pipeline in gitlab", + )}`, + ); + } else { + console.log(`\n\t${decorators.magenta(logsPath)}`); + } + break; + } } } } @@ -244,7 +262,23 @@ export async function run( let testFn = generator(assertion.parsed.args); const test = new Test( assertion.original_line, - async () => await testFn(network, backchannelMap, configBasePath), + async () => { + // Find the first network that contains the node and run the test on it. + let nodeName = assertion.parsed.args.node_name!; + for (let network of networks) { + let nodes: NetworkNode[]; + try { + nodes = network.getNodes(nodeName); + } catch { + continue; + } + + setClient(network.client); + await testFn(network, backchannelMap, configBasePath); + return; + } + throw new Error(`Node or Group: ${nodeName} not present`); + }, ); suite.addTest(test); test.timeout(0); diff --git a/javascript/packages/orchestrator/src/types.ts b/javascript/packages/orchestrator/src/types.ts index a92f31f4c..4b6d871b7 100644 --- a/javascript/packages/orchestrator/src/types.ts +++ b/javascript/packages/orchestrator/src/types.ts @@ -282,7 +282,7 @@ export interface MultiAddressByNode { } export interface TestDefinition { - network: string; + networks: string[]; creds: string; description?: string; assertions: Assertion[];