Skip to content

Commit 00fae1b

Browse files
dbanks12dbanks12
authored
feat: benchmark avm simulator (#12985)
This PR does _not_ integrate benchmarks into CI. It updates the simulator tests to support benchmarking, adds a bench test, and pretty-prints simulator benchmarks. ## AvmSimulator - instrCounter tracked in machine state. When a nested call returns, its parent absorbs its instrCounter. This might seem weird, but it's the metric we want. If it feels too wrong, i'm fine having both an instrCounter and a totalInstrCounter. Or we can rename this one totalInstrCounter for clarity. ## PublicTxSimulationTester, SimpleContractDataSource - SimpleContractDataSource now tracks contract & function names so that `getDebugFunctionName()` works properly in simulator tests - Tester only creates a single PublicTxSimulator that is used for all simulations instead of one per simulation - Test can create a `TestExecutorMetrics` and pass it into `PublicTxSimulationTester` constructor so that many test cases can aggregate metrics into the same class. ## Metrics / Benchmarking I opted _not_ to use the telemetry based benchmarking used by `e2e_block_building.test.ts`. Instead, I created a custom `TestExecutorMetrics` for benchmarking the simulator in exactly the way that works for us. We can easily add `toGithubActionsBenchmark()` adapter function if it is valuable. Running the tests with `BENCH_OUTPUT_MD` set will output the results to the specified markdown file. Running them without that env var set will `log.info` them. ## New AMM test isolated to public simulation for measurements This is brittle. It gives us measurements, but will break if any changes are made to AMM. ![image](https://github.com/user-attachments/assets/abffd658-5b79-430a-9a68-822ba911e997) ![image](https://github.com/user-attachments/assets/78816f68-c470-41d8-991c-731e512cf1a1) ![image](https://github.com/user-attachments/assets/2d8f8782-bb25-4928-b36e-2b584098834d)
1 parent 69f426e commit 00fae1b

40 files changed

+1382
-301
lines changed

‎yarn-project/bb-prover/src/avm_proving_tests/avm_check_circuit1.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ describe('AVM WitGen & Circuit – check circuit', () => {
1919
let tester: AvmProvingTester;
2020

2121
beforeEach(async () => {
22-
tester = await AvmProvingTester.create(/*checkCircuitOnly*/ true);
22+
tester = await AvmProvingTester.new(/*checkCircuitOnly*/ true);
2323
avmTestContractInstance = await tester.registerAndDeployContract(
2424
/*constructorArgs=*/ [],
2525
/*deployer=*/ AztecAddress.fromNumber(420),

‎yarn-project/bb-prover/src/avm_proving_tests/avm_check_circuit2.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ describe('AVM WitGen & Circuit – check circuit', () => {
1313
let tester: AvmProvingTester;
1414

1515
beforeEach(async () => {
16-
tester = await AvmProvingTester.create(/*checkCircuitOnly*/ true);
16+
tester = await AvmProvingTester.new(/*checkCircuitOnly*/ true);
1717
avmTestContractInstance = await tester.registerAndDeployContract(
1818
/*constructorArgs=*/ [],
1919
/*deployer=*/ AztecAddress.fromNumber(420),
@@ -44,7 +44,7 @@ describe('AVM WitGen & Circuit – check circuit', () => {
4444
// FIXME(dbanks12): fails with "Lookup PERM_MAIN_ALU failed."
4545
it.skip('top-level exceptional halts due to a non-existent contract in app-logic and teardown', async () => {
4646
// don't insert contracts into trees, and make sure retrieval fails
47-
const tester = await AvmProvingTester.create(/*checkCircuitOnly=*/ true);
47+
const tester = await AvmProvingTester.new(/*checkCircuitOnly=*/ true);
4848
await tester.simProveVerify(
4949
sender,
5050
/*setupCalls=*/ [],

‎yarn-project/bb-prover/src/avm_proving_tests/avm_check_circuit3.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ describe('AVM WitGen & Circuit – check circuit', () => {
1313
let tester: AvmProvingTester;
1414

1515
beforeEach(async () => {
16-
tester = await AvmProvingTester.create(/*checkCircuitOnly*/ true);
16+
tester = await AvmProvingTester.new(/*checkCircuitOnly*/ true);
1717
avmTestContractInstance = await tester.registerAndDeployContract(
1818
/*constructorArgs=*/ [],
1919
/*deployer=*/ AztecAddress.fromNumber(420),

‎yarn-project/bb-prover/src/avm_proving_tests/avm_contract_class_limits.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ describe('AVM WitGen & Circuit – check circuit - contract class limits', () =>
1515
let avmTestContractAddress: AztecAddress;
1616

1717
beforeEach(async () => {
18-
tester = await AvmProvingTester.create(/*checkCircuitOnly=*/ true);
18+
tester = await AvmProvingTester.new(/*checkCircuitOnly=*/ true);
1919
// create enough unique contract classes to hit the limit
2020
instances = [];
2121
for (let i = 0; i <= MAX_PUBLIC_CALLS_TO_UNIQUE_CONTRACT_CLASS_IDS; i++) {

‎yarn-project/bb-prover/src/avm_proving_tests/avm_contract_updates.test.ts‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { Fr } from '@aztec/foundation/fields';
22
import { AvmTestContractArtifact } from '@aztec/noir-contracts.js/AvmTest';
33
import { ProtocolContractAddress } from '@aztec/protocol-contracts';
4-
import { DEFAULT_BLOCK_NUMBER } from '@aztec/simulator/public/fixtures';
4+
import { defaultGlobals } from '@aztec/simulator/public/fixtures';
55
import { AztecAddress } from '@aztec/stdlib/aztec-address';
66
import type { ContractInstanceWithAddress } from '@aztec/stdlib/contract';
77
import { ScheduledDelayChange, ScheduledValueChange, SharedMutableValuesWithHash } from '@aztec/stdlib/shared-mutable';
@@ -43,7 +43,8 @@ describe.skip('AVM WitGen & Circuit - contract updates', () => {
4343
async () => {
4444
// Contract was not originally the avmTestContract
4545
const originalClassId = new Fr(27);
46-
const tester = await AvmProvingTester.create(/*checkCircuitOnly*/ true);
46+
const globals = defaultGlobals();
47+
const tester = await AvmProvingTester.new(/*checkCircuitOnly*/ true, globals);
4748

4849
avmTestContractInstance = await tester.registerAndDeployContract(
4950
/*constructorArgs=*/ [],
@@ -59,7 +60,7 @@ describe.skip('AVM WitGen & Circuit - contract updates', () => {
5960
avmTestContractInstance.address,
6061
avmTestContractInstance.originalContractClassId,
6162
avmTestContractInstance.currentContractClassId,
62-
DEFAULT_BLOCK_NUMBER,
63+
globals.blockNumber.toNumber(),
6364
);
6465

6566
await tester.simProveVerify(
@@ -80,8 +81,8 @@ describe.skip('AVM WitGen & Circuit - contract updates', () => {
8081
async () => {
8182
// Contract was not originally the avmTestContract
8283
const originalClassId = new Fr(27);
83-
84-
const tester = await AvmProvingTester.create(/*checkCircuitOnly*/ true);
84+
const globals = defaultGlobals();
85+
const tester = await AvmProvingTester.new(/*checkCircuitOnly*/ true, globals);
8586
avmTestContractInstance = await tester.registerAndDeployContract(
8687
/*constructorArgs=*/ [],
8788
sender,
@@ -96,7 +97,7 @@ describe.skip('AVM WitGen & Circuit - contract updates', () => {
9697
avmTestContractInstance.address,
9798
avmTestContractInstance.originalContractClassId,
9899
avmTestContractInstance.currentContractClassId,
99-
DEFAULT_BLOCK_NUMBER + 1,
100+
globals.blockNumber.toNumber() + 1,
100101
);
101102

102103
await expect(
@@ -120,7 +121,8 @@ describe.skip('AVM WitGen & Circuit - contract updates', () => {
120121
// Contract was not originally the avmTestContract
121122
const newClassId = new Fr(27);
122123

123-
const tester = await AvmProvingTester.create(/*checkCircuitOnly*/ true);
124+
const globals = defaultGlobals();
125+
const tester = await AvmProvingTester.new(/*checkCircuitOnly*/ true, globals);
124126
avmTestContractInstance = await tester.registerAndDeployContract(
125127
/*constructorArgs=*/ [],
126128
sender,
@@ -134,7 +136,7 @@ describe.skip('AVM WitGen & Circuit - contract updates', () => {
134136
avmTestContractInstance.address,
135137
avmTestContractInstance.currentContractClassId,
136138
newClassId,
137-
DEFAULT_BLOCK_NUMBER + 1,
139+
globals.blockNumber.toNumber() + 1,
138140
);
139141

140142
await tester.simProveVerify(
@@ -156,7 +158,8 @@ describe.skip('AVM WitGen & Circuit - contract updates', () => {
156158
// Contract was not originally the avmTestContract
157159
const newClassId = new Fr(27);
158160

159-
const tester = await AvmProvingTester.create(/*checkCircuitOnly*/ true);
161+
const globals = defaultGlobals();
162+
const tester = await AvmProvingTester.new(/*checkCircuitOnly*/ true, globals);
160163
avmTestContractInstance = await tester.registerAndDeployContract(
161164
/*constructorArgs=*/ [],
162165
sender,
@@ -170,7 +173,7 @@ describe.skip('AVM WitGen & Circuit - contract updates', () => {
170173
avmTestContractInstance.address,
171174
avmTestContractInstance.currentContractClassId,
172175
newClassId,
173-
DEFAULT_BLOCK_NUMBER - 1,
176+
globals.blockNumber.toNumber() - 1,
174177
);
175178

176179
await expect(

‎yarn-project/bb-prover/src/avm_proving_tests/avm_proven_amm.test.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ describe('AVM Witgen & Circuit apps tests: AMM', () => {
1515
jest.setTimeout(TIMEOUT);
1616
const admin = AztecAddress.fromNumber(42);
1717
const liquidityProvider = AztecAddress.fromNumber(111);
18-
const otherLiqduidityProvider = AztecAddress.fromNumber(222);
18+
const otherLiquidityProvider = AztecAddress.fromNumber(222);
1919
const swapper = AztecAddress.fromNumber(333);
2020

2121
let token0: ContractInstanceWithAddress;
@@ -25,7 +25,7 @@ describe('AVM Witgen & Circuit apps tests: AMM', () => {
2525
let tester: AvmProvingTester;
2626

2727
beforeEach(async () => {
28-
tester = await AvmProvingTester.create(/*checkCircuitOnly*/ true);
28+
tester = await AvmProvingTester.new(/*checkCircuitOnly*/ true);
2929
});
3030

3131
// TODO(dbanks12): add tester support for authwit and finish implementing this test
@@ -52,8 +52,8 @@ describe('AVM Witgen & Circuit apps tests: AMM', () => {
5252

5353
await mint(/*to=*/ liquidityProvider, /*amount=*/ INITIAL_TOKEN_BALANCE, token0);
5454
await mint(/*to=*/ liquidityProvider, /*amount=*/ INITIAL_TOKEN_BALANCE, token1);
55-
await mint(/*to=*/ otherLiqduidityProvider, /*amount=*/ INITIAL_TOKEN_BALANCE, token0);
56-
await mint(/*to=*/ otherLiqduidityProvider, /*amount=*/ INITIAL_TOKEN_BALANCE, token1);
55+
await mint(/*to=*/ otherLiquidityProvider, /*amount=*/ INITIAL_TOKEN_BALANCE, token0);
56+
await mint(/*to=*/ otherLiquidityProvider, /*amount=*/ INITIAL_TOKEN_BALANCE, token1);
5757
await mint(/*to=*/ swapper, /*amount=*/ INITIAL_TOKEN_BALANCE, token0);
5858

5959
//const ammBalancesBefore = await getAmmBalances();

‎yarn-project/bb-prover/src/avm_proving_tests/avm_proven_token.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ describe('AVM Witgen & Circuit apps tests: TokenContract', () => {
2020
let tester: AvmProvingTester;
2121

2222
beforeEach(async () => {
23-
tester = await AvmProvingTester.create(/*checkCircuitOnly*/ true);
23+
tester = await AvmProvingTester.new(/*checkCircuitOnly*/ true);
2424

2525
const constructorArgs = [admin, /*name=*/ 'Token', /*symbol=*/ 'TOK', /*decimals=*/ new Fr(18)];
2626
token = await tester.registerAndDeployContract(constructorArgs, /*deployer=*/ admin, TokenContractArtifact);

‎yarn-project/bb-prover/src/avm_proving_tests/avm_proving_and_verification.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ describe('AVM WitGen & Circuit – proving and verification', () => {
1212
let tester: AvmProvingTester;
1313

1414
beforeEach(async () => {
15-
tester = await AvmProvingTester.create(/*checkCircuitOnly*/ false);
15+
tester = await AvmProvingTester.new(/*checkCircuitOnly*/ false);
1616
avmTestContractInstance = await tester.registerAndDeployContract(
1717
/*constructorArgs=*/ [],
1818
/*deployer=*/ AztecAddress.fromNumber(420),

‎yarn-project/bb-prover/src/avm_proving_tests/avm_proving_tester.ts‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import type { AvmCircuitInputs } from '@aztec/stdlib/avm';
44
import { AztecAddress } from '@aztec/stdlib/aztec-address';
55
import type { MerkleTreeWriteOperations } from '@aztec/stdlib/interfaces/server';
66
import { makeAvmCircuitInputs } from '@aztec/stdlib/testing';
7+
import type { GlobalVariables } from '@aztec/stdlib/tx';
78
import { VerificationKeyData } from '@aztec/stdlib/vks';
89
import { NativeWorldStateService } from '@aztec/world-state';
910

@@ -30,16 +31,18 @@ export class AvmProvingTester extends PublicTxSimulationTester {
3031
private checkCircuitOnly: boolean,
3132
merkleTree: MerkleTreeWriteOperations,
3233
contractDataSource: SimpleContractDataSource,
34+
globals?: GlobalVariables,
3335
) {
34-
super(merkleTree, contractDataSource);
36+
super(merkleTree, contractDataSource, globals);
3537
}
3638

37-
static override async create(checkCircuitOnly: boolean = false) {
39+
// overriding parent class' create is a pain, so we use a different nam
40+
static async new(checkCircuitOnly: boolean = false, globals?: GlobalVariables) {
3841
const bbWorkingDirectory = await fs.mkdtemp(path.join(tmpdir(), 'bb-'));
3942

4043
const contractDataSource = new SimpleContractDataSource();
4144
const merkleTrees = await (await NativeWorldStateService.tmp()).fork();
42-
return new AvmProvingTester(bbWorkingDirectory, checkCircuitOnly, merkleTrees, contractDataSource);
45+
return new AvmProvingTester(bbWorkingDirectory, checkCircuitOnly, merkleTrees, contractDataSource, globals);
4346
}
4447

4548
async prove(avmCircuitInputs: AvmCircuitInputs): Promise<BBResult> {
@@ -108,16 +111,17 @@ export class AvmProvingTesterV2 extends PublicTxSimulationTester {
108111
private bbWorkingDirectory: string,
109112
contractDataSource: SimpleContractDataSource,
110113
merkleTrees: MerkleTreeWriteOperations,
114+
globals?: GlobalVariables,
111115
) {
112-
super(merkleTrees, contractDataSource);
116+
super(merkleTrees, contractDataSource, globals);
113117
}
114118

115-
static override async create() {
119+
static async new(globals?: GlobalVariables) {
116120
const bbWorkingDirectory = await fs.mkdtemp(path.join(tmpdir(), 'bb-'));
117121

118122
const contractDataSource = new SimpleContractDataSource();
119123
const merkleTrees = await (await NativeWorldStateService.tmp()).fork();
120-
return new AvmProvingTesterV2(bbWorkingDirectory, contractDataSource, merkleTrees);
124+
return new AvmProvingTesterV2(bbWorkingDirectory, contractDataSource, merkleTrees, globals);
121125
}
122126

123127
async proveV2(avmCircuitInputs: AvmCircuitInputs): Promise<BBResult> {

‎yarn-project/bb-prover/src/avm_proving_tests/avm_public_fee_payment.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ describe('AVM WitGen & Circuit – public fee payment', () => {
1616
let tester: AvmProvingTester;
1717

1818
beforeEach(async () => {
19-
tester = await AvmProvingTester.create(/*checkCircuitOnly*/ true);
19+
tester = await AvmProvingTester.new(/*checkCircuitOnly*/ true);
2020

2121
await tester.registerFeeJuiceContract();
2222
await tester.setFeePayerBalance(feePayer, initialFeeJuiceBalance);

0 commit comments

Comments
 (0)