From 805da6fb35a104cdc58e677772ba7740e04f0632 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Tue, 4 Jun 2024 12:31:17 +0000 Subject: [PATCH 1/7] Autoscaling --- yarn-project/aztec/terraform/node/main.tf | 168 ++++++++++++++-------- 1 file changed, 109 insertions(+), 59 deletions(-) diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf index 5e1c6cc006b6..34afffbc200e 100644 --- a/yarn-project/aztec/terraform/node/main.tf +++ b/yarn-project/aztec/terraform/node/main.tf @@ -151,7 +151,7 @@ resource "aws_ecs_task_definition" "aztec-node" { [ { "name": "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}", - "image": "${var.DOCKERHUB_ACCOUNT}/aztec:${var.DEPLOY_TAG}", + "image": "${var.FULL_IMAGE}", "command": ["start", "--node", "--archiver", "--sequencer", "--prover"], "essential": true, "memoryReservation": 3776, @@ -187,7 +187,7 @@ resource "aws_ecs_task_definition" "aztec-node" { }, { "name": "DEBUG", - "value": "aztec:*,-json-rpc:json_proxy:*,-aztec:avm_simulator:*,discv5:*,libp2p:*" + "value": "aztec:*,-json-rpc:json_proxy:*,-aztec:avm_simulator:*" }, { "name": "ETHEREUM_HOST", @@ -306,22 +306,6 @@ resource "aws_ecs_task_definition" "aztec-node" { "name": "P2P_PEER_CHECK_INTERVAL_MS", "value": "2000" }, - { - "name": "ACVM_WORKING_DIRECTORY", - "value": "/usr/src/acvm" - }, - { - "name": "BB_WORKING_DIRECTORY", - "value": "/usr/src/bb" - }, - { - "name": "ACVM_BINARY_PATH", - "value": "/usr/src/noir/noir-repo/target/release/acvm" - }, - { - "name": "BB_BINARY_PATH", - "value": "/usr/src/barretenberg/cpp/build/bin/bb" - }, { "name": "PROVER_AGENTS", "value": "0" @@ -550,35 +534,30 @@ resource "aws_lb_listener" "aztec-node-udp-listener" { // Configuration for proving agents resource "aws_cloudwatch_log_group" "aztec-proving-agent-log-group" { - count = local.total_agents - name = "/fargate/service/${var.DEPLOY_TAG}/aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}" + count = local.node_count + name = "/fargate/service/${var.DEPLOY_TAG}/aztec-proving-agent-group-${count.index + 1}" retention_in_days = 14 } resource "aws_service_discovery_service" "aztec-proving-agent" { - count = local.total_agents - name = "${var.DEPLOY_TAG}-aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}" + count = local.node_count + name = "${var.DEPLOY_TAG}-aztec-proving-agent-group-${count.index + 1}" health_check_custom_config { failure_threshold = 1 } - dns_config { namespace_id = data.terraform_remote_state.setup_iac.outputs.local_service_discovery_id - dns_records { ttl = 60 type = "A" } - dns_records { ttl = 60 type = "SRV" } - routing_policy = "MULTIVALUE" } - # Terraform just fails if this resource changes and you have registered instances. provisioner "local-exec" { when = destroy @@ -588,23 +567,22 @@ resource "aws_service_discovery_service" "aztec-proving-agent" { # Define task definitions for each node. resource "aws_ecs_task_definition" "aztec-proving-agent" { - count = local.total_agents - family = "${var.DEPLOY_TAG}-aztec-proving-agent-${floor(count.index / local.agents_per_sequencer) + 1}-${(count.index % local.agents_per_sequencer) + 1}" + count = local.node_count + family = "${var.DEPLOY_TAG}-aztec-proving-agent-group-${count.index + 1}" requires_compatibilities = ["FARGATE"] network_mode = "awsvpc" cpu = "16384" - memory = "65536" + memory = "98304" execution_role_arn = data.terraform_remote_state.setup_iac.outputs.ecs_task_execution_role_arn task_role_arn = data.terraform_remote_state.aztec2_iac.outputs.cloudwatch_logging_ecs_role_arn - - container_definitions = < Date: Tue, 4 Jun 2024 12:31:30 +0000 Subject: [PATCH 2/7] New variables --- yarn-project/aztec/terraform/node/variables.tf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/yarn-project/aztec/terraform/node/variables.tf b/yarn-project/aztec/terraform/node/variables.tf index 9cf51486bce5..4463487e2c3b 100644 --- a/yarn-project/aztec/terraform/node/variables.tf +++ b/yarn-project/aztec/terraform/node/variables.tf @@ -79,3 +79,11 @@ variable "PROVING_ENABLED" { type = bool default = true } + +variable "IMAGE_TAG" { + type = string +} + +variable "FULL_IMAGE" { + type = string +} From ba98c29aaac4c4425d2ad3ba9f0ce46b8447326d Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 5 Jun 2024 09:25:06 +0000 Subject: [PATCH 3/7] TF Updates --- yarn-project/aztec/terraform/node/main.tf | 54 +++++++++++++++-------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf index 965623025e63..9cdb618b93c0 100644 --- a/yarn-project/aztec/terraform/node/main.tf +++ b/yarn-project/aztec/terraform/node/main.tf @@ -56,10 +56,10 @@ data "terraform_remote_state" "l1_contracts" { locals { publisher_private_keys = [var.SEQ_1_PUBLISHER_PRIVATE_KEY, var.SEQ_2_PUBLISHER_PRIVATE_KEY] node_p2p_private_keys = [var.NODE_1_PRIVATE_KEY, var.NODE_2_PRIVATE_KEY] - node_count = length(local.publisher_private_keys) - data_dir = "/usr/src/yarn-project/aztec/data" - agents_per_sequencer = var.AGENTS_PER_SEQUENCER - total_agents = local.node_count * local.agents_per_sequencer + #node_count = length(local.publisher_private_keys) + node_count = 1 + data_dir = "/usr/src/yarn-project/aztec/data" + agents_per_sequencer = var.AGENTS_PER_SEQUENCER } resource "aws_cloudwatch_log_group" "aztec-node-log-group" { @@ -115,18 +115,32 @@ resource "aws_efs_file_system" "node_data_store" { } } -resource "aws_efs_mount_target" "private_az1" { +# resource "aws_efs_mount_target" "private_az1" { +# count = local.node_count +# file_system_id = aws_efs_file_system.node_data_store[count.index].id +# subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id +# security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] +# } + +# resource "aws_efs_mount_target" "private_az2" { +# count = local.node_count +# file_system_id = aws_efs_file_system.node_data_store[count.index].id +# subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id +# security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] +# } + +resource "aws_efs_mount_target" "public_az1" { count = local.node_count file_system_id = aws_efs_file_system.node_data_store[count.index].id - subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id - security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] + subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_id + security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id] } -resource "aws_efs_mount_target" "private_az2" { +resource "aws_efs_mount_target" "public_az2" { count = local.node_count file_system_id = aws_efs_file_system.node_data_store[count.index].id - subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id - security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] + subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_id + security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id] } # Define task definitions for each node. @@ -187,7 +201,7 @@ resource "aws_ecs_task_definition" "aztec-node" { }, { "name": "DEBUG", - "value": "aztec:*,-json-rpc:json_proxy:*,-aztec:avm_simulator:*" + "value": "aztec:*,-json-rpc:json_proxy:*,-aztec:avm_simulator:*,libp2p:*,discv5:*" }, { "name": "ETHEREUM_HOST", @@ -364,11 +378,11 @@ resource "aws_ecs_service" "aztec-node" { } - load_balancer { - target_group_arn = aws_lb_target_group.aztec-node-tcp[count.index].arn - container_name = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" - container_port = var.NODE_P2P_TCP_PORT + count.index - } + # load_balancer { + # target_group_arn = aws_lb_target_group.aztec-node-tcp[count.index].arn + # container_name = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" + # container_port = var.NODE_P2P_TCP_PORT + count.index + # } # load_balancer { # target_group_arn = aws_lb_target_group.aztec-node-udp[count.index].arn @@ -664,10 +678,11 @@ resource "aws_cloudwatch_metric_alarm" "cpu_high" { count = local.node_count alarm_name = "${var.DEPLOY_TAG}-proving-agent-cpu-high-${count.index + 1}" comparison_operator = "GreaterThanThreshold" - evaluation_periods = "2" + evaluation_periods = "1" metric_name = "CPUUtilization" namespace = "AWS/ECS" - period = "10" + period = "60" + datapoints_to_alarm = 1 statistic = "Maximum" threshold = "20" alarm_description = "Alert when CPU utilization is greater than 20%" @@ -686,6 +701,7 @@ resource "aws_cloudwatch_metric_alarm" "cpu_low" { metric_name = "CPUUtilization" namespace = "AWS/ECS" period = "60" + datapoints_to_alarm = 3 statistic = "Maximum" threshold = "20" alarm_description = "Alarm when CPU utilization is less than 20%" @@ -709,7 +725,7 @@ resource "aws_appautoscaling_target" "ecs_proving_agent" { # Create Scaling Policy for Scaling Out resource "aws_appautoscaling_policy" "scale_out" { count = local.node_count - name = "${var.DEPLOY_TAG}-scale-out-${count.index}" + name = "${var.DEPLOY_TAG}-scale-out-${count.index + 1}" policy_type = "StepScaling" resource_id = aws_appautoscaling_target.ecs_proving_agent[count.index].resource_id scalable_dimension = aws_appautoscaling_target.ecs_proving_agent[count.index].scalable_dimension From 52a6bc1cb3b5c3fb74f63edf01dea1050efd4c07 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 5 Jun 2024 09:27:00 +0000 Subject: [PATCH 4/7] Updated hash --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d795da1139f6..a55a1a8493ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ name: aztec-devnet services: pxe: - image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:afe0263560a82f2c84b9d76451ec4585a3ed389251fb90e916faaa85d9eb9a0d} + image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:edc36351a531c2d76c2a96e2a67dc1f4020d5dcb9c7132d7df8e75a8518ed1b3} # need to run bb for proofs and bb is only built for x86 platform: linux/amd64 environment: @@ -28,7 +28,7 @@ services: expose: - 8080:8080 cli: - image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:afe0263560a82f2c84b9d76451ec4585a3ed389251fb90e916faaa85d9eb9a0d} + image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:edc36351a531c2d76c2a96e2a67dc1f4020d5dcb9c7132d7df8e75a8518ed1b3} # run the same image as pxe platform: linux/amd64 environment: From ea3aaa6a1079ae30cab947a695387d94e4adaea9 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 5 Jun 2024 09:28:21 +0000 Subject: [PATCH 5/7] Updated token name --- yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts | 4 ++-- .../end-to-end/src/e2e_token_contract/token_contract_test.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts b/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts index a6a1e2132624..4f65251ed013 100644 --- a/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts +++ b/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts @@ -53,8 +53,8 @@ type ProvenSetup = { */ export class FullProverTest { - static TOKEN_NAME = 'Aztec Token'; - static TOKEN_SYMBOL = 'AZT'; + static TOKEN_NAME = 'USDC'; + static TOKEN_SYMBOL = 'USD'; static TOKEN_DECIMALS = 18n; private snapshotManager: ISnapshotManager; logger: DebugLogger; diff --git a/yarn-project/end-to-end/src/e2e_token_contract/token_contract_test.ts b/yarn-project/end-to-end/src/e2e_token_contract/token_contract_test.ts index 6510e292f5c1..fe57a47bc147 100644 --- a/yarn-project/end-to-end/src/e2e_token_contract/token_contract_test.ts +++ b/yarn-project/end-to-end/src/e2e_token_contract/token_contract_test.ts @@ -24,8 +24,8 @@ import { TokenSimulator } from '../simulators/token_simulator.js'; const { E2E_DATA_PATH: dataPath } = process.env; export class TokenContractTest { - static TOKEN_NAME = 'Aztec Token'; - static TOKEN_SYMBOL = 'AZT'; + static TOKEN_NAME = 'USDC'; + static TOKEN_SYMBOL = 'USD'; static TOKEN_DECIMALS = 18n; private snapshotManager: ISnapshotManager; logger: DebugLogger; From 6dc8cdbd4a01a038d605219006681c4289519350 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Thu, 6 Jun 2024 09:27:28 +0000 Subject: [PATCH 6/7] TF Updates --- yarn-project/aztec/terraform/node/main.tf | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf index 9cdb618b93c0..13ad6615f7fc 100644 --- a/yarn-project/aztec/terraform/node/main.tf +++ b/yarn-project/aztec/terraform/node/main.tf @@ -56,8 +56,8 @@ data "terraform_remote_state" "l1_contracts" { locals { publisher_private_keys = [var.SEQ_1_PUBLISHER_PRIVATE_KEY, var.SEQ_2_PUBLISHER_PRIVATE_KEY] node_p2p_private_keys = [var.NODE_1_PRIVATE_KEY, var.NODE_2_PRIVATE_KEY] - #node_count = length(local.publisher_private_keys) - node_count = 1 + node_count = length(local.publisher_private_keys) + #node_count = 1 data_dir = "/usr/src/yarn-project/aztec/data" agents_per_sequencer = var.AGENTS_PER_SEQUENCER } @@ -201,7 +201,7 @@ resource "aws_ecs_task_definition" "aztec-node" { }, { "name": "DEBUG", - "value": "aztec:*,-json-rpc:json_proxy:*,-aztec:avm_simulator:*,libp2p:*,discv5:*" + "value": "aztec:*,-json-rpc:json_proxy:*,-aztec:avm_simulator:*" }, { "name": "ETHEREUM_HOST", @@ -280,10 +280,6 @@ resource "aws_ecs_task_definition" "aztec-node" { "name": "P2P_TCP_LISTEN_IP", "value": "0.0.0.0" }, - { - "name": "P2P_ANNOUNCE_TCP_HOSTNAME", - "value": "/ip4/${data.terraform_remote_state.aztec-network_iac.outputs.p2p_eip}" - }, { "name": "P2P_ANNOUNCE_PORT", "value": "${var.NODE_P2P_TCP_PORT + count.index}" @@ -547,9 +543,6 @@ resource "aws_security_group_rule" "allow-node-udp-out" { - - - // Configuration for proving agents resource "aws_cloudwatch_log_group" "aztec-proving-agent-log-group" { From 9b620501f79cfb6891fde11106ed2fb6e4e7ffd3 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Thu, 6 Jun 2024 09:31:17 +0000 Subject: [PATCH 7/7] Update Terraform --- yarn-project/aztec/terraform/node/main.tf | 56 +++++++++++------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf index 13ad6615f7fc..af8c409acfd7 100644 --- a/yarn-project/aztec/terraform/node/main.tf +++ b/yarn-project/aztec/terraform/node/main.tf @@ -436,22 +436,22 @@ resource "aws_lb_listener_rule" "api" { } } -resource "aws_lb_target_group" "aztec-node-tcp" { - count = local.node_count - name = "${var.DEPLOY_TAG}-node-${count.index + 1}-p2p-tcp-target" - port = var.NODE_P2P_TCP_PORT + count.index - protocol = "TCP" - target_type = "ip" - vpc_id = data.terraform_remote_state.setup_iac.outputs.vpc_id +# resource "aws_lb_target_group" "aztec-node-tcp" { +# count = local.node_count +# name = "${var.DEPLOY_TAG}-node-${count.index + 1}-p2p-tcp-target" +# port = var.NODE_P2P_TCP_PORT + count.index +# protocol = "TCP" +# target_type = "ip" +# vpc_id = data.terraform_remote_state.setup_iac.outputs.vpc_id - health_check { - protocol = "TCP" - interval = 10 - healthy_threshold = 2 - unhealthy_threshold = 2 - port = var.NODE_P2P_TCP_PORT + count.index - } -} +# health_check { +# protocol = "TCP" +# interval = 10 +# healthy_threshold = 2 +# unhealthy_threshold = 2 +# port = var.NODE_P2P_TCP_PORT + count.index +# } +# } resource "aws_security_group_rule" "allow-node-tcp-in" { count = local.node_count @@ -473,21 +473,21 @@ resource "aws_security_group_rule" "allow-node-tcp-out" { security_group_id = data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id } -resource "aws_lb_listener" "aztec-node-tcp-listener" { - count = local.node_count - load_balancer_arn = data.terraform_remote_state.aztec-network_iac.outputs.nlb_arn - port = var.NODE_P2P_TCP_PORT + count.index - protocol = "TCP" +# resource "aws_lb_listener" "aztec-node-tcp-listener" { +# count = local.node_count +# load_balancer_arn = data.terraform_remote_state.aztec-network_iac.outputs.nlb_arn +# port = var.NODE_P2P_TCP_PORT + count.index +# protocol = "TCP" - tags = { - name = "aztec-node-${count.index}-tcp-listener" - } +# tags = { +# name = "aztec-node-${count.index}-tcp-listener" +# } - default_action { - type = "forward" - target_group_arn = aws_lb_target_group.aztec-node-tcp[count.index].arn - } -} +# default_action { +# type = "forward" +# target_group_arn = aws_lb_target_group.aztec-node-tcp[count.index].arn +# } +# } # resource "aws_lb_target_group" "aztec-node-udp" {