Skip to content

Commit be91d80

Browse files
ludamadalexghr
ludamad
and
alexghr
authored
chore: simplify otel config, 1val setup, fix pod dns, retries (#10344)
There were a few things in the way of getting a devnet up - Added retries as we had 'intrinsic gas too low' before reth got a chance to sync - use cluster dns explicitly before host networkinge - work around an issue with broker external host --------- Co-authored-by: Alex Gherghisan <alexghr@users.noreply.github.com>
1 parent 2c2169b commit be91d80

25 files changed

Lines changed: 188 additions & 102 deletions

‎spartan/aztec-network/files/config/deploy-l1-contracts.sh‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,19 @@ CHAIN_ID=$1
66

77
# Run the deploy-l1-contracts command and capture the output
88
output=""
9-
# if INIT_VALIDATORS is true, then we need to pass the validators flag to the deploy-l1-contracts command
10-
if [ "$INIT_VALIDATORS" = "true" ]; then
11-
output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --mnemonic "$MNEMONIC" --validators $2 --l1-chain-id $CHAIN_ID)
12-
else
13-
output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --mnemonic "$MNEMONIC" --l1-chain-id $CHAIN_ID)
14-
fi
9+
MAX_RETRIES=5
10+
RETRY_DELAY=60
11+
for attempt in $(seq 1 $MAX_RETRIES); do
12+
# if INIT_VALIDATORS is true, then we need to pass the validators flag to the deploy-l1-contracts command
13+
if [ "${INIT_VALIDATORS:-false}" = "true" ]; then
14+
output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --mnemonic "$MNEMONIC" --validators $2 --l1-chain-id $CHAIN_ID) && break
15+
else
16+
output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --mnemonic "$MNEMONIC" --l1-chain-id $CHAIN_ID) && break
17+
fi
18+
echo "Attempt $attempt failed. Retrying in $RETRY_DELAY seconds..."
19+
sleep "$RETRY_DELAY"
20+
done || { echo "All l1 contract deploy attempts failed."; exit 1; }
21+
1522

1623
echo "$output"
1724

‎spartan/aztec-network/files/config/setup-service-addresses.sh‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,17 +81,29 @@ fi
8181

8282
if [ "${PROVER_BROKER_EXTERNAL_HOST}" != "" ]; then
8383
PROVER_BROKER_ADDR="${PROVER_BROKER_EXTERNAL_HOST}"
84-
elif [ "${NETWORK_PUBLIC}" = "true" ]; then
85-
PROVER_BROKER_ADDR=$(get_service_address "prover-broker" "${PROVER_BROKER_PORT}")
8684
else
8785
PROVER_BROKER_ADDR="http://${SERVICE_NAME}-prover-broker.${NAMESPACE}:${PROVER_BROKER_PORT}"
8886
fi
8987

88+
# Configure OTEL_COLLECTOR_ENDPOINT if not set in values file
89+
if [ "${TELEMETRY:-false}" = "true" ] && [ "${OTEL_COLLECTOR_ENDPOINT}" = "" ]; then
90+
OTEL_COLLECTOR_PORT=${OTEL_COLLECTOR_PORT:-4318}
91+
OTEL_COLLECTOR_ENDPOINT="http://metrics-opentelemetry-collector.metrics:$OTEL_COLLECTOR_PORT"
92+
fi
9093

9194
# Write addresses to file for sourcing
9295
echo "export ETHEREUM_HOST=${ETHEREUM_ADDR}" >> /shared/config/service-addresses
9396
echo "export BOOT_NODE_HOST=${BOOT_NODE_ADDR}" >> /shared/config/service-addresses
9497
echo "export PROVER_NODE_HOST=${PROVER_NODE_ADDR}" >> /shared/config/service-addresses
9598
echo "export PROVER_BROKER_HOST=${PROVER_BROKER_ADDR}" >> /shared/config/service-addresses
99+
100+
if [ "${OTEL_COLLECTOR_ENDPOINT}" != "" ]; then
101+
echo "export OTEL_COLLECTOR_ENDPOINT=$OTEL_COLLECTOR_ENDPOINT" >> /shared/config/service-addresses
102+
echo "export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=$OTEL_COLLECTOR_ENDPOINT/v1/logs" >> /shared/config/service-addresses
103+
echo "export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=$OTEL_COLLECTOR_ENDPOINT/v1/metrics" >> /shared/config/service-addresses
104+
echo "export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=$OTEL_COLLECTOR_ENDPOINT/v1/traces" >> /shared/config/service-addresses
105+
fi
106+
107+
96108
echo "Addresses configured:"
97109
cat /shared/config/service-addresses

‎spartan/aztec-network/templates/_helpers.tpl‎

Lines changed: 4 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -68,30 +68,6 @@ http://{{ include "aztec-network.fullname" . }}-validator.{{ .Release.Namespace
6868
http://{{ include "aztec-network.fullname" . }}-metrics.{{ .Release.Namespace }}
6969
{{- end -}}
7070

71-
{{- define "aztec-network.otelCollectorMetricsEndpoint" -}}
72-
{{- if .Values.telemetry.enabled -}}
73-
{{- if .Values.telemetry.otelCollectorEndpoint -}}
74-
{{- .Values.telemetry.otelCollectorEndpoint -}}/v1/metrics
75-
{{- end -}}
76-
{{- end -}}
77-
{{- end -}}
78-
79-
{{- define "aztec-network.otelCollectorTracesEndpoint" -}}
80-
{{- if .Values.telemetry.enabled -}}
81-
{{- if .Values.telemetry.otelCollectorEndpoint -}}
82-
{{- .Values.telemetry.otelCollectorEndpoint -}}/v1/traces
83-
{{- end -}}
84-
{{- end -}}
85-
{{- end -}}
86-
87-
{{- define "aztec-network.otelCollectorLogsEndpoint" -}}
88-
{{- if .Values.telemetry.enabled -}}
89-
{{- if .Values.telemetry.otelCollectorEndpoint -}}
90-
{{- .Values.telemetry.otelCollectorEndpoint -}}/v1/logs
91-
{{- end -}}
92-
{{- end -}}
93-
{{- end -}}
94-
9571
{{- define "helpers.flag" -}}
9672
{{- $name := index . 0 -}}
9773
{{- $value := index . 1 -}}
@@ -153,6 +129,10 @@ Service Address Setup Container
153129
value: "{{ .Values.network.public }}"
154130
- name: NAMESPACE
155131
value: {{ .Release.Namespace }}
132+
- name: TELEMETRY
133+
value: "{{ .Values.telemetry.enabled }}"
134+
- name: OTEL_COLLECTOR_ENDPOINT
135+
value: "{{ .Values.telemetry.otelCollectorEndpoint }}"
156136
- name: EXTERNAL_ETHEREUM_HOST
157137
value: "{{ .Values.ethereum.externalHost }}"
158138
- name: ETHEREUM_PORT

‎spartan/aztec-network/templates/boot-node.yaml‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ spec:
1717
{{- include "aztec-network.selectorLabels" . | nindent 8 }}
1818
app: boot-node
1919
spec:
20+
dnsPolicy: ClusterFirstWithHostNet
2021
{{- if .Values.network.public }}
2122
hostNetwork: true
2223
{{- include "aztec-network.publicAntiAffinity" . | nindent 6 }}
@@ -32,17 +33,18 @@ spec:
3233
- -c
3334
- |
3435
source /shared/config/service-addresses
36+
cat /shared/config/service-addresses
3537
echo "Awaiting ethereum node at ${ETHEREUM_HOST}"
3638
until curl -s -X POST -H 'Content-Type: application/json' \
3739
-d '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \
3840
${ETHEREUM_HOST} | grep -q reth; do
39-
echo "Waiting for Ethereum node..."
41+
echo "Waiting for Ethereum node ${ETHEREUM_HOST}..."
4042
sleep 5
4143
done
4244
echo "Ethereum node is ready!"
4345
{{- if .Values.telemetry.enabled }}
44-
until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do
45-
echo "Waiting for OpenTelemetry collector..."
46+
until curl --head --silent $OTEL_COLLECTOR_ENDPOINT > /dev/null; do
47+
echo "Waiting for OpenTelemetry collector $OTEL_COLLECTOR_ENDPOINT..."
4648
sleep 5
4749
done
4850
echo "OpenTelemetry collector is ready!"
@@ -70,6 +72,8 @@ spec:
7072
- name: scripts
7173
mountPath: /scripts
7274
env:
75+
- name: TELEMETRY
76+
value: "{{ .Values.telemetry.enabled }}"
7377
- name: INIT_VALIDATORS
7478
value: "true"
7579
- name: MNEMONIC
@@ -152,12 +156,6 @@ spec:
152156
value: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
153157
- name: OTEL_RESOURCE_ATTRIBUTES
154158
value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }}
155-
- name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
156-
value: {{ include "aztec-network.otelCollectorMetricsEndpoint" . | quote }}
157-
- name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
158-
value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
159-
- name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
160-
value: {{ include "aztec-network.otelCollectorLogsEndpoint" . | quote }}
161159
- name: PROVER_REAL_PROOFS
162160
value: "{{ .Values.aztec.realProofs }}"
163161
- name: PXE_PROVER_ENABLED

‎spartan/aztec-network/templates/deploy-l1-verifier.yaml‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,20 +44,29 @@ spec:
4444
chmod +x /tmp/setup-service-addresses.sh
4545
/tmp/setup-service-addresses.sh
4646
source /shared/config/service-addresses
47+
cat /shared/config/service-addresses
4748
4849
until curl -s -X GET "$BOOT_NODE_HOST/status"; do
4950
echo "Waiting for Aztec node $BOOT_NODE_HOST..."
5051
sleep 5
5152
done
5253
echo "Boot node is ready!"
5354
54-
export ROLLUP_CONTRACT_ADDRESS=$(curl -X POST -H 'Content-Type: application/json' \
55+
l1_contracts=$(curl -X POST -H 'Content-Type: application/json' \
5556
-d '{"jsonrpc":"2.0","method":"node_getL1ContractAddresses","params":[],"id":1}' \
56-
"$BOOT_NODE_HOST" \
57-
| jq -r '.result.rollupAddress.value')
58-
57+
"$BOOT_NODE_HOST")
58+
echo "L1 Contracts"
59+
echo $l1_contracts
60+
export ROLLUP_CONTRACT_ADDRESS=$(echo $l1_contracts | jq -r '.result.rollupAddress')
61+
[ -z "$ROLLUP_CONTRACT_ADDRESS" ] && echo "Could not retrieve rollup address!" && exit 1
5962
echo "Rollup contract address: $ROLLUP_CONTRACT_ADDRESS"
60-
node /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-verifier --verifier real
63+
MAX_RETRIES=5
64+
RETRY_DELAY=60
65+
for attempt in $(seq 1 $MAX_RETRIES); do
66+
node /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-verifier --verifier real
67+
echo "Attempt $attempt failed. Retrying in $RETRY_DELAY seconds..."
68+
sleep "$RETRY_DELAY"
69+
done || { echo "All L1 verifier deploy attempts failed." >&2; exit 1; }
6170
echo "L1 verifier deployed"
6271
env:
6372
- name: NODE_NO_WARNINGS

‎spartan/aztec-network/templates/prover-agent.yaml‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ spec:
3434
serviceAccountName: {{ include "aztec-network.fullname" . }}-node
3535
{{- if .Values.network.public }}
3636
hostNetwork: true
37+
dnsPolicy: ClusterFirstWithHostNet
3738
{{- end }}
3839
volumes:
3940
- name: config
@@ -50,14 +51,15 @@ spec:
5051
- -c
5152
- |
5253
source /shared/config/service-addresses
54+
cat /shared/config/service-addresses
5355
until curl -s -X POST ${PROVER_BROKER_HOST}/status; do
5456
echo "Waiting for broker ${PROVER_BROKER_HOST} ..."
5557
sleep 5
5658
done
5759
echo "Broker is ready!"
5860
{{- if .Values.telemetry.enabled }}
59-
until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do
60-
echo "Waiting for OpenTelemetry collector..."
61+
until curl --head --silent $OTEL_COLLECTOR_ENDPOINT > /dev/null; do
62+
echo "Waiting for OpenTelemetry collector $OTEL_COLLECTOR_ENDPOINT..."
6163
sleep 5
6264
done
6365
echo "OpenTelemetry collector is ready!"
@@ -95,12 +97,6 @@ spec:
9597
value: {{ join "," .Values.proverAgent.proofTypes | quote }}
9698
- name: OTEL_RESOURCE_ATTRIBUTES
9799
value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }}
98-
- name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
99-
value: {{ include "aztec-network.otelCollectorMetricsEndpoint" . | quote }}
100-
- name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
101-
value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
102-
- name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
103-
value: {{ include "aztec-network.otelCollectorLogsEndpoint" . | quote }}
104100
resources:
105101
{{- toYaml .Values.proverAgent.resources | nindent 12 }}
106102
{{- end }}

‎spartan/aztec-network/templates/prover-broker.yaml‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ spec:
2020
serviceAccountName: {{ include "aztec-network.fullname" . }}-node
2121
{{- if .Values.network.public }}
2222
hostNetwork: true
23+
dnsPolicy: ClusterFirstWithHostNet
2324
{{- end }}
2425
volumes:
2526
- name: config
@@ -36,9 +37,10 @@ spec:
3637
- -c
3738
- |
3839
source /shared/config/service-addresses
40+
cat /shared/config/service-addresses
3941
{{- if .Values.telemetry.enabled }}
40-
until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do
41-
echo "Waiting for OpenTelemetry collector..."
42+
until curl --head --silent $OTEL_COLLECTOR_ENDPOINT > /dev/null; do
43+
echo "Waiting for OpenTelemetry collector $OTEL_COLLECTOR_ENDPOINT..."
4244
sleep 5
4345
done
4446
echo "OpenTelemetry collector is ready!"
@@ -76,15 +78,11 @@ spec:
7678
value: "{{ .Values.proverBroker.dataDirectory }}"
7779
- name: OTEL_RESOURCE_ATTRIBUTES
7880
value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }}
79-
- name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
80-
value: {{ include "aztec-network.otelCollectorMetricsEndpoint" . | quote }}
81-
- name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
82-
value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
83-
- name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
84-
value: {{ include "aztec-network.otelCollectorLogsEndpoint" . | quote }}
8581
resources:
8682
{{- toYaml .Values.proverBroker.resources | nindent 12 }}
83+
{{- end }}
8784
---
85+
# Headless service for StatefulSet DNS entries
8886
apiVersion: v1
8987
kind: Service
9088
metadata:
@@ -99,4 +97,3 @@ spec:
9997
ports:
10098
- port: {{ .Values.proverBroker.service.nodePort }}
10199
name: node
102-
{{ end }}

‎spartan/aztec-network/templates/prover-node.yaml‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ spec:
1919
spec:
2020
{{- if .Values.network.public }}
2121
hostNetwork: true
22+
dnsPolicy: ClusterFirstWithHostNet
2223
{{- include "aztec-network.publicAntiAffinity" . | nindent 6 }}
2324
{{- end }}
2425
serviceAccountName: {{ include "aztec-network.fullname" . }}-node
@@ -35,7 +36,7 @@ spec:
3536
until curl -s -X POST -H 'Content-Type: application/json' \
3637
-d '{"jsonrpc":"2.0","method":"web3_clientVersion","params":[],"id":67}' \
3738
${ETHEREUM_HOST} | grep -q reth; do
38-
echo "Waiting for Ethereum node..."
39+
echo "Waiting for Ethereum node ${ETHEREUM_HOST}..."
3940
sleep 5
4041
done
4142
echo "Ethereum node is ready!"
@@ -51,8 +52,8 @@ spec:
5152
fi
5253
5354
{{- if .Values.telemetry.enabled }}
54-
until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do
55-
echo "Waiting for OpenTelemetry collector..."
55+
until curl --head --silent $OTEL_COLLECTOR_ENDPOINT > /dev/null; do
56+
echo "Waiting for OpenTelemetry collector $OTEL_COLLECTOR_ENDPOINT..."
5657
sleep 5
5758
done
5859
echo "OpenTelemetry collector is ready!"
@@ -142,12 +143,6 @@ spec:
142143
value: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
143144
- name: OTEL_RESOURCE_ATTRIBUTES
144145
value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }}
145-
- name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
146-
value: {{ include "aztec-network.otelCollectorMetricsEndpoint" . | quote }}
147-
- name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
148-
value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
149-
- name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
150-
value: {{ include "aztec-network.otelCollectorLogsEndpoint" . | quote }}
151146
- name: L1_CHAIN_ID
152147
value: "{{ .Values.ethereum.chainId }}"
153148
- name: P2P_ENABLED

‎spartan/aztec-network/templates/pxe.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ spec:
1919
serviceAccountName: {{ include "aztec-network.fullname" . }}-node
2020
{{- if .Values.network.public }}
2121
hostNetwork: true
22+
dnsPolicy: ClusterFirstWithHostNet
2223
{{- end }}
2324
volumes:
2425
- name: config
@@ -37,6 +38,7 @@ spec:
3738
- -c
3839
- |
3940
source /shared/config/service-addresses
41+
cat /shared/config/service-addresses
4042
until curl --head --silent ${BOOT_NODE_HOST}/status; do
4143
echo "Waiting for boot node..."
4244
sleep 5
@@ -68,6 +70,7 @@ spec:
6870
- "-c"
6971
- |
7072
source /shared/config/service-addresses
73+
cat /shared/config/service-addresses
7174
{{- if .Values.network.public }}
7275
# If the network is public, we need to use the boot node URL
7376
export AZTEC_NODE_URL=${BOOT_NODE_HOST}

‎spartan/aztec-network/templates/rbac.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,4 +55,4 @@ roleRef:
5555
subjects:
5656
- kind: ServiceAccount
5757
name: {{ include "aztec-network.fullname" . }}-node
58-
namespace: {{ .Release.Namespace }}
58+
namespace: {{ .Release.Namespace }}

0 commit comments

Comments
 (0)