feat(l1): add rpc error rates to metrics and panels

cmd/ethrex/initializers.rs

@@ -11,6 +11,7 @@ use ethrex_common::types::Genesis;
 use ethrex_config::networks::Network;
 
 use ethrex_metrics::profiling::{FunctionProfilingLayer, initialize_block_processing_profile};
+use ethrex_metrics::rpc::initialize_rpc_metrics;
 use ethrex_p2p::rlpx::initiator::RLPxInitiator;
 use ethrex_p2p::{
     discv4::peer_table::PeerTable,
@@ -89,6 +90,7 @@ pub fn init_metrics(opts: &Options, tracker: TaskTracker) {
     );
 
     initialize_block_processing_profile();
+    initialize_rpc_metrics();
 
     tracker.spawn(metrics_api);
 }

crates/blockchain/metrics/api.rs

@@ -1,10 +1,8 @@
 use axum::{Router, routing::get};
 
-use crate::profiling::gather_profiling_metrics;
-
 use crate::{
-    MetricsApiError, metrics_blocks::METRICS_BLOCKS, metrics_process::METRICS_PROCESS,
-    metrics_transactions::METRICS_TX,
+    MetricsApiError, gather_default_metrics, metrics_blocks::METRICS_BLOCKS,
+    metrics_process::METRICS_PROCESS, metrics_transactions::METRICS_TX,
 };
 
 pub async fn start_prometheus_metrics_api(
@@ -33,10 +31,10 @@ pub(crate) async fn get_metrics() -> String {
     };
 
     ret_string.push('\n');
-    match gather_profiling_metrics() {
+    match gather_default_metrics() {
         Ok(string) => ret_string.push_str(&string),
         Err(_) => {
-            tracing::error!("Failed to register METRICS_PROFILING");
+            tracing::error!("Failed to gather default Prometheus metrics");
             return String::new();
         }
     };

crates/blockchain/metrics/mod.rs

@@ -10,6 +10,8 @@ pub mod metrics_process;
 pub mod metrics_transactions;
 #[cfg(feature = "api")]
 pub mod profiling;
+#[cfg(feature = "api")]
+pub mod rpc;
 
 /// A macro to conditionally enable metrics-related code.
 ///
@@ -70,3 +72,24 @@ pub enum MetricsError {
     #[error("MetricsL2Error {0}")]
     FromUtf8Error(#[from] std::string::FromUtf8Error),
 }
+
+#[cfg(feature = "api")]
+/// Returns all metrics currently registered in Prometheus' default registry.
+///
+/// Both profiling and RPC metrics register with this default registry, and the
+/// metrics API surfaces them by calling this helper.
+pub fn gather_default_metrics() -> Result<String, MetricsError> {
+    use prometheus::{Encoder, TextEncoder};
+
+    let encoder = TextEncoder::new();
+    let metric_families = prometheus::gather();
+
+    let mut buffer = Vec::new();
+    encoder
+        .encode(&metric_families, &mut buffer)
+        .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?;
+
+    let res = String::from_utf8(buffer)?;
+
+    Ok(res)
+}

crates/blockchain/metrics/profiling.rs

@@ -1,17 +1,18 @@
-use prometheus::{Encoder, HistogramTimer, HistogramVec, TextEncoder, register_histogram_vec};
-use std::{future::Future, sync::LazyLock};
+use prometheus::{HistogramTimer, HistogramVec, register_histogram_vec};
+use std::sync::LazyLock;
 use tracing::{
     Subscriber,
     field::{Field, Visit},
     span::{Attributes, Id},
 };
 use tracing_subscriber::{Layer, layer::Context, registry::LookupSpan};
 
-use crate::MetricsError;
-
 pub static METRICS_BLOCK_PROCESSING_PROFILE: LazyLock<HistogramVec> =
     LazyLock::new(initialize_histogram_vec);
 
+// Metrics defined in this module register into the Prometheus default registry.
+// The metrics API exposes them by calling `gather_default_metrics()`.
+
 fn initialize_histogram_vec() -> HistogramVec {
     register_histogram_vec!(
         "function_duration_seconds",
@@ -111,45 +112,6 @@ where
     }
 }
 
-/// Records the duration of an async operation in the function profiling histogram.
-///
-/// This provides a lightweight alternative to the `#[instrument]` attribute when you need
-/// manual control over timing instrumentation, such as in RPC handlers.
-///
-/// # Parameters
-/// * `namespace` - Category for the metric (e.g., "rpc", "engine", "block_execution")
-/// * `function_name` - Name identifier for the operation being timed
-/// * `future` - The async operation to time
-///
-/// Use this function when you need to instrument an async operation for duration metrics,
-/// but cannot or do not want to use the `#[instrument]` attribute (for example, in RPC handlers).
-pub async fn record_async_duration<Fut, T>(namespace: &str, function_name: &str, future: Fut) -> T
-where
-    Fut: Future<Output = T>,
-{
-    let timer = METRICS_BLOCK_PROCESSING_PROFILE
-        .with_label_values(&[namespace, function_name])
-        .start_timer();
-
-    let output = future.await;
-    timer.observe_duration();
-    output
-}
-
-pub fn gather_profiling_metrics() -> Result<String, MetricsError> {
-    let encoder = TextEncoder::new();
-    let metric_families = prometheus::gather();
-
-    let mut buffer = Vec::new();
-    encoder
-        .encode(&metric_families, &mut buffer)
-        .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?;
-
-    let res = String::from_utf8(buffer)?;
-
-    Ok(res)
-}
-
 pub fn initialize_block_processing_profile() {
     METRICS_BLOCK_PROCESSING_PROFILE.reset();
 }

crates/blockchain/metrics/rpc.rs

@@ -0,0 +1,81 @@
+use prometheus::{CounterVec, HistogramVec, register_counter_vec, register_histogram_vec};
+use std::{future::Future, sync::LazyLock};
+
+pub static METRICS_RPC_REQUEST_OUTCOMES: LazyLock<CounterVec> =
+    LazyLock::new(initialize_rpc_outcomes_counter);
+
+pub static METRICS_RPC_DURATION: LazyLock<HistogramVec> =
+    LazyLock::new(initialize_rpc_duration_histogram);
+
+// Metrics defined in this module register into the Prometheus default registry.
+// The metrics API exposes them by calling `gather_default_metrics()`.
+
+fn initialize_rpc_outcomes_counter() -> CounterVec {
+    register_counter_vec!(
+        "rpc_requests_total",
+        "Total number of RPC requests partitioned by namespace, method, and outcome",
+        &["namespace", "function_name", "outcome"],
+    )
+    .unwrap()
+}
+
+fn initialize_rpc_duration_histogram() -> HistogramVec {
+    register_histogram_vec!(
+        "rpc_request_duration_seconds",
+        "Histogram of RPC request handling duration partitioned by namespace and method",
+        &["namespace", "function_name"],
+    )
+    .unwrap()
+}
+
+/// Represents the outcome of an RPC request when recording metrics.
+#[derive(Clone, Copy)]
+pub enum RpcOutcome {
+    Success,
+    Error,
+}
+
+impl RpcOutcome {
+    fn as_label(&self) -> &'static str {
+        match self {
+            RpcOutcome::Success => "success",
+            RpcOutcome::Error => "error",
+        }
+    }
+}
+
+pub fn record_rpc_outcome(namespace: &str, function_name: &str, outcome: RpcOutcome) {
+    METRICS_RPC_REQUEST_OUTCOMES
+        .with_label_values(&[namespace, function_name, outcome.as_label()])
+        .inc();
+}
+
+pub fn initialize_rpc_metrics() {
+    METRICS_RPC_REQUEST_OUTCOMES.reset();
+    METRICS_RPC_DURATION.reset();
+}
+
+/// Records the duration of an async operation in the function profiling histogram.
+///
+/// This provides a lightweight alternative to the `#[instrument]` attribute when you need
+/// manual control over timing instrumentation, such as in RPC handlers.
+///
+/// # Parameters
+/// * `namespace` - Category for the metric (e.g., "rpc", "engine", "block_execution")
+/// * `function_name` - Name identifier for the operation being timed
+/// * `future` - The async operation to time
+///
+/// Use this function when you need to instrument an async operation for duration metrics,
+/// but cannot or do not want to use the `#[instrument]` attribute (for example, in RPC handlers).
+pub async fn record_async_duration<Fut, T>(namespace: &str, function_name: &str, future: Fut) -> T
+where
+    Fut: Future<Output = T>,
+{
+    let timer = METRICS_RPC_DURATION
+        .with_label_values(&[namespace, function_name])
+        .start_timer();
+
+    let output = future.await;
+    timer.observe_duration();
+    output
+}

crates/networking/rpc/rpc.rs

@@ -55,7 +55,7 @@ use bytes::Bytes;
 use ethrex_blockchain::Blockchain;
 use ethrex_blockchain::error::ChainError;
 use ethrex_common::types::Block;
-use ethrex_metrics::profiling::record_async_duration;
+use ethrex_metrics::rpc::{RpcOutcome, record_async_duration, record_rpc_outcome};
 use ethrex_p2p::peer_handler::PeerHandler;
 use ethrex_p2p::sync_manager::SyncManager;
 use ethrex_p2p::types::Node;
@@ -196,11 +196,24 @@ pub trait RpcHandler: Sized {
             Ok(RpcNamespace::Engine) => "engine",
             _ => "rpc",
         };
+        let method = req.method.as_str();
+
+        let result =
+            record_async_duration(
+                namespace,
+                method,
+                async move { request.handle(context).await },
+            )
+            .await;
+
+        let outcome = if result.is_ok() {
+            RpcOutcome::Success
+        } else {
+            RpcOutcome::Error
+        };
+        record_rpc_outcome(namespace, method, outcome);
 
-        record_async_duration(namespace, req.method.as_str(), async move {
-            request.handle(context).await
-        })
-        .await
+        result
     }
 
     async fn handle(&self, context: RpcApiContext) -> Result<Value, RpcErr>;