diff --git a/Cargo.lock b/Cargo.lock index 0227f1395ac71..8087e7971e941 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2874,9 +2874,8 @@ dependencies = [ [[package]] name = "jsonrpsee" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68c414081c784e9b647fabc7cdb6c9642961db3a0486e64b8c0df77addf7d0ff" +version = "0.5.1" +source = "git+https://github.com/paritytech/jsonrpsee?branch=mh-metrics-middleware#10e586af7baa3fbbfb71a95ae42aae809c35d773" dependencies = [ "jsonrpsee-http-server", "jsonrpsee-proc-macros", @@ -2888,9 +2887,8 @@ dependencies = [ [[package]] name = "jsonrpsee-http-server" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab3411d86daf8326aa199cc15968d56e150ba2424b2c29902ff91f522509ee0" +version = "0.5.1" +source = "git+https://github.com/paritytech/jsonrpsee?branch=mh-metrics-middleware#10e586af7baa3fbbfb71a95ae42aae809c35d773" dependencies = [ "futures-channel", "futures-util", @@ -2908,9 +2906,8 @@ dependencies = [ [[package]] name = "jsonrpsee-proc-macros" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e973e34991fd2c15d90afb74380a3c60765072840ed10a85e4d79936dc9d44" +version = "0.5.1" +source = "git+https://github.com/paritytech/jsonrpsee?branch=mh-metrics-middleware#10e586af7baa3fbbfb71a95ae42aae809c35d773" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2920,9 +2917,8 @@ dependencies = [ [[package]] name = "jsonrpsee-types" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c58249ee61c792968214fcb2be6c690e10ace22d8778f90eba7946f629f8e04" +version = "0.5.1" +source = "git+https://github.com/paritytech/jsonrpsee?branch=mh-metrics-middleware#10e586af7baa3fbbfb71a95ae42aae809c35d773" dependencies = [ "anyhow", "async-trait", @@ -2939,9 +2935,8 @@ dependencies = [ [[package]] name = "jsonrpsee-utils" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1647435d03b0f05fadcf498e2e74802d47e70af5a0dc4baeab62d4f1f8f289b" +version = "0.5.1" +source = "git+https://github.com/paritytech/jsonrpsee?branch=mh-metrics-middleware#10e586af7baa3fbbfb71a95ae42aae809c35d773" dependencies = [ "arrayvec 0.7.1", "beef", @@ -2961,9 +2956,8 @@ dependencies = [ [[package]] name = "jsonrpsee-ws-client" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a649c95ca940835fa49a072f17fa7843734dc5044aea5131937f0340073e0e" +version = "0.5.1" +source = "git+https://github.com/paritytech/jsonrpsee?branch=mh-metrics-middleware#10e586af7baa3fbbfb71a95ae42aae809c35d773" dependencies = [ "async-trait", "fnv", @@ -2985,9 +2979,8 @@ dependencies = [ [[package]] name = "jsonrpsee-ws-server" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94ec627cdbe1b26a1ba184b58e4792e3646a2908540d2c071d83dfbd9f62b6b" +version = "0.5.1" +source = "git+https://github.com/paritytech/jsonrpsee?branch=mh-metrics-middleware#10e586af7baa3fbbfb71a95ae42aae809c35d773" dependencies = [ "futures-channel", "futures-util", @@ -10286,8 +10279,8 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f559b464de2e2bdabcac6a210d12e9b5a5973c251e102c44c585c71d51bd78e" dependencies = [ - "cfg-if 1.0.0", - "rand 0.8.4", + "cfg-if 0.1.10", + "rand 0.7.3", "static_assertions", ] diff --git a/Cargo.toml b/Cargo.toml index f30b223a9b205..3c29c1ef262a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -276,3 +276,12 @@ zeroize = { opt-level = 3 } [profile.release] # Substrate runtime requires unwinding. panic = "unwind" + +[patch.crates-io] +jsonrpsee ={git = "https://github.com/paritytech/jsonrpsee" ,branch = "mh-metrics-middleware" } +jsonrpsee-types ={git = "https://github.com/paritytech/jsonrpsee" ,branch = "mh-metrics-middleware" } +jsonrpsee-utils ={git = "https://github.com/paritytech/jsonrpsee" ,branch = "mh-metrics-middleware" } +jsonrpsee-http-server ={git = "https://github.com/paritytech/jsonrpsee" ,branch = "mh-metrics-middleware" } +jsonrpsee-proc-macros ={git = "https://github.com/paritytech/jsonrpsee" ,branch = "mh-metrics-middleware" } +jsonrpsee-ws-client ={git = "https://github.com/paritytech/jsonrpsee" ,branch = "mh-metrics-middleware" } +jsonrpsee-ws-server ={git = "https://github.com/paritytech/jsonrpsee" ,branch = "mh-metrics-middleware" } diff --git a/client/rpc-servers/src/lib.rs b/client/rpc-servers/src/lib.rs index 3b772f0d6fa58..072146a2cfc46 100644 --- a/client/rpc-servers/src/lib.rs +++ b/client/rpc-servers/src/lib.rs @@ -25,8 +25,11 @@ use jsonrpsee::{ ws_server::{WsServerBuilder, WsServerHandle}, RpcModule, }; +use prometheus_endpoint::Registry; use std::net::SocketAddr; +use crate::middleware::{RpcMetrics, RpcMiddleware}; + const MEGABYTE: usize = 1024 * 1024; /// Maximal payload accepted by RPC servers. @@ -38,42 +41,7 @@ pub const WS_MAX_BUFFER_CAPACITY_DEFAULT: usize = 16 * MEGABYTE; /// Default maximum number of connections for WS RPC servers. const WS_MAX_CONNECTIONS: usize = 100; -/*/// RPC server-specific prometheus metrics. -#[derive(Debug, Clone, Default)] -pub struct ServerMetrics { - /// Number of sessions opened. - session_opened: Option>, - /// Number of sessions closed. - session_closed: Option>, -} - -impl ServerMetrics { - /// Create new WebSocket RPC server metrics. - pub fn new(registry: Option<&Registry>) -> Result { - registry - .map(|r| { - Ok(Self { - session_opened: register( - Counter::new( - "rpc_sessions_opened", - "Number of persistent RPC sessions opened", - )?, - r, - )? - .into(), - session_closed: register( - Counter::new( - "rpc_sessions_closed", - "Number of persistent RPC sessions closed", - )?, - r, - )? - .into(), - }) - }) - .unwrap_or_else(|| Ok(Default::default())) - } -}*/ +pub mod middleware; /// Type alias for http server pub type HttpServer = HttpServerHandle; @@ -84,18 +52,17 @@ pub type WsServer = WsServerHandle; pub fn start_http( addrs: &[SocketAddr], cors: Option<&Vec>, - maybe_max_payload_mb: Option, - module: RpcModule, + max_payload_mb: Option, + prometheus_registry: Option<&Registry>, + rpc_api: RpcModule, rt: tokio::runtime::Handle, ) -> Result { - let max_request_body_size = maybe_max_payload_mb + let max_request_body_size = max_payload_mb .map(|mb| mb.saturating_mul(MEGABYTE)) .unwrap_or(RPC_MAX_PAYLOAD_DEFAULT); let mut acl = AccessControlBuilder::new(); - log::info!("Starting JSON-RPC HTTP server: addr={:?}, allowed origins={:?}", addrs, cors); - if let Some(cors) = cors { // Whitelist listening address. // NOTE: set_allowed_hosts will whitelist both ports but only one will used. @@ -103,16 +70,24 @@ pub fn start_http( acl = acl.set_allowed_origins(cors)?; }; - let builder = HttpServerBuilder::default() + let builder = HttpServerBuilder::new() .max_request_body_size(max_request_body_size as u32) .set_access_control(acl.build()) .custom_tokio_runtime(rt.clone()); - let server = tokio::task::block_in_place(|| rt.block_on(async { builder.build(addrs) }))?; - - let rpc_api = build_rpc_api(module); - let handle = server.start(rpc_api)?; + let rpc_api = build_rpc_api(rpc_api); + let handle = if let Some(prometheus_registry) = prometheus_registry { + let metrics = RpcMetrics::new(&prometheus_registry)?; + let middleware = RpcMiddleware::new(metrics, "http".into()); + let builder = builder.set_middleware(middleware); + let server = tokio::task::block_in_place(|| rt.block_on(async { builder.build(addrs) }))?; + server.start(rpc_api)? + } else { + let server = tokio::task::block_in_place(|| rt.block_on(async { builder.build(addrs) }))?; + server.start(rpc_api)? + }; + log::info!("Starting JSON-RPC HTTP server: addr={:?}, allowed origins={:?}", addrs, cors); Ok(handle) } @@ -121,22 +96,21 @@ pub fn start_ws( addrs: &[SocketAddr], max_connections: Option, cors: Option<&Vec>, - maybe_max_payload_mb: Option, - module: RpcModule, + max_payload_mb: Option, + prometheus_registry: Option<&Registry>, + rpc_api: RpcModule, rt: tokio::runtime::Handle, ) -> Result { - let max_request_body_size = maybe_max_payload_mb + let max_request_body_size = max_payload_mb .map(|mb| mb.saturating_mul(MEGABYTE)) .unwrap_or(RPC_MAX_PAYLOAD_DEFAULT); let max_connections = max_connections.unwrap_or(WS_MAX_CONNECTIONS); - let mut builder = WsServerBuilder::default() + let mut builder = WsServerBuilder::new() .max_request_body_size(max_request_body_size as u32) .max_connections(max_connections as u64) .custom_tokio_runtime(rt.clone()); - log::info!("Starting JSON-RPC WS server: addrs={:?}, allowed origins={:?}", addrs, cors); - if let Some(cors) = cors { // Whitelist listening address. // NOTE: set_allowed_hosts will whitelist both ports but only one will used. @@ -144,11 +118,19 @@ pub fn start_ws( builder = builder.set_allowed_origins(cors)?; } - let server = tokio::task::block_in_place(|| rt.block_on(builder.build(addrs)))?; - - let rpc_api = build_rpc_api(module); - let handle = server.start(rpc_api)?; + let rpc_api = build_rpc_api(rpc_api); + let handle = if let Some(prometheus_registry) = prometheus_registry { + let metrics = RpcMetrics::new(&prometheus_registry)?; + let middleware = RpcMiddleware::new(metrics, "ws".into()); + let builder = builder.set_middleware(middleware); + let server = tokio::task::block_in_place(|| rt.block_on(builder.build(addrs)))?; + server.start(rpc_api)? + } else { + let server = tokio::task::block_in_place(|| rt.block_on(builder.build(addrs)))?; + server.start(rpc_api)? + }; + log::info!("Starting JSON-RPC WS server: addrs={:?}, allowed origins={:?}", addrs, cors); Ok(handle) } diff --git a/client/rpc-servers/src/middleware.rs b/client/rpc-servers/src/middleware.rs new file mode 100644 index 0000000000000..1c265790f96c5 --- /dev/null +++ b/client/rpc-servers/src/middleware.rs @@ -0,0 +1,171 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! RPC middlware to collect prometheus metrics on RPC calls. + +use jsonrpsee::types::middleware::Middleware; +use prometheus_endpoint::{ + register, Counter, CounterVec, HistogramOpts, HistogramVec, Opts, PrometheusError, Registry, + U64, +}; + +/// Metrics for RPC middleware storing information about the number of requests started/completed, +/// calls started/completed and their timings. +#[derive(Debug, Clone)] +pub struct RpcMetrics { + /// Number of RPC requests received since the server started. + requests_started: CounterVec, + /// Number of RPC requests completed since the server started. + requests_finished: CounterVec, + /// Histogram over RPC execution times. + calls_time: HistogramVec, + /// Number of calls started. + calls_started: CounterVec, + /// Number of calls completed. + calls_finished: CounterVec, + /// Number of Websocket sessions opened (Websocket only). + ws_sessions_opened: Option>, + /// Number of Websocket sessions closed (Websocket only). + ws_sessions_closed: Option>, +} + +impl RpcMetrics { + /// Create an instance of metrics + pub fn new(metrics_registry: &Registry) -> Result { + Ok(Self { + requests_started: register( + CounterVec::new( + Opts::new( + "rpc_requests_started", + "Number of RPC requests (not calls) received by the server.", + ), + &["protocol"], + )?, + metrics_registry, + )?, + requests_finished: register( + CounterVec::new( + Opts::new( + "rpc_requests_finished", + "Number of RPC requests (not calls) processed by the server.", + ), + &["protocol"], + )?, + metrics_registry, + )?, + calls_time: register( + HistogramVec::new( + HistogramOpts::new("rpc_calls_time", "Total time [μs] of processed RPC calls"), + &["protocol", "method"], + )?, + metrics_registry, + )?, + calls_started: register( + CounterVec::new( + Opts::new( + "rpc_calls_started", + "Number of received RPC calls (unique un-batched requests)", + ), + &["protocol", "method"], + )?, + metrics_registry, + )?, + calls_finished: register( + CounterVec::new( + Opts::new( + "rpc_calls_finished", + "Number of processed RPC calls (unique un-batched requests)", + ), + &["protocol", "method", "is_error"], + )?, + metrics_registry, + )?, + ws_sessions_opened: register( + Counter::new("rpc_sessions_opened", "Number of persistent RPC sessions opened")?, + metrics_registry, + )? + .into(), + ws_sessions_closed: register( + Counter::new("rpc_sessions_closed", "Number of persistent RPC sessions closed")?, + metrics_registry, + )? + .into(), + }) + } +} + +#[derive(Clone)] +/// Middleware for RPC calls +pub struct RpcMiddleware { + metrics: RpcMetrics, + transport_label: &'static str, +} + +impl RpcMiddleware { + /// Create a new [`RpcMiddleware`] with the provided [`RpcMetrics`]. + pub fn new(metrics: RpcMetrics, transport_label: &'static str) -> Self { + Self { metrics, transport_label } + } +} + +impl Middleware for RpcMiddleware { + type Instant = std::time::Instant; + + fn on_connect(&self) { + self.metrics.ws_sessions_opened.as_ref().map(|counter| counter.inc()); + } + + fn on_request(&self) -> Self::Instant { + let now = std::time::Instant::now(); + self.metrics.requests_started.with_label_values(&[self.transport_label]).inc(); + now + } + + fn on_call(&self, name: &str) { + log::trace!(target: "rpc_metrics", "[{}] on_call name={}", self.transport_label, name); + self.metrics + .calls_started + .with_label_values(&[self.transport_label, name]) + .inc(); + } + + fn on_result(&self, name: &str, success: bool, started_at: Self::Instant) { + const TRUE: &str = "true"; + const FALSE: &str = "false"; + let micros = started_at.elapsed().as_micros(); + log::trace!(target: "rpc_metrics", "[{}] on_result name={}, success={}, started_at={:?}; call took {}μs", self.transport_label, name, success, started_at, micros); + self.metrics + .calls_time + .with_label_values(&[self.transport_label, name]) + .observe(micros as _); + + self.metrics + .calls_finished + .with_label_values(&[self.transport_label, name, if success { TRUE } else { FALSE }]) + .inc(); + } + + fn on_response(&self, started_at: Self::Instant) { + log::trace!(target: "rpc_metrics", "[{}] on_response started_at={:?}", self.transport_label, started_at); + self.metrics.requests_finished.with_label_values(&[self.transport_label]).inc(); + } + + fn on_disconnect(&self) { + self.metrics.ws_sessions_closed.as_ref().map(|counter| counter.inc()); + } +} diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 01abae5f1dafa..c0e46937fb9e3 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -341,6 +341,7 @@ where &[http_addr, http_addr2], config.rpc_cors.as_ref(), config.rpc_max_payload, + config.prometheus_registry(), gen_rpc_module(deny_unsafe(ws_addr, &config.rpc_methods))?, config.tokio_handle.clone(), ) @@ -351,6 +352,7 @@ where config.rpc_ws_max_connections, config.rpc_cors.as_ref(), config.rpc_max_payload, + config.prometheus_registry(), gen_rpc_module(deny_unsafe(http_addr, &config.rpc_methods))?, config.tokio_handle.clone(), )