diff --git a/modules/infra/api/gr_infra.h b/modules/infra/api/gr_infra.h index 4bb0b55f9..7c110d8c0 100644 --- a/modules/infra/api/gr_infra.h +++ b/modules/infra/api/gr_infra.h @@ -195,6 +195,23 @@ struct gr_infra_iface_set_req { // struct gr_infra_iface_set_resp { }; +#define GR_INFRA_IFACE_STATS_GET REQUEST_TYPE(GR_INFRA_MODULE, 0x0006) + +// struct gr_infra_iface_stats_get_req { }; + +struct gr_iface_stats { + uint16_t iface_id; + uint64_t rx_packets; + uint64_t rx_bytes; + uint64_t tx_packets; + uint64_t tx_bytes; +}; + +struct gr_infra_iface_stats_get_resp { + uint16_t n_stats; + struct gr_iface_stats stats[/* n_stats */]; +}; + // port rxqs /////////////////////////////////////////////////////////////////// #define GR_INFRA_RXQ_LIST REQUEST_TYPE(GR_INFRA_MODULE, 0x0010) @@ -309,4 +326,36 @@ struct gr_infra_cpu_affinity_set_req { // struct gr_infra_cpu_affinity_set_resp { }; +// Helper function to convert iface type enum to string +static inline const char *iface_type_to_str(gr_iface_type_t type) { + switch (type) { + case GR_IFACE_TYPE_UNDEF: + return "undef"; + case GR_IFACE_TYPE_LOOPBACK: + return "loopback"; + case GR_IFACE_TYPE_PORT: + return "port"; + case GR_IFACE_TYPE_VLAN: + return "vlan"; + case GR_IFACE_TYPE_IPIP: + return "ipip"; + case GR_IFACE_TYPE_COUNT: + break; + } + return "?"; +} + +// Helper function to convert iface mode enum to string +static inline const char *iface_mode_to_str(gr_iface_mode_t mode) { + switch (mode) { + case GR_IFACE_MODE_L3: + return "l3"; + case GR_IFACE_MODE_L1_XC: + return "l1-xc"; + case GR_IFACE_MODE_COUNT: + break; + } + return "?"; +} + #endif diff --git a/modules/infra/api/stats.c b/modules/infra/api/stats.c index 19a78e2f0..c532cb20e 100644 --- a/modules/infra/api/stats.c +++ b/modules/infra/api/stats.c @@ -200,17 +200,74 @@ static struct api_out stats_reset(const void * /*request*/, void ** /*response*/ atomic_store(&worker->stats_reset, true); iface = NULL; - while ((iface = iface_next(GR_IFACE_TYPE_PORT, iface)) != NULL) { - struct iface_info_port *port = (struct iface_info_port *)iface->info; - if ((ret = rte_eth_stats_reset(port->port_id)) < 0) - return api_out(-ret, 0); - if ((ret = rte_eth_xstats_reset(port->port_id)) < 0) - return api_out(-ret, 0); + + while ((iface = iface_next(GR_IFACE_TYPE_UNDEF, iface)) != NULL) { + struct iface_stats *sw_stats = iface_get_stats(iface->id); + // Reset software stats for all interface types. + if (sw_stats != NULL) { + memset(sw_stats, 0, sizeof(*sw_stats)); + } + + if (iface->type == GR_IFACE_TYPE_PORT) { + struct iface_info_port *port = (struct iface_info_port *)iface->info; + if ((ret = rte_eth_stats_reset(port->port_id)) < 0) + return api_out(-ret, 0); + if ((ret = rte_eth_xstats_reset(port->port_id)) < 0) + return api_out(-ret, 0); + } } return api_out(0, 0); } +static struct api_out iface_stats_get(const void * /*request*/, void **response) { + struct gr_infra_iface_stats_get_resp *resp = NULL; + struct gr_iface_stats *stats_vec = NULL; + struct iface *iface = NULL; + int ret = 0; + + while ((iface = iface_next(GR_IFACE_TYPE_UNDEF, iface)) != NULL) { + struct iface_stats *sw_stats = iface_get_stats(iface->id); + if (sw_stats == NULL) + continue; + + // Create a single stats object per interface + struct gr_iface_stats s; + s.iface_id = iface->id; + s.rx_packets = 0; + s.rx_bytes = 0; + s.tx_packets = 0; + s.tx_bytes = 0; + + // Aggregate per-core stats + for (int i = 0; i < RTE_MAX_LCORE; i++) { + s.rx_packets += sw_stats->rx_packets[i]; + s.rx_bytes += sw_stats->rx_bytes[i]; + s.tx_packets += sw_stats->tx_packets[i]; + s.tx_bytes += sw_stats->tx_bytes[i]; + } + gr_vec_add(stats_vec, s); + } + + size_t n_stats = gr_vec_len(stats_vec); + size_t len = sizeof(*resp) + n_stats * sizeof(struct gr_iface_stats); + if ((resp = calloc(1, len)) == NULL) { + ret = -ENOMEM; + goto err; + } + + resp->n_stats = n_stats; + memcpy(resp->stats, stats_vec, n_stats * sizeof(struct gr_iface_stats)); + + gr_vec_free(stats_vec); + *response = resp; + return api_out(0, len); +err: + gr_vec_free(stats_vec); + free(resp); + return api_out(-ret, 0); +} + static int telemetry_sw_stats_get(const char * /*cmd*/, const char * /*params*/, struct rte_tel_data *d) { struct stat *stats = NULL, *s; @@ -278,6 +335,131 @@ telemetry_sw_stats_get(const char * /*cmd*/, const char * /*params*/, struct rte return -1; } +static int +telemetry_ifaces_info_get(const char * /*cmd*/, const char * /*params*/, struct rte_tel_data *d) { + struct iface *iface = NULL; + + rte_tel_data_start_dict(d); + + while ((iface = iface_next(GR_IFACE_TYPE_UNDEF, iface)) != NULL) { + if (iface->type != GR_IFACE_TYPE_LOOPBACK) { + struct rte_tel_data *iface_container = rte_tel_data_alloc(); + if (iface_container == NULL) { + goto err; + } + rte_tel_data_start_dict(iface_container); + + rte_tel_data_add_dict_string(iface_container, "name", iface->name); + rte_tel_data_add_dict_uint(iface_container, "id", iface->id); + rte_tel_data_add_dict_string( + iface_container, "type", iface_type_to_str(iface->type) + ); + rte_tel_data_add_dict_uint(iface_container, "mtu", iface->mtu); + + struct rte_tel_data *flags_array = rte_tel_data_alloc(); + if (flags_array == NULL) { + rte_tel_data_free(iface_container); + goto err; + } + rte_tel_data_start_array(flags_array, RTE_TEL_STRING_VAL); + if (iface->flags & GR_IFACE_F_UP) + rte_tel_data_add_array_string(flags_array, "up"); + if (iface->state & GR_IFACE_S_RUNNING) + rte_tel_data_add_array_string(flags_array, "running"); + rte_tel_data_add_dict_container(iface_container, "flags", flags_array, 0); + + rte_tel_data_add_dict_string( + iface_container, "mode", iface_mode_to_str(iface->mode) + ); + rte_tel_data_add_dict_uint(iface_container, "vrf_id", iface->vrf_id); + + struct rte_tel_data *stats_container = rte_tel_data_alloc(); + if (stats_container == NULL) { + rte_tel_data_free(iface_container); + goto err; + } + rte_tel_data_start_dict(stats_container); + + // Software stats + struct iface_stats *sw_stats = iface_get_stats(iface->id); + if (sw_stats != NULL) { + uint64_t rx_pkts = 0, rx_bytes = 0, tx_pkts = 0, tx_bytes = 0; + for (int i = 0; i < RTE_MAX_LCORE; i++) { + rx_pkts += sw_stats->rx_packets[i]; + rx_bytes += sw_stats->rx_bytes[i]; + tx_pkts += sw_stats->tx_packets[i]; + tx_bytes += sw_stats->tx_bytes[i]; + } + rte_tel_data_add_dict_uint(stats_container, "rx_packets", rx_pkts); + rte_tel_data_add_dict_uint(stats_container, "rx_bytes", rx_bytes); + rte_tel_data_add_dict_uint(stats_container, "tx_packets", tx_pkts); + rte_tel_data_add_dict_uint(stats_container, "tx_bytes", tx_bytes); + } + + // Get hardware stats for physical ports. + if (iface->type == GR_IFACE_TYPE_PORT) { + struct iface_info_port *port = (struct + iface_info_port *)iface->info; + + struct rte_eth_stats eth_stats; + if (rte_eth_stats_get(port->port_id, ð_stats) == 0) { + rte_tel_data_add_dict_uint( + stats_container, "rx_missed", eth_stats.imissed + ); + rte_tel_data_add_dict_uint( + stats_container, "tx_errors", eth_stats.oerrors + ); + } + + int ret = rte_eth_xstats_get(port->port_id, NULL, 0); + if (ret > 0) { + unsigned num = ret; + struct rte_eth_xstat *xstats = calloc(num, sizeof(*xstats)); + struct rte_eth_xstat_name *names = calloc( + num, sizeof(*names) + ); + if (xstats != NULL && names != NULL + && rte_eth_xstats_get_names(port->port_id, names, num) + == (int)num + && rte_eth_xstats_get(port->port_id, xstats, num) + == (int)num) { + for (unsigned i = 0; i < num; i++) { + if (xstats[i].value > 0) { + rte_tel_data_add_dict_uint( + stats_container, + names[i].name, + xstats[i].value + ); + } + } + } + free(xstats); + free(names); + } + + if (rte_tel_data_add_dict_container( + iface_container, "statistics", stats_container, 0 + ) + != 0) { + rte_tel_data_free(stats_container); + rte_tel_data_free(iface_container); + goto err; + } + } + + if (rte_tel_data_add_dict_container(d, iface->name, iface_container, 0) + != 0) { + rte_tel_data_free(iface_container); + goto err; + } + } + } + return 0; + +err: + return -1; +} + static struct gr_api_handler stats_get_handler = { .name = "stats get", .request_type = GR_INFRA_STATS_GET, @@ -290,12 +472,24 @@ static struct gr_api_handler stats_reset_handler = { .callback = stats_reset, }; +static struct gr_api_handler iface_stats_get_handler = { + .name = "iface stats get", + .request_type = GR_INFRA_IFACE_STATS_GET, + .callback = iface_stats_get, +}; + RTE_INIT(infra_stats_init) { gr_register_api_handler(&stats_get_handler); gr_register_api_handler(&stats_reset_handler); + gr_register_api_handler(&iface_stats_get_handler); rte_telemetry_register_cmd( "/grout/stats/graph", telemetry_sw_stats_get, "Returns statistics of each graph node. No parameters" ); + rte_telemetry_register_cmd( + "/grout/iface", + telemetry_ifaces_info_get, + "Returns information per interface. No parameters" + ); } diff --git a/modules/infra/cli/iface.c b/modules/infra/cli/iface.c index b53fac174..7ba31f5f2 100644 --- a/modules/infra/cli/iface.c +++ b/modules/infra/cli/iface.c @@ -327,12 +327,75 @@ static cmd_status_t iface_list(const struct gr_api_client *c, const struct ec_pn return CMD_ERROR; } +static cmd_status_t iface_stats(const struct gr_api_client *c, const struct ec_pnode * /*p*/) { + struct gr_infra_iface_stats_get_resp *resp = NULL; + struct libscols_table *table = NULL; + cmd_status_t status = CMD_ERROR; + void *resp_ptr = NULL; + int ret; + + // Send the new API request and wait for the response + ret = gr_api_client_send_recv(c, GR_INFRA_IFACE_STATS_GET, 0, NULL, &resp_ptr); + if (ret < 0) { + errorf("failed to get interface stats: %s", strerror(-ret)); + goto end; + } + + resp = resp_ptr; + + table = scols_new_table(); + if (table == NULL) { + errorf("failed to create table: %s", strerror(errno)); + goto end; + } + + scols_table_new_column(table, "INTERFACE", 0, 0); + scols_table_new_column(table, "RX_PACKETS", SCOLS_FL_RIGHT, 0); + scols_table_new_column(table, "RX_BYTES", SCOLS_FL_RIGHT, 0); + scols_table_new_column(table, "TX_PACKETS", SCOLS_FL_RIGHT, 0); + scols_table_new_column(table, "TX_BYTES", SCOLS_FL_RIGHT, 0); + scols_table_set_column_separator(table, " "); + + for (uint16_t i = 0; i < resp->n_stats; i++) { + struct libscols_line *line = scols_table_new_line(table, NULL); + struct gr_iface iface; + if (line == NULL) { + errorf("failed to create line: %s", strerror(errno)); + goto end; + } + + if (iface_from_id(c, resp->stats[i].iface_id, &iface) == 0) + scols_line_set_data(line, 0, iface.name); + else + scols_line_set_data(line, 0, "?"); + + scols_line_sprintf(line, 1, "%lu", resp->stats[i].rx_packets); + scols_line_sprintf(line, 2, "%lu", resp->stats[i].rx_bytes); + scols_line_sprintf(line, 3, "%lu", resp->stats[i].tx_packets); + scols_line_sprintf(line, 4, "%lu", resp->stats[i].tx_bytes); + } + + scols_print_table(table); + status = CMD_SUCCESS; + +end: + if (table) + scols_unref_table(table); + free(resp_ptr); + return status; +} + static cmd_status_t iface_show(const struct gr_api_client *c, const struct ec_pnode *p) { const struct cli_iface_type *type; struct gr_iface iface; - if (arg_str(p, "NAME") == NULL || arg_str(p, "TYPE") != NULL) + if (arg_str(p, "stats") != NULL) { + return iface_stats(c, p); + } + + if (arg_str(p, "NAME") == NULL || arg_str(p, "TYPE") != NULL) { return iface_list(c, p); + } if (iface_from_name(c, arg_str(p, "NAME"), &iface) < 0) return CMD_ERROR; @@ -385,7 +448,7 @@ static int ctx_init(struct ec_node *root) { return ret; ret = CLI_COMMAND( CLI_CONTEXT(root, CTX_SHOW, CTX_ARG("interface", "Display interface details.")), - "[(name NAME)|(type TYPE)]", + "[(name NAME)|(type TYPE)|stats]", iface_show, "Show interface details.", with_help( @@ -395,7 +458,8 @@ static int ctx_init(struct ec_node *root) { with_help( "Show only this type of interface.", ec_node_dyn("TYPE", complete_iface_types, NULL) - ) + ), + with_help("Show interface statistics.", ec_node_str("stats", "stats")) ); return ret; diff --git a/modules/infra/control/gr_iface.h b/modules/infra/control/gr_iface.h index 3be111126..cf5980d13 100644 --- a/modules/infra/control/gr_iface.h +++ b/modules/infra/control/gr_iface.h @@ -72,6 +72,15 @@ struct iface *get_vrf_iface(uint16_t vrf_id); struct iface *iface_loopback_create(uint16_t vrf_id); int iface_loopback_delete(uint16_t vrf_id); +struct __rte_cache_aligned iface_stats { + uint64_t rx_packets[RTE_MAX_LCORE]; + uint64_t rx_bytes[RTE_MAX_LCORE]; + uint64_t tx_packets[RTE_MAX_LCORE]; + uint64_t tx_bytes[RTE_MAX_LCORE]; +}; + +struct iface_stats *iface_get_stats(uint16_t ifid); + #define MAX_IFACES 1024 #define MAX_VRFS 256 diff --git a/modules/infra/control/iface.c b/modules/infra/control/iface.c index 835d4e68d..ff1578a82 100644 --- a/modules/infra/control/iface.c +++ b/modules/infra/control/iface.c @@ -21,6 +21,8 @@ static STAILQ_HEAD(, iface_type) types = STAILQ_HEAD_INITIALIZER(types); +static struct iface_stats stats[MAX_IFACES]; + struct iface_type *iface_type_get(gr_iface_type_t type_id) { struct iface_type *t; STAILQ_FOREACH (t, &types, next) @@ -90,6 +92,8 @@ struct iface *iface_create(const struct gr_iface *conf, const void *api_info) { ifaces[ifid] = iface; + memset(&stats[ifid], 0, sizeof(stats[ifid])); + gr_event_push(GR_EVENT_IFACE_POST_ADD, iface); return iface; @@ -172,6 +176,10 @@ struct iface *iface_from_id(uint16_t ifid) { return iface; } +struct iface_stats *iface_get_stats(uint16_t ifid) { + return &stats[ifid]; +} + int iface_get_eth_addr(uint16_t ifid, struct rte_ether_addr *mac) { struct iface *iface = iface_from_id(ifid); struct iface_type *type; diff --git a/modules/infra/datapath/eth_input.c b/modules/infra/datapath/eth_input.c index 8fde9ccf6..f02ece01c 100644 --- a/modules/infra/datapath/eth_input.c +++ b/modules/infra/datapath/eth_input.c @@ -84,6 +84,11 @@ eth_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, u } iface = eth_in->iface; } + + struct iface_stats *stats = iface_get_stats(eth_in->iface->id); + stats->rx_packets[rte_lcore_id()] += 1; + stats->rx_bytes[rte_lcore_id()] += rte_pktmbuf_pkt_len(m); + if (unlikely(rte_is_multicast_ether_addr(ð->dst_addr))) { if (rte_is_broadcast_ether_addr(ð->dst_addr)) eth_in->domain = ETH_DOMAIN_BROADCAST; diff --git a/modules/infra/datapath/eth_output.c b/modules/infra/datapath/eth_output.c index babeee1f6..181ea0868 100644 --- a/modules/infra/datapath/eth_output.c +++ b/modules/infra/datapath/eth_output.c @@ -27,6 +27,7 @@ eth_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, const struct iface_info_port *port; struct eth_output_mbuf_data *priv; struct iface_info_vlan *sub; + const struct iface *iface; struct rte_vlan_hdr *vlan; struct rte_ether_hdr *eth; struct rte_mbuf *mbuf; @@ -34,6 +35,7 @@ eth_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, for (uint16_t i = 0; i < nb_objs; i++) { mbuf = objs[i]; priv = eth_output_mbuf_data(mbuf); + iface = priv->iface; vlan = NULL; switch (priv->iface->type) { @@ -70,6 +72,10 @@ eth_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, eth->ether_type = priv->ether_type; mbuf->port = port->port_id; + struct iface_stats *stats = iface_get_stats(iface->id); + stats->tx_packets[rte_lcore_id()] += 1; + stats->tx_bytes[rte_lcore_id()] += rte_pktmbuf_pkt_len(mbuf); + if (gr_mbuf_is_traced(mbuf)) { struct eth_trace_data *t = gr_mbuf_trace_add(mbuf, node, sizeof(*t)); t->eth.dst_addr = eth->dst_addr; diff --git a/modules/infra/datapath/l1_xconnect.c b/modules/infra/datapath/l1_xconnect.c index 5b9944a76..14dbc658c 100644 --- a/modules/infra/datapath/l1_xconnect.c +++ b/modules/infra/datapath/l1_xconnect.c @@ -28,10 +28,19 @@ l1_xconnect_process(struct rte_graph *graph, struct rte_node *node, void **objs, mbuf = objs[i]; iface = mbuf_data(mbuf)->iface; peer = iface_from_id(iface->domain_id); + + struct iface_stats *rx_stats = iface_get_stats(iface->id); + rx_stats->rx_packets[rte_lcore_id()]++; + rx_stats->rx_bytes[rte_lcore_id()] += rte_pktmbuf_pkt_len(mbuf); + if (peer->type == GR_IFACE_TYPE_PORT) { port = (const struct iface_info_port *)peer->info; mbuf->port = port->port_id; edge = TX; + + struct iface_stats *tx_stats = iface_get_stats(peer->id); + tx_stats->tx_packets[rte_lcore_id()]++; + tx_stats->tx_bytes[rte_lcore_id()] += rte_pktmbuf_pkt_len(mbuf); } else { edge = NO_PORT; } diff --git a/modules/ipip/datapath_in.c b/modules/ipip/datapath_in.c index 545251695..ae859466a 100644 --- a/modules/ipip/datapath_in.c +++ b/modules/ipip/datapath_in.c @@ -35,8 +35,9 @@ ipip_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, struct eth_input_mbuf_data *eth_data; struct ip_local_mbuf_data *ip_data; ip4_addr_t last_src, last_dst; - uint16_t last_vrf_id; + struct iface_stats *stats; struct rte_mbuf *mbuf; + uint16_t last_vrf_id; struct iface *ipip; rte_edge_t edge; @@ -67,6 +68,9 @@ ipip_input_process(struct rte_graph *graph, struct rte_node *node, void **objs, eth_data->iface = ipip; eth_data->domain = ETH_DOMAIN_LOCAL; edge = IP_INPUT; + stats = iface_get_stats(ipip->id); + stats->rx_packets[rte_lcore_id()] += 1; + stats->rx_bytes[rte_lcore_id()] += rte_pktmbuf_pkt_len(mbuf); next: if (gr_mbuf_is_traced(mbuf) || (ipip && ipip->flags & GR_IFACE_F_PACKET_TRACE)) { struct trace_ipip_data *t = gr_mbuf_trace_add(mbuf, node, sizeof(*t)); diff --git a/modules/ipip/datapath_out.c b/modules/ipip/datapath_out.c index 923765b1c..d9d266978 100644 --- a/modules/ipip/datapath_out.c +++ b/modules/ipip/datapath_out.c @@ -69,6 +69,10 @@ ipip_output_process(struct rte_graph *graph, struct rte_node *node, void **objs, } ip_set_fields(outer, &tunnel); + struct iface_stats *stats = iface_get_stats(iface->id); + stats->tx_packets[rte_lcore_id()] += 1; + stats->tx_bytes[rte_lcore_id()] += rte_pktmbuf_pkt_len(mbuf); + // Resolve nexthop for the encapsulated packet. ip_data->nh = fib4_lookup(iface->vrf_id, ipip->remote); edge = IP_OUTPUT;