zebra: vlan to dplane Offload from main

Trigger: Zebra core seen when we convert l2vni to l3vni and back

BackTrace:
/usr/lib/x86_64-linux-gnu/frr/libfrr.so.0(_zlog_assert_failed+0xe9) [0x7f4af96989d9]
/usr/lib/frr/zebra(zebra_vxlan_if_vni_up+0x250) [0x5561022ae030]
/usr/lib/frr/zebra(netlink_vlan_change+0x2f4) [0x5561021fd354]
/usr/lib/frr/zebra(netlink_parse_info+0xff) [0x55610220d37f]
/usr/lib/frr/zebra(+0xc264a) [0x55610220d64a]
/usr/lib/x86_64-linux-gnu/frr/libfrr.so.0(thread_call+0x7d) [0x7f4af967e96d]
/usr/lib/x86_64-linux-gnu/frr/libfrr.so.0(frr_run+0xe8) [0x7f4af9637588]
/usr/lib/frr/zebra(main+0x402) [0x5561021f4d32]
/lib/x86_64-linux-gnu/libc.so.6(+0x2724a) [0x7f4af932624a]
/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0x85) [0x7f4af9326305]
/usr/lib/frr/zebra(_start+0x21) [0x5561021f72f1]

Root Cause:
In working case,
 - We get a RTM_NEWLINK whose ctx is enqueued by zebra dplane and
   dequeued by zebra main and processed i.e.
   (102000 is deleted from vxlan99) before we handle RTM_NEWVLAN.
 - So in handling of NEWVLAN (vxlan99) we bail out since find with
   vlan id 703 does not exist.

root@leaf2:mgmt:/var/log/frr# cat ~/raja_logs/working/nocras.log  | grep "RTM_NEWLINK\|QUEUED\|vxlan99\|in thread"
2024/07/18 23:09:33.741105 ZEBRA: [KMXEB-K771Y] netlink_parse_info: netlink-dp-in (NS 0) type RTM_NEWLINK(16), len=616, seq=0, pid=0
2024/07/18 23:09:33.744061 ZEBRA: [K8FXY-V65ZJ] Intf dplane ctx 0x7f2244000cf0, op INTF_INSTALL, ifindex (65), result QUEUED
2024/07/18 23:09:33.767240 ZEBRA: [KMXEB-K771Y] netlink_parse_info: netlink-dp-in (NS 0) type RTM_NEWLINK(16), len=508, seq=0, pid=0
2024/07/18 23:09:33.767380 ZEBRA: [K8FXY-V65ZJ] Intf dplane ctx 0x7f2244000cf0, op INTF_INSTALL, ifindex (73), result QUEUED
2024/07/18 23:09:33.767389 ZEBRA: [NVFT0-HS1EX] INTF_INSTALL for vxlan99(73)
2024/07/18 23:09:33.767404 ZEBRA: [TQR2A-H2RFY] Vlan-Vni(1186:1186-6000002:6000002) update for VxLAN IF vxlan99(73)
2024/07/18 23:09:33.767422 ZEBRA: [TP4VP-XZ627] Del L2-VNI 102000 intf vxlan99(73)
2024/07/18 23:09:33.767858 ZEBRA: [QYXB9-6RNNK] RTM_NEWVLAN bridge IF vxlan99 NS 0
2024/07/18 23:09:33.767866 ZEBRA: [KKZGZ-8PCDW] Cannot find VNI for VID (703) IF vxlan99 for vlan state update >>>>BAIL OUT

In failure case,
 - The NEWVLAN is received first even before processing RTM_NEWLINK.
 - Since the vxlan id 102000 is not removed from the vxlan99,
   the find with vlan id 703 returns the 102000 one and we invoke
   zebra_vxlan_if_vni_up where the interfaces don't match and assert.

root@leaf2:mgmt:/var/log/frr# cat ~/raja_logs/noworking/crash.log | grep "RTM_NEWLINK\|QUEUED\|vxlan99\|in thread"
2024/07/18 22:26:43.829370 ZEBRA: [KMXEB-K771Y] netlink_parse_info: netlink-dp-in (NS 0) type RTM_NEWLINK(16), len=616, seq=0, pid=0
2024/07/18 22:26:43.829646 ZEBRA: [K8FXY-V65ZJ] Intf dplane ctx 0x7fe07c026d30, op INTF_INSTALL, ifindex (65), result QUEUED
2024/07/18 22:26:43.853930 ZEBRA: [QYXB9-6RNNK] RTM_NEWVLAN bridge IF vxlan99 NS 0
2024/07/18 22:26:43.853949 ZEBRA: [K61WJ-XQQ3X] Intf vxlan99(73) L2-VNI 102000 is UP >>> VLAN PROCESSED BEFORE INTF EVENT
2024/07/18 22:26:43.853951 ZEBRA: [SPV50-BX2RP] RAJA zevpn_vxlanif vxlan48 and ifp vxlan99
2024/07/18 22:26:43.854005 ZEBRA: [KMXEB-K771Y] netlink_parse_info: netlink-dp-in (NS 0) type RTM_NEWLINK(16), len=508, seq=0, pid=0
2024/07/18 22:26:43.854241 ZEBRA: [KMXEB-K771Y] netlink_parse_info: netlink-dp-in (NS 0) type RTM_NEWLINK(16), len=516, seq=0, pid=0
2024/07/18 22:26:43.854251 ZEBRA: [KMXEB-K771Y] netlink_parse_info: netlink-dp-in (NS 0) type RTM_NEWLINK(16), len=544, seq=0, pid=0
ZEBRA: in thread kernel_read scheduled from zebra/kernel_netlink.c:505 kernel_read()

Fix:
Similar to FRRouting#13396, where link change
handling was offloaded to dplane, same is being done for vlan events.

Note: Prior to this change, zebra main thread was interested in the
RTNLGRP_BRVLAN. So all the kernel events pertaining to vlan was
handled by zebra main.

With this change change as well the handling of vlan events is still
with Zebra main. However we offload it via Dplane thread.

Ticket :#3878175

Signed-off-by: Rajasekar Raja <[email protected]>

Author: raja-rajasekar

zebra/dpdk/zebra_dplane_dpdk.c

@@ -407,6 +407,7 @@ static void zd_dpdk_rule_update(struct zebra_dplane_ctx *ctx)
 	case DPLANE_OP_INTF_UPDATE:
 	case DPLANE_OP_INTF_DELETE:
 	case DPLANE_OP_ROUTE_LAST:
+	case DPLANE_OP_VLAN_INSTALL,
 		break;
 	}
 }
@@ -467,6 +468,7 @@ static void zd_dpdk_process_update(struct zebra_dplane_ctx *ctx)
 	case DPLANE_OP_INTF_UPDATE:
 	case DPLANE_OP_INTF_DELETE:
 	case DPLANE_OP_ROUTE_LAST:
+	case DPLANE_OP_VLAN_INSTALL,
 		atomic_fetch_add_explicit(&dpdk_stat->ignored_updates, 1,
 					  memory_order_relaxed);
 

zebra/dplane_fpm_nl.c

@@ -990,6 +990,7 @@ static int fpm_nl_enqueue(struct fpm_nl_ctx *fnc, struct zebra_dplane_ctx *ctx)
 	case DPLANE_OP_SRV6_ENCAP_SRCADDR_SET:
 	case DPLANE_OP_NONE:
 	case DPLANE_OP_STARTUP_STAGE:
+	case DPLANE_OP_VLAN_INSTALL:
 		break;
 
 	}

zebra/if_netlink.c

@@ -64,7 +64,6 @@
 #include "zebra/netconf_netlink.h"
 #include "zebra/zebra_trace.h"
 #include "zebra/zebra_evpn_arp_nd.h"
-#include "zebra/zebra_trace.h"
 
 extern struct zebra_privs_t zserv_privs;
 
@@ -1907,6 +1906,20 @@ int netlink_tunneldump_read(struct zebra_ns *zns)
 	return 0;
 }
 
+static uint8_t netlink_get_dplane_vlan_state(uint8_t state)
+{
+	if (state == BR_STATE_LISTENING)
+		return ZEBRA_DPLANE_BR_STATE_LISTENING;
+	else if (state == BR_STATE_LEARNING)
+		return ZEBRA_DPLANE_BR_STATE_LEARNING;
+	else if (state == BR_STATE_FORWARDING)
+		return ZEBRA_DPLANE_BR_STATE_FORWARDING;
+	else if (state == BR_STATE_BLOCKING)
+		return ZEBRA_DPLANE_BR_STATE_BLOCKING;
+
+	return ZEBRA_DPLANE_BR_STATE_DISABLED;
+}
+
 /**
  * netlink_vlan_change() - Read in change about vlans from the kernel
  *
@@ -1919,14 +1932,16 @@ int netlink_tunneldump_read(struct zebra_ns *zns)
 int netlink_vlan_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 {
 	int len, rem;
-	struct interface *ifp;
 	struct br_vlan_msg *bvm;
 	struct bridge_vlan_info *vinfo;
 	struct rtattr *vtb[BRIDGE_VLANDB_ENTRY_MAX + 1] = {};
 	struct rtattr *attr;
 	uint8_t state;
 	uint32_t vrange;
 	int type;
+	uint32_t count = 0;
+	struct zebra_dplane_ctx *ctx = NULL;
+	struct zebra_vxlan_vlan_array *vlan_array = NULL;
 
 	/* We only care about state changes for now */
 	if (!(h->nlmsg_type == RTM_NEWVLAN))
@@ -1946,27 +1961,10 @@ int netlink_vlan_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 	if (bvm->family != AF_BRIDGE)
 		return 0;
 
-	ifp = if_lookup_by_index_per_ns(zebra_ns_lookup(ns_id), bvm->ifindex);
-	if (!ifp) {
-		zlog_debug("Cannot find bridge-vlan IF (%u) for vlan update",
-			   bvm->ifindex);
-		frrtrace(3, frr_zebra, netlink_intf_err, INTF_INVALID_NAME, bvm->ifindex, 4);
-		return 0;
-	}
-
-	if (!IS_ZEBRA_IF_VXLAN(ifp)) {
-		if (IS_ZEBRA_DEBUG_KERNEL)
-			zlog_debug("Ignoring non-vxlan IF (%s) for vlan update",
-				   ifp->name);
-
-		frrtrace(3, frr_zebra, netlink_intf_err, ifp->name, INTF_INVALID_INDEX, 5);
-		return 0;
-	}
-
-	if (IS_ZEBRA_DEBUG_KERNEL || IS_ZEBRA_DEBUG_VXLAN)
-		zlog_debug("%s %s IF %s NS %u",
-			   nl_msg_type_to_str(h->nlmsg_type),
-			   nl_family_to_str(bvm->family), ifp->name, ns_id);
+	ctx = dplane_ctx_alloc();
+	dplane_ctx_set_ns_id(ctx, ns_id);
+	dplane_ctx_set_op(ctx, DPLANE_OP_VLAN_INSTALL);
+	dplane_ctx_set_vlan_ifindex(ctx, bvm->ifindex);
 
 	/* Loop over "ALL" BRIDGE_VLANDB_ENTRY */
 	rem = len;
@@ -1997,28 +1995,39 @@ int netlink_vlan_change(struct nlmsghdr *h, ns_id_t ns_id, int startup)
 		if (!vtb[BRIDGE_VLANDB_ENTRY_STATE])
 			continue;
 
+		count++;
+		vlan_array =
+			XREALLOC(MTYPE_VLAN_CHANGE_ARR, vlan_array,
+				 sizeof(struct zebra_vxlan_vlan_array) +
+					 count * sizeof(struct zebra_vxlan_vlan));
+
+		memset(&vlan_array->vlans[count - 1], 0,
+		       sizeof(struct zebra_vxlan_vlan));
+
 		state = *(uint8_t *)RTA_DATA(vtb[BRIDGE_VLANDB_ENTRY_STATE]);
 
 		if (vtb[BRIDGE_VLANDB_ENTRY_RANGE])
 			vrange = *(uint32_t *)RTA_DATA(
 				vtb[BRIDGE_VLANDB_ENTRY_RANGE]);
 
-		if (IS_ZEBRA_DEBUG_KERNEL || IS_ZEBRA_DEBUG_VXLAN) {
-			if (vrange)
-				zlog_debug("VLANDB_ENTRY: VID (%u-%u) state=%s",
-					   vinfo->vid, vrange,
-					   port_state2str(state));
-			else
-				zlog_debug("VLANDB_ENTRY: VID (%u) state=%s",
-					   vinfo->vid, port_state2str(state));
-		}
-		if (vinfo)
-			frrtrace(7, frr_zebra, netlink_vlan_change, h, bvm, ns_id, vinfo, vrange,
-				 state, ifp);
-		vlan_id_range_state_change(
-			ifp, vinfo->vid, (vrange ? vrange : vinfo->vid), state);
+		vlan_array->vlans[count - 1].state =
+			netlink_get_dplane_vlan_state(state);
+		vlan_array->vlans[count - 1].vid = vinfo->vid;
+		vlan_array->vlans[count - 1].vrange = vrange;
 	}
 
+	if (count) {
+		vlan_array->count = count;
+		dplane_ctx_set_vxlan_vlan_array(ctx, vlan_array);
+		if (IS_ZEBRA_DEBUG_KERNEL || IS_ZEBRA_DEBUG_VXLAN)
+			zlog_debug("RTM_NEWVLAN for ifindex %u NS %u, enqueuing for zebra main",
+				   bvm->ifindex, ns_id);
+
+		dplane_provider_enqueue_to_zebra(ctx);
+	} else
+		dplane_ctx_fini(&ctx);
+
+
 	return 0;
 }
 

zebra/kernel_netlink.c

@@ -430,10 +430,6 @@ static int netlink_information_fetch(struct nlmsghdr *h, ns_id_t ns_id,
 	case RTM_NEWTFILTER:
 	case RTM_DELTFILTER:
 		return netlink_tfilter_change(h, ns_id, startup);
-	case RTM_NEWVLAN:
-		return netlink_vlan_change(h, ns_id, startup);
-	case RTM_DELVLAN:
-		return netlink_vlan_change(h, ns_id, startup);
 
 	/* Messages we may receive, but ignore */
 	case RTM_NEWCHAIN:
@@ -449,6 +445,8 @@ static int netlink_information_fetch(struct nlmsghdr *h, ns_id_t ns_id,
 	case RTM_NEWTUNNEL:
 	case RTM_DELTUNNEL:
 	case RTM_GETTUNNEL:
+	case RTM_NEWVLAN:
+	case RTM_DELVLAN:
 		return 0;
 	default:
 		/*
@@ -492,6 +490,10 @@ static int dplane_netlink_information_fetch(struct nlmsghdr *h, ns_id_t ns_id,
 	case RTM_DELLINK:
 		return netlink_link_change(h, ns_id, startup);
 
+	case RTM_NEWVLAN:
+	case RTM_DELVLAN:
+		return netlink_vlan_change(h, ns_id, startup);
+
 	default:
 		break;
 	}
@@ -1642,6 +1644,7 @@ static enum netlink_msg_status nl_put_msg(struct nl_batch *bth,
 	case DPLANE_OP_IPSET_ENTRY_ADD:
 	case DPLANE_OP_IPSET_ENTRY_DELETE:
 	case DPLANE_OP_STARTUP_STAGE:
+	case DPLANE_OP_VLAN_INSTALL:
 		return FRR_NETLINK_ERROR;
 
 	case DPLANE_OP_GRE_SET:
@@ -1883,8 +1886,8 @@ void kernel_init(struct zebra_ns *zns)
 	 * setsockopt multicast group subscriptions that don't fit in nl_groups
 	 */
 	grp = RTNLGRP_BRVLAN;
-	ret = setsockopt(zns->netlink.sock, SOL_NETLINK, NETLINK_ADD_MEMBERSHIP,
-			 &grp, sizeof(grp));
+	ret = setsockopt(zns->netlink_dplane_in.sock, SOL_NETLINK,
+			 NETLINK_ADD_MEMBERSHIP, &grp, sizeof(grp));
 
 	if (ret < 0)
 		zlog_notice(

zebra/kernel_socket.c

@@ -1626,6 +1626,7 @@ void kernel_update_multi(struct dplane_ctx_list_head *ctx_list)
 		case DPLANE_OP_INTF_ADDR_DEL:
 		case DPLANE_OP_STARTUP_STAGE:
 		case DPLANE_OP_SRV6_ENCAP_SRCADDR_SET:
+		case DPLANE_OP_VLAN_INSTALL:
 			zlog_err("Unhandled dplane data for %s",
 				 dplane_op2str(dplane_ctx_get_op(ctx)));
 			res = ZEBRA_DPLANE_REQUEST_FAILURE;

zebra/zebra_dplane.c

@@ -36,6 +36,8 @@ DEFINE_MTYPE_STATIC(ZEBRA, DP_PROV, "Zebra DPlane Provider");
 DEFINE_MTYPE_STATIC(ZEBRA, DP_NETFILTER, "Zebra Netfilter Internal Object");
 DEFINE_MTYPE_STATIC(ZEBRA, DP_NS, "DPlane NSes");
 
+DEFINE_MTYPE(ZEBRA, VLAN_CHANGE_ARR, "Vlan Change Array");
+
 #ifndef AOK
 #  define AOK 0
 #endif
@@ -375,6 +377,14 @@ struct dplane_srv6_encap_ctx {
 	struct in6_addr srcaddr;
 };
 
+/*
+ * VLAN info for the dataplane
+ */
+struct dplane_vlan_info {
+	ifindex_t ifindex;
+	struct zebra_vxlan_vlan_array *vlan_array;
+};
+
 /*
  * The context block used to exchange info about route updates across
  * the boundary between the zebra main context (and pthread) and the
@@ -421,6 +431,7 @@ struct zebra_dplane_ctx {
 		struct dplane_pw_info pw;
 		struct dplane_br_port_info br_port;
 		struct dplane_intf_info intf;
+		struct dplane_vlan_info vlan_info;
 		struct dplane_mac_info macinfo;
 		struct dplane_neigh_info neigh;
 		struct dplane_rule_info rule;
@@ -893,6 +904,11 @@ static void dplane_ctx_free_internal(struct zebra_dplane_ctx *ctx)
 	case DPLANE_OP_STARTUP_STAGE:
 	case DPLANE_OP_SRV6_ENCAP_SRCADDR_SET:
 		break;
+	case DPLANE_OP_VLAN_INSTALL:
+		if (ctx->u.vlan_info.vlan_array)
+			XFREE(MTYPE_VLAN_CHANGE_ARR,
+			      ctx->u.vlan_info.vlan_array);
+		break;
 	}
 }
 
@@ -1245,6 +1261,10 @@ const char *dplane_op2str(enum dplane_op_e op)
 	case DPLANE_OP_SRV6_ENCAP_SRCADDR_SET:
 		ret = "SRV6_ENCAP_SRCADDR_SET";
 		break;
+
+	case DPLANE_OP_VLAN_INSTALL:
+		ret = "NEW_VLAN";
+		break;
 	}
 
 	return ret;
@@ -3399,6 +3419,35 @@ uint32_t dplane_get_in_queue_len(void)
 				    memory_order_seq_cst);
 }
 
+void dplane_ctx_set_vlan_ifindex(struct zebra_dplane_ctx *ctx, ifindex_t ifindex)
+{
+	DPLANE_CTX_VALID(ctx);
+	ctx->u.vlan_info.ifindex = ifindex;
+}
+
+ifindex_t dplane_ctx_get_vlan_ifindex(struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return ctx->u.vlan_info.ifindex;
+}
+
+void dplane_ctx_set_vxlan_vlan_array(struct zebra_dplane_ctx *ctx,
+				     struct zebra_vxlan_vlan_array *vlan_array)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	ctx->u.vlan_info.vlan_array = vlan_array;
+}
+
+const struct zebra_vxlan_vlan_array *
+dplane_ctx_get_vxlan_vlan_array(struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return ctx->u.vlan_info.vlan_array;
+}
+
 /*
  * Internal helper that copies information from a zebra ns object; this is
  * called in the zebra main pthread context as part of dplane ctx init.
@@ -6868,6 +6917,12 @@ static void kernel_dplane_log_detail(struct zebra_dplane_ctx *ctx)
 			   dplane_op2str(dplane_ctx_get_op(ctx)),
 			   &ctx->u.srv6_encap.srcaddr);
 		break;
+
+	case DPLANE_OP_VLAN_INSTALL:
+		zlog_debug("Dplane %s on idx %u",
+			   dplane_op2str(dplane_ctx_get_op(ctx)),
+			   dplane_ctx_get_vlan_ifindex(ctx));
+		break;
 	}
 }
 
@@ -7042,6 +7097,7 @@ static void kernel_dplane_handle_result(struct zebra_dplane_ctx *ctx)
 	case DPLANE_OP_INTF_ADDR_ADD:
 	case DPLANE_OP_INTF_ADDR_DEL:
 	case DPLANE_OP_INTF_NETCONFIG:
+	case DPLANE_OP_VLAN_INSTALL:
 		break;
 
 	case DPLANE_OP_SRV6_ENCAP_SRCADDR_SET:

zebra/zebra_dplane.h

@@ -24,6 +24,8 @@
 extern "C" {
 #endif
 
+DECLARE_MTYPE(VLAN_CHANGE_ARR);
+
 /* Retrieve the dataplane API version number; see libfrr.h to decode major,
  * minor, sub version values.
  * Plugins should pay attention to the major version number, at least, to
@@ -204,6 +206,9 @@ enum dplane_op_e {
 	DPLANE_OP_TC_FILTER_DELETE,
 	DPLANE_OP_TC_FILTER_UPDATE,
 
+	/* VLAN update */
+	DPLANE_OP_VLAN_INSTALL,
+
 	/* Startup Control */
 	DPLANE_OP_STARTUP_STAGE,
 
@@ -213,6 +218,13 @@ enum dplane_op_e {
 	DPLANE_OP_ROUTE_LAST,
 };
 
+/* Operational status of Bridge Ports */
+#define ZEBRA_DPLANE_BR_STATE_DISABLED	 0x01
+#define ZEBRA_DPLANE_BR_STATE_LISTENING	 0x02
+#define ZEBRA_DPLANE_BR_STATE_LEARNING	 0x04
+#define ZEBRA_DPLANE_BR_STATE_FORWARDING 0x08
+#define ZEBRA_DPLANE_BR_STATE_BLOCKING	 0x10
+
 /*
  * The vxlan/evpn neighbor management code needs some values to use
  * when programming neighbor changes. Offer some platform-neutral values
@@ -1094,6 +1106,26 @@ void dplane_set_in_queue_limit(uint32_t limit, bool set);
 /* Retrieve the current queue depth of incoming, unprocessed updates */
 uint32_t dplane_get_in_queue_len(void);
 
+void dplane_ctx_set_vlan_ifindex(struct zebra_dplane_ctx *ctx,
+				 ifindex_t ifindex);
+ifindex_t dplane_ctx_get_vlan_ifindex(struct zebra_dplane_ctx *ctx);
+struct zebra_vxlan_vlan_array;
+
+/*
+ * In netlink_vlan_change(), the memory allocated for vlan_array is freed
+ * in two cases
+ *  1) Inline free in netlink_vlan_change() when there are no new
+ *     vlans to process i.e. nothing is enqueued to main thread.
+ *  2) Dplane-ctx takes over the vlan memory which gets freed in
+ *     rib_process_dplane_results() after handling the vlan install
+ *
+ * Note: MTYPE of interest for this purpose is MTYPE_VLAN_CHANGE_ARR
+ */
+void dplane_ctx_set_vxlan_vlan_array(struct zebra_dplane_ctx *ctx,
+				     struct zebra_vxlan_vlan_array *vlan_array);
+const struct zebra_vxlan_vlan_array *
+dplane_ctx_get_vxlan_vlan_array(struct zebra_dplane_ctx *ctx);
+
 /*
  * Vty/cli apis
  */