Pass CIBIR ID to XDP (#2479)

Co-authored-by: Matt Olson <maolson@microsoft.com>

Author: nibanks

scripts/prepare-machine.ps1

@@ -66,7 +66,10 @@ param (
     [switch]$NoCodeCoverage,
 
     [Parameter(Mandatory = $false)]
-    [switch]$Xdp
+    [switch]$Xdp,
+
+    [Parameter(Mandatory = $false)]
+    [switch]$Force
 )
 
 #Requires -RunAsAdministrator
@@ -137,12 +140,13 @@ function Download-CoreNet-Deps {
 }
 
 function Download-Xdp-Kit {
+    if ($Force) { rm -Force -Recurse $ArtifactsPath }
     if (!(Test-Path $ArtifactsPath)) { mkdir $ArtifactsPath }
     $XdpPath = Join-Path $ArtifactsPath "xdp"
     if (!(Test-Path $XdpPath)) {
         Write-Host "Downloading XDP Kit"
         $ZipPath = Join-Path $ArtifactsPath "xdp.zip"
-        Invoke-WebRequest -Uri "https://lolafiles.blob.core.windows.net/nibanks/xdp.zip" -OutFile $ZipPath
+        Invoke-WebRequest -Uri "https://lolafiles.blob.core.windows.net/nibanks/xdp-latest.zip" -OutFile $ZipPath
         Expand-Archive -Path $ZipPath -DestinationPath $XdpPath -Force
         Remove-Item -Path $ZipPath
     }

src/core/listener.c

@@ -303,6 +303,18 @@ MsQuicListenerStart(
 #ifdef QUIC_OWNING_PROCESS
     UdpConfig.OwningProcess = NULL;     // Owning process not supported for listeners.
 #endif
+#ifdef QUIC_USE_RAW_DATAPATH
+    UdpConfig.CibirIdLength = Listener->CibirId[0];
+    UdpConfig.CibirIdOffsetSrc = MsQuicLib.CidServerIdLength + 2;
+    UdpConfig.CibirIdOffsetDst = MsQuicLib.CidServerIdLength + 2;
+    if (UdpConfig.CibirIdLength) {
+        CXPLAT_DBG_ASSERT(UdpConfig.CibirIdLength <= sizeof(UdpConfig.CibirId));
+        CxPlatCopyMemory(
+            UdpConfig.CibirId,
+            &Listener->CibirId[2],
+            UdpConfig.CibirIdLength);
+    }
+#endif
 
     CXPLAT_TEL_ASSERT(Listener->Binding == NULL);
     Status =

src/inc/quic_datapath.h

@@ -466,6 +466,12 @@ typedef struct CXPLAT_UDP_CONFIG {
 #ifdef QUIC_OWNING_PROCESS
     QUIC_PROCESS OwningProcess;         // Kernel client-only
 #endif
+#ifdef QUIC_USE_RAW_DATAPATH
+    uint8_t CibirIdLength;              // CIBIR ID length. Value of 0 indicates CIBIR isn't used
+    uint8_t CibirIdOffsetSrc;           // CIBIR ID offset in source CID
+    uint8_t CibirIdOffsetDst;           // CIBIR ID offset in destination CID
+    uint8_t CibirId[6];                 // CIBIR ID data
+#endif
 } CXPLAT_UDP_CONFIG;
 
 //

src/platform/datapath_raw.c

@@ -286,6 +286,12 @@ CxPlatSocketCreateUdp(
     CxPlatRundownInitialize(&(*NewSocket)->Rundown);
     (*NewSocket)->Datapath = Datapath;
     (*NewSocket)->CallbackContext = Config->CallbackContext;
+    (*NewSocket)->CibirIdLength = Config->CibirIdLength;
+    (*NewSocket)->CibirIdOffsetSrc = Config->CibirIdOffsetSrc;
+    (*NewSocket)->CibirIdOffsetDst = Config->CibirIdOffsetDst;
+    if (Config->CibirIdLength) {
+        memcpy((*NewSocket)->CibirId, Config->CibirId, Config->CibirIdLength);
+    }
 
     if (Config->RemoteAddress) {
         CXPLAT_FRE_ASSERT(!QuicAddrIsWildCard(Config->RemoteAddress));  // No wildcard remote addresses allowed.

src/platform/datapath_raw.h

@@ -256,8 +256,13 @@ typedef struct CXPLAT_SOCKET {
     void* CallbackContext;
     QUIC_ADDR LocalAddress;
     QUIC_ADDR RemoteAddress;
-    BOOLEAN Wildcard;   // Using a wildcard local address. Optimization to avoid always reading LocalAddress.
-    BOOLEAN Connected;  // Bound to a remote address
+    BOOLEAN Wildcard;           // Using a wildcard local address. Optimization
+                                // to avoid always reading LocalAddress.
+    BOOLEAN Connected;          // Bound to a remote address
+    uint8_t CibirIdLength;      // CIBIR ID length. Value of 0 indicates CIBIR isn't used
+    uint8_t CibirIdOffsetSrc;   // CIBIR ID offset in source CID
+    uint8_t CibirIdOffsetDst;   // CIBIR ID offset in destination CID
+    uint8_t CibirId[6];         // CIBIR ID data
 
 } CXPLAT_SOCKET;
 

src/platform/datapath_raw_socket.c

@@ -144,6 +144,27 @@ CxPlatTryAddSocket(
         goto Error;
     }
 
+    if (Socket->CibirIdLength) {
+        Option = TRUE;
+        Result =
+            setsockopt(
+                Socket->AuxSocket,
+                SOL_SOCKET,
+                SO_REUSEADDR,
+                (char*)&Option,
+                sizeof(Option));
+        if (Result == SOCKET_ERROR) {
+            int Error = SocketError();
+            QuicTraceEvent(
+                DatapathErrorStatus,
+                "[data][%p] ERROR, %u, %s.",
+                Socket,
+                Error,
+                "Set SO_REUSEADDR");
+            goto Error;
+        }
+    }
+
     CxPlatConvertToMappedV6(&Socket->LocalAddress, &MappedAddress);
 #if QUIC_ADDRESS_FAMILY_INET6 != AF_INET6
     if (MappedAddress.Ipv6.sin6_family == QUIC_ADDRESS_FAMILY_INET6) {

src/platform/datapath_raw_xdp.c

@@ -755,7 +755,7 @@ CxPlatDpRawInterfaceUpdateRules(
         .SubLayer = XDP_HOOK_INSPECT,
     };
 
-    const UINT32 Flags = 0; // TODO: support native/generic forced flags.
+    const UINT32 Flags = XDP_CREATE_PROGRAM_FLAG_SHARE; // TODO: support native/generic forced flags.
 
     for (uint32_t i = 0; i < Interface->QueueCount; i++) {
 
@@ -798,9 +798,10 @@ CxPlatDpRawInterfaceUpdateRules(
 
 _IRQL_requires_max_(PASSIVE_LEVEL)
 void
-CxPlatDpRawInterfaceAddRule(
+CxPlatDpRawInterfaceAddRules(
     _In_ XDP_INTERFACE* Interface,
-    _In_ const XDP_RULE* NewRule
+    _In_reads_(Count) const XDP_RULE* Rules,
+    _In_ uint8_t Count
     )
 {
 #pragma warning(push)
@@ -809,7 +810,7 @@ CxPlatDpRawInterfaceAddRule(
     CxPlatLockAcquire(&Interface->RuleLock);
     // TODO - Don't always allocate a new array?
 
-    if (Interface->RuleCount + 1 == 0) {
+    if ((uint32_t)Interface->RuleCount + (uint32_t)Count > UINT8_MAX) {
         QuicTraceEvent(
             LibraryError,
             "[ lib] ERROR, %s.",
@@ -819,7 +820,7 @@ CxPlatDpRawInterfaceAddRule(
     }
 
     const size_t OldSize = sizeof(XDP_RULE) * (size_t)Interface->RuleCount;
-    const size_t NewSize = sizeof(XDP_RULE) * ((size_t)Interface->RuleCount + 1);
+    const size_t NewSize = sizeof(XDP_RULE) * ((size_t)Interface->RuleCount + Count);
 
     XDP_RULE* NewRules = CxPlatAlloc(NewSize, RULE_TAG);
     if (NewRules == NULL) {
@@ -835,8 +836,9 @@ CxPlatDpRawInterfaceAddRule(
     if (Interface->RuleCount > 0) {
         memcpy(NewRules, Interface->Rules, OldSize);
     }
-    NewRules[Interface->RuleCount] = *NewRule;
-    Interface->RuleCount++;
+    for (uint8_t i = 0; i < Count; i++) {
+        NewRules[Interface->RuleCount++] = Rules[i];
+    }
 
     if (Interface->Rules != NULL) {
         CxPlatFree(Interface->Rules, RULE_TAG);
@@ -852,46 +854,62 @@ CxPlatDpRawInterfaceAddRule(
 
 _IRQL_requires_max_(PASSIVE_LEVEL)
 void
-CxPlatDpRawInterfaceRemoveRule(
+CxPlatDpRawInterfaceRemoveRules(
     _In_ XDP_INTERFACE* Interface,
-    _In_ const XDP_RULE* Rule
+    _In_reads_(Count) const XDP_RULE* Rules,
+    _In_ uint8_t Count
     )
 {
     CxPlatLockAcquire(&Interface->RuleLock);
 
-    for (uint8_t i = 0; i < Interface->RuleCount; i++) {
-        if (Interface->Rules[i].Match != Rule->Match) {
-            continue;
-        }
+    BOOLEAN UpdateRules = FALSE;
 
-        if (Rule->Match == XDP_MATCH_UDP_DST) {
-            if (Rule->Pattern.Port != Interface->Rules[i].Pattern.Port) {
+    for (uint8_t j = 0; j < Count; j++) {
+        for (uint8_t i = 0; i < Interface->RuleCount; i++) {
+            if (Interface->Rules[i].Match != Rules[j].Match) {
                 continue;
             }
-        } else if (Rule->Match == XDP_MATCH_IPV4_UDP_TUPLE) {
-            if (Rule->Pattern.Tuple.DestinationPort != Interface->Rules[i].Pattern.Tuple.DestinationPort ||
-                Rule->Pattern.Tuple.SourcePort != Interface->Rules[i].Pattern.Tuple.SourcePort ||
-                memcmp(&Rule->Pattern.Tuple.DestinationAddress.Ipv4, &Interface->Rules[i].Pattern.Tuple.DestinationAddress.Ipv4, sizeof(IN_ADDR)) != 0 ||
-                memcmp(&Rule->Pattern.Tuple.SourceAddress.Ipv4, &Interface->Rules[i].Pattern.Tuple.SourceAddress.Ipv4, sizeof(IN_ADDR)) != 0) {
-                continue;
+
+            if (Rules[j].Match == XDP_MATCH_UDP_DST) {
+                if (Rules[j].Pattern.Port != Interface->Rules[i].Pattern.Port) {
+                    continue;
+                }
+            } else if (Rules[j].Match == XDP_MATCH_QUIC_FLOW_SRC_CID || Rules[j].Match == XDP_MATCH_QUIC_FLOW_DST_CID) {
+                if (Rules[j].Pattern.QuicFlow.UdpPort != Interface->Rules[i].Pattern.QuicFlow.UdpPort ||
+                    Rules[j].Pattern.QuicFlow.CidLength != Interface->Rules[i].Pattern.QuicFlow.CidLength ||
+                    Rules[j].Pattern.QuicFlow.CidOffset != Interface->Rules[i].Pattern.QuicFlow.CidOffset ||
+                    memcmp(Rules[j].Pattern.QuicFlow.CidData, Interface->Rules[i].Pattern.QuicFlow.CidData, Rules[j].Pattern.QuicFlow.CidLength) != 0) {
+                    continue;
+                }
+            } else if (Rules[j].Match == XDP_MATCH_IPV4_UDP_TUPLE) {
+                if (Rules[j].Pattern.Tuple.DestinationPort != Interface->Rules[i].Pattern.Tuple.DestinationPort ||
+                    Rules[j].Pattern.Tuple.SourcePort != Interface->Rules[i].Pattern.Tuple.SourcePort ||
+                    memcmp(&Rules[j].Pattern.Tuple.DestinationAddress.Ipv4, &Interface->Rules[i].Pattern.Tuple.DestinationAddress.Ipv4, sizeof(IN_ADDR)) != 0 ||
+                    memcmp(&Rules[j].Pattern.Tuple.SourceAddress.Ipv4, &Interface->Rules[i].Pattern.Tuple.SourceAddress.Ipv4, sizeof(IN_ADDR)) != 0) {
+                    continue;
+                }
+            } else if (Rules[j].Match == XDP_MATCH_IPV6_UDP_TUPLE) {
+                if (Rules[j].Pattern.Tuple.DestinationPort != Interface->Rules[i].Pattern.Tuple.DestinationPort ||
+                    Rules[j].Pattern.Tuple.SourcePort != Interface->Rules[i].Pattern.Tuple.SourcePort ||
+                    memcmp(&Rules[j].Pattern.Tuple.DestinationAddress.Ipv6, &Interface->Rules[i].Pattern.Tuple.DestinationAddress.Ipv6, sizeof(IN6_ADDR)) != 0 ||
+                    memcmp(&Rules[j].Pattern.Tuple.SourceAddress.Ipv6, &Interface->Rules[i].Pattern.Tuple.SourceAddress.Ipv6, sizeof(IN6_ADDR)) != 0) {
+                    continue;
+                }
+            } else {
+                CXPLAT_FRE_ASSERT(FALSE); // Should not be possible!
             }
-        } else if (Rule->Match == XDP_MATCH_IPV6_UDP_TUPLE) {
-            if (Rule->Pattern.Tuple.DestinationPort != Interface->Rules[i].Pattern.Tuple.DestinationPort ||
-                Rule->Pattern.Tuple.SourcePort != Interface->Rules[i].Pattern.Tuple.SourcePort ||
-                memcmp(&Rule->Pattern.Tuple.DestinationAddress.Ipv6, &Interface->Rules[i].Pattern.Tuple.DestinationAddress.Ipv6, sizeof(IN6_ADDR)) != 0 ||
-                memcmp(&Rule->Pattern.Tuple.SourceAddress.Ipv6, &Interface->Rules[i].Pattern.Tuple.SourceAddress.Ipv6, sizeof(IN6_ADDR)) != 0) {
-                continue;
+
+            if (i < Interface->RuleCount - 1) {
+                memmove(&Interface->Rules[i], &Interface->Rules[i + 1], sizeof(XDP_RULE) * (Interface->RuleCount - i - 1));
             }
-        } else {
-            CXPLAT_FRE_ASSERT(FALSE); // Should not be possible!
+            Interface->RuleCount--;
+            UpdateRules = TRUE;
+            break;
         }
+    }
 
-        if (i < Interface->RuleCount - 1) {
-            memmove(&Interface->Rules[i], &Interface->Rules[i + 1], sizeof(XDP_RULE) * (Interface->RuleCount - i - 1));
-        }
-        Interface->RuleCount--;
+    if (UpdateRules) {
         CxPlatDpRawInterfaceUpdateRules(Interface);
-        break;
     }
 
     CxPlatLockRelease(&Interface->RuleLock);
@@ -1052,21 +1070,57 @@ CxPlatDpRawPlumbRulesOnSocket(
     XDP_DATAPATH* Xdp = (XDP_DATAPATH*)Socket->Datapath;
 
     if (Socket->Wildcard) {
-        const XDP_RULE Rule = {
-            .Match = XDP_MATCH_UDP_DST,
-            .Pattern.Port = Socket->LocalAddress.Ipv4.sin_port,
-            .Action = XDP_PROGRAM_ACTION_REDIRECT,
-            .Redirect.TargetType = XDP_REDIRECT_TARGET_TYPE_XSK,
-            .Redirect.Target = NULL,
-        };
 
-        CXPLAT_LIST_ENTRY* Entry;
-        for (Entry = Xdp->Interfaces.Flink; Entry != &Xdp->Interfaces; Entry = Entry->Flink) {
-            XDP_INTERFACE* Interface = CONTAINING_RECORD(Entry, XDP_INTERFACE, Link);
-            if (IsCreated) {
-                CxPlatDpRawInterfaceAddRule(Interface, &Rule);
-            } else {
-                CxPlatDpRawInterfaceRemoveRule(Interface, &Rule);
+        if (Socket->CibirIdLength) {
+            XDP_RULE Rules[] = {
+                {
+                .Match = XDP_MATCH_QUIC_FLOW_SRC_CID,
+                .Pattern.QuicFlow.UdpPort = Socket->LocalAddress.Ipv4.sin_port,
+                .Pattern.QuicFlow.CidLength = Socket->CibirIdLength,
+                .Pattern.QuicFlow.CidOffset = Socket->CibirIdOffsetSrc,
+                .Action = XDP_PROGRAM_ACTION_REDIRECT,
+                .Redirect.TargetType = XDP_REDIRECT_TARGET_TYPE_XSK,
+                .Redirect.Target = NULL,
+                },
+                {
+                .Match = XDP_MATCH_QUIC_FLOW_DST_CID,
+                .Pattern.QuicFlow.UdpPort = Socket->LocalAddress.Ipv4.sin_port,
+                .Pattern.QuicFlow.CidLength = Socket->CibirIdLength,
+                .Pattern.QuicFlow.CidOffset = Socket->CibirIdOffsetDst,
+                .Action = XDP_PROGRAM_ACTION_REDIRECT,
+                .Redirect.TargetType = XDP_REDIRECT_TARGET_TYPE_XSK,
+                .Redirect.Target = NULL,
+                }
+            };
+            memcpy(Rules[0].Pattern.QuicFlow.CidData, Socket->CibirId, Socket->CibirIdLength);
+            memcpy(Rules[1].Pattern.QuicFlow.CidData, Socket->CibirId, Socket->CibirIdLength);
+
+            CXPLAT_LIST_ENTRY* Entry;
+            for (Entry = Xdp->Interfaces.Flink; Entry != &Xdp->Interfaces; Entry = Entry->Flink) {
+                XDP_INTERFACE* Interface = CONTAINING_RECORD(Entry, XDP_INTERFACE, Link);
+                if (IsCreated) {
+                    CxPlatDpRawInterfaceAddRules(Interface, Rules, 2);
+                } else {
+                    CxPlatDpRawInterfaceRemoveRules(Interface, Rules, 2);
+                }
+            }
+        } else {
+            const XDP_RULE Rule = {
+                .Match = XDP_MATCH_UDP_DST,
+                .Pattern.Port = Socket->LocalAddress.Ipv4.sin_port,
+                .Action = XDP_PROGRAM_ACTION_REDIRECT,
+                .Redirect.TargetType = XDP_REDIRECT_TARGET_TYPE_XSK,
+                .Redirect.Target = NULL,
+            };
+
+            CXPLAT_LIST_ENTRY* Entry;
+            for (Entry = Xdp->Interfaces.Flink; Entry != &Xdp->Interfaces; Entry = Entry->Flink) {
+                XDP_INTERFACE* Interface = CONTAINING_RECORD(Entry, XDP_INTERFACE, Link);
+                if (IsCreated) {
+                    CxPlatDpRawInterfaceAddRules(Interface, &Rule, 1);
+                } else {
+                    CxPlatDpRawInterfaceRemoveRules(Interface, &Rule, 1);
+                }
             }
         }
 
@@ -1098,9 +1152,9 @@ CxPlatDpRawPlumbRulesOnSocket(
         for (Entry = Xdp->Interfaces.Flink; Entry != &Xdp->Interfaces; Entry = Entry->Flink) {
             XDP_INTERFACE* Interface = CONTAINING_RECORD(Entry, XDP_INTERFACE, Link);
             if (IsCreated) {
-                CxPlatDpRawInterfaceAddRule(Interface, &Rule);
+                CxPlatDpRawInterfaceAddRules(Interface, &Rule, 1);
             } else {
-                CxPlatDpRawInterfaceRemoveRule(Interface, &Rule);
+                CxPlatDpRawInterfaceRemoveRules(Interface, &Rule, 1);
             }
         }
     }