Initial Changes for RDMA as a platform for MsQuic

Rdma/Mana/mana_ndspi.h

@@ -0,0 +1,207 @@
+#pragma once
+
+#ifndef _MANA_NDSPI_H_
+#define _MANA_NDSPI_H_
+
+typedef enum _ND2_MANA_REQUEST_TYPE {
+    Nd2ManaRequestTypeSend,
+    Nd2ManaRequestTypeBind,
+    Nd2ManaRequestTypeInvalidate,
+    Nd2ManaRequestTypeRead,
+    Nd2ManaRequestTypeWrite,
+    Nd2ManaRequestTypeRecv = 1 << 7,
+    Nd2ManaRequestTypeRecvWithInvalidate,
+    Nd2ManaRequestTypeRecvWithImmediate,
+    Nd2ManaRequestTypeRecvRdmaWithImmediate,
+} ND2_MANA_REQUEST_TYPE;
+
+typedef struct _ND2_MANA_RESULT {
+    HRESULT                 Status;
+    ULONG                   BytesTransferred;
+    VOID*                   QueuePairContext;
+    VOID*                   RequestContext;
+    ND2_MANA_REQUEST_TYPE   RequestType;
+    union {
+        UINT32 ImmediateData;
+        UINT32 RKey;
+    };
+} ND2_MANA_RESULT;
+
+//
+// Completion Queue
+//
+#undef INTERFACE
+#define INTERFACE IND2ManaCompletionQueue
+
+// {28925856-4FCC-4479-915C-034426A50B9E}
+DEFINE_GUID(IID_IND2ManaCompletionQueue,
+    0x28925856, 0x4fcc, 0x4479, 0x91, 0x5c, 0x3, 0x44, 0x26, 0xa5, 0xb, 0x9e);
+
+DECLARE_INTERFACE_(IND2ManaCompletionQueue, IND2CompletionQueue)
+{
+    // *** IUnknown methods ***
+    IFACEMETHOD(QueryInterface)(
+        THIS_
+        REFIID riid,
+        __deref_out LPVOID* ppvObj
+        ) PURE;
+
+    IFACEMETHOD_(ULONG, AddRef)(
+        THIS
+        ) PURE;
+
+    IFACEMETHOD_(ULONG, Release)(
+        THIS
+        ) PURE;
+
+    // *** IND2Overlapped methods ***
+    IFACEMETHOD(CancelOverlappedRequests)(
+        THIS
+        ) PURE;
+
+    IFACEMETHOD(GetOverlappedResult)(
+        THIS_
+        __in OVERLAPPED* pOverlapped,
+        BOOL wait
+        ) PURE;
+
+    // *** IND2CompletionQueue methods ***
+    IFACEMETHOD(GetNotifyAffinity)(
+        THIS_
+        __out USHORT* pGroup,
+        __out KAFFINITY* pAffinity
+        ) PURE;
+
+    IFACEMETHOD(Resize)(
+        THIS_
+        ULONG queueDepth
+        ) PURE;
+
+    IFACEMETHOD(Notify)(
+        THIS_
+        ULONG type,
+        __inout OVERLAPPED* pOverlapped
+        ) PURE;
+
+    IFACEMETHOD_(ULONG, GetResults)(
+        THIS_
+        __out_ecount_part(nResults, return) ND2_RESULT results[],
+        ULONG nResults
+        ) PURE;
+
+    // *** IND2ManaCompletionQueue methods ***
+    STDMETHOD_(ULONG, GetManaResults)(
+        THIS_
+        __out_ecount_part(nResults, return) ND2_MANA_RESULT results[],
+        ULONG nResults
+        ) PURE;
+};
+
+//
+// QueuePair
+//
+#undef INTERFACE
+#define INTERFACE IND2ManaQueuePair
+
+// {8B1811D5-52DD-4084-8119-15C11AB32FB9}
+DEFINE_GUID(IID_IND2ManaQueuePair,
+    0x8b1811d5, 0x52dd, 0x4084, 0x81, 0x19, 0x15, 0xc1, 0x1a, 0xb3, 0x2f, 0xb9);
+
+DECLARE_INTERFACE_(IND2ManaQueuePair, IND2QueuePair)
+{
+    // *** IUnknown methods ***
+    IFACEMETHOD(QueryInterface)(
+        THIS_
+        REFIID riid,
+        __deref_out LPVOID* ppvObj
+        ) PURE;
+
+    IFACEMETHOD_(ULONG, AddRef)(
+        THIS
+        ) PURE;
+
+    IFACEMETHOD_(ULONG, Release)(
+        THIS
+        ) PURE;
+
+    // *** IND2QueuePair methods ***
+    IFACEMETHOD(Flush)(
+        THIS
+        ) PURE;
+
+    IFACEMETHOD(Send)(
+        THIS_
+        __in_opt VOID* requestContext,
+        __in_ecount_opt(nSge) const ND2_SGE sge[],
+        ULONG nSge,
+        ULONG flags
+        ) PURE;
+
+    IFACEMETHOD(Receive)(
+        THIS_
+        __in_opt VOID* requestContext,
+        __in_ecount_opt(nSge) const ND2_SGE sge[],
+        ULONG nSge
+        ) PURE;
+
+    // RemoteToken available thorugh IND2Mw::GetRemoteToken.
+    IFACEMETHOD(Bind)(
+        THIS_
+        __in_opt VOID* requestContext,
+        __in IUnknown* pMemoryRegion,
+        __inout IUnknown* pMemoryWindow,
+        __in_bcount(cbBuffer) const VOID* pBuffer,
+        SIZE_T cbBuffer,
+        ULONG flags
+        ) PURE;
+
+    IFACEMETHOD(Invalidate)(
+        THIS_
+        __in_opt VOID* requestContext,
+        __in IUnknown* pMemoryWindow,
+        ULONG flags
+        ) PURE;
+
+    IFACEMETHOD(Read)(
+        THIS_
+        __in_opt VOID* requestContext,
+        __in_ecount_opt(nSge) const ND2_SGE sge[],
+        ULONG nSge,
+        UINT64 remoteAddress,
+        UINT32 remoteToken,
+        ULONG flags
+        ) PURE;
+
+    IFACEMETHOD(Write)(
+        THIS_
+        __in_opt VOID* requestContext,
+        __in_ecount_opt(nSge) const ND2_SGE sge[],
+        ULONG nSge,
+        UINT64 remoteAddress,
+        UINT32 remoteToken,
+        ULONG flags
+        ) PURE;
+
+    // *** IND2ManaQueuePair methods ***
+    STDMETHOD(SendWithImmediate)(
+        THIS_
+        __in_opt VOID* requestContext,
+        __in_ecount_opt(nSge) const ND2_SGE sge[],
+        ULONG nSge,
+        ULONG flags,
+        UINT32 immediateData
+        ) PURE;
+
+    STDMETHOD(WriteWithImmediate)(
+        THIS_
+        __in_opt VOID* requestContext,
+        __in_ecount_opt(nSge) const ND2_SGE sge[],
+        ULONG nSge,
+        UINT64 remoteAddress,
+        UINT32 remoteToken,
+        ULONG flags,
+        UINT32 immediateData
+        ) PURE;
+};
+
+#endif /* _MANA_NDSPI_H_ */

scripts/prepare-machine.ps1

@@ -94,7 +94,10 @@ param (
     [switch]$DisableTest,
 
     [Parameter(Mandatory = $false)]
-    [switch]$InstallCoreNetCiDeps
+    [switch]$InstallCoreNetCiDeps,
+
+    [Parameter(Mandatory = $false)]
+    [switch]$InstallNetworkDirect
 )
 
 # Admin is required because a lot of things are installed to the local machine
@@ -144,6 +147,11 @@ if ($ForBuild) {
     $InstallNasm = $true
     $InstallJom = $true
     $InstallCoreNetCiDeps = $true; # For kernel signing certs
+
+    if (!$IsLinux && !$IsMacOS)
+    {
+        $InstallNetworkDirect = $true
+    }
 }
 
 if ($ForTest) {
@@ -164,6 +172,11 @@ if ($ForTest) {
         $InstallDuoNic = $true;
     }
 
+    if (!$IsLinux && !$IsMacOS)
+    {
+        $InstallNetworkDirect = $true
+    }
+
     #$InstallCodeCoverage = $true # Ideally we'd enable this by default, but it
                                   # hangs sometimes, so we only want to install
                                   # for jobs that absoultely need it.
@@ -472,6 +485,12 @@ function Install-Clog2Text {
     Install-DotnetTool -ToolName "Microsoft.Logging.CLOG2Text.Lttng" -Version "0.0.1" -NuGetPath $NuGetPath
 }
 
+function Install-NetworkDirect {
+    Write-Host "Install NetworkDirect"
+    Install-Package NetworkDirect  -ProviderName NuGet -Destination $RootDir\Rdma -ExcludeVersion -Force
+}
+
+
 # We remove OpenSSL path for kernel builds because it's not needed.
 if ($ForKernel) {
     git rm $RootDir/submodules/openssl
@@ -515,6 +534,7 @@ if ($InstallJOM) { Install-JOM }
 if ($InstallPerl) { Install-Perl }
 if ($InstallCodeCoverage) { Install-OpenCppCoverage }
 if ($InstallTestCertificates) { Install-TestCertificates }
+if ($InstallNetworkDirect) { Install-NetworkDirect }
 
 if ($IsLinux) {
     if ($InstallClog2Text) {

src/core/library.c

@@ -683,6 +683,7 @@ QuicLibraryLazyInitialize(
             sizeof(QUIC_RX_PACKET),
             &DatapathCallbacks,
             NULL,                   // TcpCallbacks
+            NULL,                   // RdmaCallbacks
             &MsQuicLib.WorkerPool,
             MsQuicLib.ExecutionConfig,
             &MsQuicLib.Datapath);

src/generated/linux/datapath_rdma_ringbuffer.c.clog.h

@@ -0,0 +1,49 @@
+#ifndef CLOG_DO_NOT_INCLUDE_HEADER
+#include <clog.h>
+#endif
+#undef TRACEPOINT_PROVIDER
+#define TRACEPOINT_PROVIDER CLOG_DATAPATH_RDMA_RINGBUFFER_C
+#undef TRACEPOINT_PROBE_DYNAMIC_LINKAGE
+#define  TRACEPOINT_PROBE_DYNAMIC_LINKAGE
+#undef TRACEPOINT_INCLUDE
+#define TRACEPOINT_INCLUDE "datapath_rdma_ringbuffer.c.clog.h.lttng.h"
+#if !defined(DEF_CLOG_DATAPATH_RDMA_RINGBUFFER_C) || defined(TRACEPOINT_HEADER_MULTI_READ)
+#define DEF_CLOG_DATAPATH_RDMA_RINGBUFFER_C
+#include <lttng/tracepoint.h>
+#define __int64 __int64_t
+#include "datapath_rdma_ringbuffer.c.clog.h.lttng.h"
+#endif
+#include <lttng/tracepoint-event.h>
+#ifndef _clog_MACRO_QuicTraceEvent
+#define _clog_MACRO_QuicTraceEvent  1
+#define QuicTraceEvent(a, ...) _clog_CAT(_clog_ARGN_SELECTOR(__VA_ARGS__), _clog_CAT(_,a(#a, __VA_ARGS__)))
+#endif
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*----------------------------------------------------------
+// Decoder Ring for AllocFailure
+// Allocation of '%s' failed. (%llu bytes)
+// QuicTraceEvent(
+            AllocFailure,
+            "Allocation of '%s' failed. (%llu bytes)",
+            "SendCompletionTable",
+            0);
+// arg2 = arg2 = "SendCompletionTable" = arg2
+// arg3 = arg3 = 0 = arg3
+----------------------------------------------------------*/
+#ifndef _clog_4_ARGS_TRACE_AllocFailure
+#define _clog_4_ARGS_TRACE_AllocFailure(uniqueId, encoded_arg_string, arg2, arg3)\
+tracepoint(CLOG_DATAPATH_RDMA_RINGBUFFER_C, AllocFailure , arg2, arg3);\
+
+#endif
+
+
+
+
+#ifdef __cplusplus
+}
+#endif
+#ifdef CLOG_INLINE_IMPLEMENTATION
+#include "quic.clog_datapath_rdma_ringbuffer.c.clog.h.c"
+#endif

src/generated/linux/datapath_rdma_ringbuffer.c.clog.h.lttng.h

@@ -0,0 +1,23 @@
+
+
+
+/*----------------------------------------------------------
+// Decoder Ring for AllocFailure
+// Allocation of '%s' failed. (%llu bytes)
+// QuicTraceEvent(
+            AllocFailure,
+            "Allocation of '%s' failed. (%llu bytes)",
+            "SendCompletionTable",
+            0);
+// arg2 = arg2 = "SendCompletionTable" = arg2
+// arg3 = arg3 = 0 = arg3
+----------------------------------------------------------*/
+TRACEPOINT_EVENT(CLOG_DATAPATH_RDMA_RINGBUFFER_C, AllocFailure,
+    TP_ARGS(
+        const char *, arg2,
+        unsigned long long, arg3), 
+    TP_FIELDS(
+        ctf_string(arg2, arg2)
+        ctf_integer(uint64_t, arg3, arg3)
+    )
+)