From 0bea3830d7924adc3f3dba0595d7baa164223960 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Mon, 8 Sep 2025 15:13:47 -0500 Subject: [PATCH 01/25] using locally attached references for JasperFx, Weasel, and Marten --- .../Wolverine.Marten/Wolverine.Marten.csproj | 4 +- .../Wolverine.Postgresql.csproj | 5 +- .../Wolverine.RDBMS/Wolverine.RDBMS.csproj | 5 +- .../Wolverine.SqlServer.csproj | 2 +- src/Wolverine/Wolverine.csproj | 7 +- wolverine.sln | 107 ++++++++++++++++++ 6 files changed, 116 insertions(+), 14 deletions(-) diff --git a/src/Persistence/Wolverine.Marten/Wolverine.Marten.csproj b/src/Persistence/Wolverine.Marten/Wolverine.Marten.csproj index f2c23f81d..4a2dc7865 100644 --- a/src/Persistence/Wolverine.Marten/Wolverine.Marten.csproj +++ b/src/Persistence/Wolverine.Marten/Wolverine.Marten.csproj @@ -9,10 +9,8 @@ false + - - - diff --git a/src/Persistence/Wolverine.Postgresql/Wolverine.Postgresql.csproj b/src/Persistence/Wolverine.Postgresql/Wolverine.Postgresql.csproj index 297045e70..3f2db6415 100644 --- a/src/Persistence/Wolverine.Postgresql/Wolverine.Postgresql.csproj +++ b/src/Persistence/Wolverine.Postgresql/Wolverine.Postgresql.csproj @@ -12,10 +12,7 @@ - - - - + diff --git a/src/Persistence/Wolverine.RDBMS/Wolverine.RDBMS.csproj b/src/Persistence/Wolverine.RDBMS/Wolverine.RDBMS.csproj index 573403b78..51225786b 100644 --- a/src/Persistence/Wolverine.RDBMS/Wolverine.RDBMS.csproj +++ b/src/Persistence/Wolverine.RDBMS/Wolverine.RDBMS.csproj @@ -11,10 +11,7 @@ - - - - + diff --git a/src/Persistence/Wolverine.SqlServer/Wolverine.SqlServer.csproj b/src/Persistence/Wolverine.SqlServer/Wolverine.SqlServer.csproj index 554a7aed7..692ccf8fc 100644 --- a/src/Persistence/Wolverine.SqlServer/Wolverine.SqlServer.csproj +++ b/src/Persistence/Wolverine.SqlServer/Wolverine.SqlServer.csproj @@ -13,7 +13,7 @@ - + diff --git a/src/Wolverine/Wolverine.csproj b/src/Wolverine/Wolverine.csproj index 51e3ebfd5..49aec928e 100644 --- a/src/Wolverine/Wolverine.csproj +++ b/src/Wolverine/Wolverine.csproj @@ -4,8 +4,6 @@ WolverineFx - - @@ -40,5 +38,10 @@ + + + + + diff --git a/wolverine.sln b/wolverine.sln index c10cbb1dd..3c7f3e6bc 100644 --- a/wolverine.sln +++ b/wolverine.sln @@ -289,6 +289,22 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wolverine.SignalR", "src\Tr EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wolverine.SignalR.Tests", "src\Transports\SignalR\Wolverine.SignalR.Tests\Wolverine.SignalR.Tests.csproj", "{3F62DB30-9A29-487A-9EE2-22A097E3EE3F}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Attached", "Attached", "{3B05C001-84E3-1311-E8D3-16CD1127E3D5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JasperFx", "..\jasperfx\src\JasperFx\JasperFx.csproj", "{AFD6655F-131F-4DDD-B148-3D4276F028D4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JasperFx.Events", "..\jasperfx\src\JasperFx.Events\JasperFx.Events.csproj", "{891F4009-E643-4083-B33E-8AB582446A42}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EventTests", "..\jasperfx\src\EventTests\EventTests.csproj", "{ED2CCB28-1469-4216-8D16-FAB64A896A28}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JasperFx.RuntimeCompiler", "..\jasperfx\src\JasperFx.RuntimeCompiler\JasperFx.RuntimeCompiler.csproj", "{EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Weasel.Core", "..\weasel\src\Weasel.Core\Weasel.Core.csproj", "{357619FF-D2E6-4877-83FF-924C88523A53}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Weasel.Postgresql", "..\weasel\src\Weasel.Postgresql\Weasel.Postgresql.csproj", "{0445097D-4DC5-47F3-8D1F-DF80CD65A62B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Weasel.SqlServer", "..\weasel\src\Weasel.SqlServer\Weasel.SqlServer.csproj", "{CE6643E5-E869-41DF-8058-17FA45116652}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1619,6 +1635,90 @@ Global {3F62DB30-9A29-487A-9EE2-22A097E3EE3F}.Release|x64.Build.0 = Release|Any CPU {3F62DB30-9A29-487A-9EE2-22A097E3EE3F}.Release|x86.ActiveCfg = Release|Any CPU {3F62DB30-9A29-487A-9EE2-22A097E3EE3F}.Release|x86.Build.0 = Release|Any CPU + {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Debug|x64.ActiveCfg = Debug|Any CPU + {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Debug|x64.Build.0 = Debug|Any CPU + {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Debug|x86.ActiveCfg = Debug|Any CPU + {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Debug|x86.Build.0 = Debug|Any CPU + {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Release|Any CPU.Build.0 = Release|Any CPU + {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Release|x64.ActiveCfg = Release|Any CPU + {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Release|x64.Build.0 = Release|Any CPU + {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Release|x86.ActiveCfg = Release|Any CPU + {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Release|x86.Build.0 = Release|Any CPU + {891F4009-E643-4083-B33E-8AB582446A42}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {891F4009-E643-4083-B33E-8AB582446A42}.Debug|Any CPU.Build.0 = Debug|Any CPU + {891F4009-E643-4083-B33E-8AB582446A42}.Debug|x64.ActiveCfg = Debug|Any CPU + {891F4009-E643-4083-B33E-8AB582446A42}.Debug|x64.Build.0 = Debug|Any CPU + {891F4009-E643-4083-B33E-8AB582446A42}.Debug|x86.ActiveCfg = Debug|Any CPU + {891F4009-E643-4083-B33E-8AB582446A42}.Debug|x86.Build.0 = Debug|Any CPU + {891F4009-E643-4083-B33E-8AB582446A42}.Release|Any CPU.ActiveCfg = Release|Any CPU + {891F4009-E643-4083-B33E-8AB582446A42}.Release|Any CPU.Build.0 = Release|Any CPU + {891F4009-E643-4083-B33E-8AB582446A42}.Release|x64.ActiveCfg = Release|Any CPU + {891F4009-E643-4083-B33E-8AB582446A42}.Release|x64.Build.0 = Release|Any CPU + {891F4009-E643-4083-B33E-8AB582446A42}.Release|x86.ActiveCfg = Release|Any CPU + {891F4009-E643-4083-B33E-8AB582446A42}.Release|x86.Build.0 = Release|Any CPU + {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Debug|Any CPU.Build.0 = Debug|Any CPU + {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Debug|x64.ActiveCfg = Debug|Any CPU + {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Debug|x64.Build.0 = Debug|Any CPU + {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Debug|x86.ActiveCfg = Debug|Any CPU + {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Debug|x86.Build.0 = Debug|Any CPU + {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Release|Any CPU.ActiveCfg = Release|Any CPU + {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Release|Any CPU.Build.0 = Release|Any CPU + {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Release|x64.ActiveCfg = Release|Any CPU + {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Release|x64.Build.0 = Release|Any CPU + {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Release|x86.ActiveCfg = Release|Any CPU + {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Release|x86.Build.0 = Release|Any CPU + {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Debug|x64.ActiveCfg = Debug|Any CPU + {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Debug|x64.Build.0 = Debug|Any CPU + {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Debug|x86.ActiveCfg = Debug|Any CPU + {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Debug|x86.Build.0 = Debug|Any CPU + {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Release|Any CPU.Build.0 = Release|Any CPU + {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Release|x64.ActiveCfg = Release|Any CPU + {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Release|x64.Build.0 = Release|Any CPU + {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Release|x86.ActiveCfg = Release|Any CPU + {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Release|x86.Build.0 = Release|Any CPU + {357619FF-D2E6-4877-83FF-924C88523A53}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {357619FF-D2E6-4877-83FF-924C88523A53}.Debug|Any CPU.Build.0 = Debug|Any CPU + {357619FF-D2E6-4877-83FF-924C88523A53}.Debug|x64.ActiveCfg = Debug|Any CPU + {357619FF-D2E6-4877-83FF-924C88523A53}.Debug|x64.Build.0 = Debug|Any CPU + {357619FF-D2E6-4877-83FF-924C88523A53}.Debug|x86.ActiveCfg = Debug|Any CPU + {357619FF-D2E6-4877-83FF-924C88523A53}.Debug|x86.Build.0 = Debug|Any CPU + {357619FF-D2E6-4877-83FF-924C88523A53}.Release|Any CPU.ActiveCfg = Release|Any CPU + {357619FF-D2E6-4877-83FF-924C88523A53}.Release|Any CPU.Build.0 = Release|Any CPU + {357619FF-D2E6-4877-83FF-924C88523A53}.Release|x64.ActiveCfg = Release|Any CPU + {357619FF-D2E6-4877-83FF-924C88523A53}.Release|x64.Build.0 = Release|Any CPU + {357619FF-D2E6-4877-83FF-924C88523A53}.Release|x86.ActiveCfg = Release|Any CPU + {357619FF-D2E6-4877-83FF-924C88523A53}.Release|x86.Build.0 = Release|Any CPU + {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Debug|x64.ActiveCfg = Debug|Any CPU + {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Debug|x64.Build.0 = Debug|Any CPU + {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Debug|x86.ActiveCfg = Debug|Any CPU + {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Debug|x86.Build.0 = Debug|Any CPU + {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Release|Any CPU.Build.0 = Release|Any CPU + {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Release|x64.ActiveCfg = Release|Any CPU + {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Release|x64.Build.0 = Release|Any CPU + {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Release|x86.ActiveCfg = Release|Any CPU + {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Release|x86.Build.0 = Release|Any CPU + {CE6643E5-E869-41DF-8058-17FA45116652}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CE6643E5-E869-41DF-8058-17FA45116652}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CE6643E5-E869-41DF-8058-17FA45116652}.Debug|x64.ActiveCfg = Debug|Any CPU + {CE6643E5-E869-41DF-8058-17FA45116652}.Debug|x64.Build.0 = Debug|Any CPU + {CE6643E5-E869-41DF-8058-17FA45116652}.Debug|x86.ActiveCfg = Debug|Any CPU + {CE6643E5-E869-41DF-8058-17FA45116652}.Debug|x86.Build.0 = Debug|Any CPU + {CE6643E5-E869-41DF-8058-17FA45116652}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CE6643E5-E869-41DF-8058-17FA45116652}.Release|Any CPU.Build.0 = Release|Any CPU + {CE6643E5-E869-41DF-8058-17FA45116652}.Release|x64.ActiveCfg = Release|Any CPU + {CE6643E5-E869-41DF-8058-17FA45116652}.Release|x64.Build.0 = Release|Any CPU + {CE6643E5-E869-41DF-8058-17FA45116652}.Release|x86.ActiveCfg = Release|Any CPU + {CE6643E5-E869-41DF-8058-17FA45116652}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1752,5 +1852,12 @@ Global {1EF7D49F-DDB8-469A-88A0-4A8D6237561C} = {84D32C8B-9CCE-4925-9AEC-8F445C7A2E3D} {36645C4B-BE1F-4184-A14C-D5BFA3F86A86} = {1EF7D49F-DDB8-469A-88A0-4A8D6237561C} {3F62DB30-9A29-487A-9EE2-22A097E3EE3F} = {1EF7D49F-DDB8-469A-88A0-4A8D6237561C} + {AFD6655F-131F-4DDD-B148-3D4276F028D4} = {3B05C001-84E3-1311-E8D3-16CD1127E3D5} + {891F4009-E643-4083-B33E-8AB582446A42} = {3B05C001-84E3-1311-E8D3-16CD1127E3D5} + {ED2CCB28-1469-4216-8D16-FAB64A896A28} = {3B05C001-84E3-1311-E8D3-16CD1127E3D5} + {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8} = {3B05C001-84E3-1311-E8D3-16CD1127E3D5} + {357619FF-D2E6-4877-83FF-924C88523A53} = {3B05C001-84E3-1311-E8D3-16CD1127E3D5} + {0445097D-4DC5-47F3-8D1F-DF80CD65A62B} = {3B05C001-84E3-1311-E8D3-16CD1127E3D5} + {CE6643E5-E869-41DF-8058-17FA45116652} = {3B05C001-84E3-1311-E8D3-16CD1127E3D5} EndGlobalSection EndGlobal From c7c34760525f452b1c2a8bc49f5db67a7711d3bf Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Tue, 9 Sep 2025 09:46:57 -0500 Subject: [PATCH 02/25] Tests on MessagingSubscriptionsAddedHandler --- .../Runtime/Routing/IMessageRoute.cs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/Wolverine/Runtime/Routing/IMessageRoute.cs b/src/Wolverine/Runtime/Routing/IMessageRoute.cs index 183da16b1..4a396bf0a 100644 --- a/src/Wolverine/Runtime/Routing/IMessageRoute.cs +++ b/src/Wolverine/Runtime/Routing/IMessageRoute.cs @@ -27,6 +27,42 @@ public class MessageSubscriptionDescriptor public string ContentType { get; set; } = "application/json"; public string Description { get; set; } = string.Empty; + public override string ToString() + { + return + $"{nameof(Endpoint)}: {Endpoint}, {nameof(ContentType)}: {ContentType}, {nameof(Description)}: {Description}"; + } + + protected bool Equals(MessageSubscriptionDescriptor other) + { + return Endpoint.Equals(other.Endpoint) && ContentType == other.ContentType && Description == other.Description; + } + + public override bool Equals(object? obj) + { + if (obj is null) + { + return false; + } + + if (ReferenceEquals(this, obj)) + { + return true; + } + + if (obj.GetType() != GetType()) + { + return false; + } + + return Equals((MessageSubscriptionDescriptor)obj); + } + + public override int GetHashCode() + { + return HashCode.Combine(Endpoint, ContentType, Description); + } + // TODO -- add something about envelope rules? } From ab89e7a0afb7b7c43b23e8ee609e93142afb92fe Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Tue, 9 Sep 2025 13:59:47 -0500 Subject: [PATCH 03/25] using new JasperFx helper for kebab casing --- src/Wolverine/Util/WolverineMessageNaming.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Wolverine/Util/WolverineMessageNaming.cs b/src/Wolverine/Util/WolverineMessageNaming.cs index 9534a14be..603c0a589 100644 --- a/src/Wolverine/Util/WolverineMessageNaming.cs +++ b/src/Wolverine/Util/WolverineMessageNaming.cs @@ -18,19 +18,13 @@ public bool TryDetermineName(Type messageType, out string messageTypeName) { if (messageType.CanBeCastTo()) { - messageTypeName = PascalToKebabCase(messageType.NameInCode()); + messageTypeName = messageType.NameInCode().PascalToKebabCase(); return true; } messageTypeName = default!; return false; } - - // TODO - Move this to JasperFx - internal static string PascalToKebabCase(string value) - { - return value.SplitPascalCase().Replace(' ', '_').ToLowerInvariant(); - } } internal class InteropAttributeForwardingNaming : IMessageTypeNaming From ee030a5509bae8483940f64f27f995d9a45a4337 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Fri, 12 Sep 2025 11:41:12 -0500 Subject: [PATCH 04/25] silly quick stop setting for critter watch testing --- .../Runtime/WolverineRuntime.HostService.cs | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/Wolverine/Runtime/WolverineRuntime.HostService.cs b/src/Wolverine/Runtime/WolverineRuntime.HostService.cs index 214bbb58f..4ccc9b64c 100644 --- a/src/Wolverine/Runtime/WolverineRuntime.HostService.cs +++ b/src/Wolverine/Runtime/WolverineRuntime.HostService.cs @@ -170,6 +170,8 @@ private void logCodeGenerationConfiguration() } } + public StopMode StopMode { get; set; } = StopMode.Normal; + public async Task StopAsync(CancellationToken cancellationToken) { if (_hasStopped) @@ -184,7 +186,7 @@ public async Task StopAsync(CancellationToken cancellationToken) // Latch health checks ASAP DisableHealthChecks(); - if (_persistence.IsValueCreated) + if (_persistence.IsValueCreated && StopMode == StopMode.Normal) { try { @@ -206,10 +208,13 @@ public async Task StopAsync(CancellationToken cancellationToken) } } - // This MUST be called before draining the endpoints - await teardownAgentsAsync(); - - await _endpoints.DrainAsync(); + if (StopMode == StopMode.Normal) + { + // This MUST be called before draining the endpoints + await teardownAgentsAsync(); + + await _endpoints.DrainAsync(); + } DurabilitySettings.Cancel(); @@ -321,4 +326,14 @@ internal Task StartLightweightAsync() return StartAsync(CancellationToken.None); } +} + +public enum StopMode +{ + Normal, + + /// + /// Honestly, don't use this except in Wolverine testing... + /// + Quick } \ No newline at end of file From 22d1c02fc505e93af24f4557e236489612eafcb5 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Fri, 12 Sep 2025 13:15:21 -0500 Subject: [PATCH 05/25] eliminating an unused method on IDeadLetterAdminService --- .../MessageDatabase.DeadLetterAdminService.cs | 8 -------- .../DeadLetterManagement/IDeadLetterAdminService.cs | 3 --- .../MultiStoreDeadLetterAdminService.cs | 10 ---------- 3 files changed, 21 deletions(-) diff --git a/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs b/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs index a51d71867..13004acdc 100644 --- a/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs +++ b/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs @@ -59,14 +59,6 @@ public async Task> SummarizeAllAsync(string return envelopes; } - public Task> SummarizeByDatabaseAsync(string serviceName, - Uri database, - TimeRange range, - CancellationToken token) - { - return SummarizeAllAsync(serviceName, range, token); - } - protected virtual string toTopClause(DeadLetterEnvelopeQuery query) { return ""; diff --git a/src/Wolverine/Persistence/Durability/DeadLetterManagement/IDeadLetterAdminService.cs b/src/Wolverine/Persistence/Durability/DeadLetterManagement/IDeadLetterAdminService.cs index 335e592ac..5ece75f6e 100644 --- a/src/Wolverine/Persistence/Durability/DeadLetterManagement/IDeadLetterAdminService.cs +++ b/src/Wolverine/Persistence/Durability/DeadLetterManagement/IDeadLetterAdminService.cs @@ -7,9 +7,6 @@ public interface IDeadLetterAdminService { Task> SummarizeAllAsync(string serviceName, TimeRange range, CancellationToken token); - Task> SummarizeByDatabaseAsync(string serviceName, Uri database, - TimeRange range, - CancellationToken token); Task QueryAsync(DeadLetterEnvelopeQuery query, CancellationToken token); diff --git a/src/Wolverine/Persistence/Durability/DeadLetterManagement/MultiStoreDeadLetterAdminService.cs b/src/Wolverine/Persistence/Durability/DeadLetterManagement/MultiStoreDeadLetterAdminService.cs index 83def6151..74ba97d0d 100644 --- a/src/Wolverine/Persistence/Durability/DeadLetterManagement/MultiStoreDeadLetterAdminService.cs +++ b/src/Wolverine/Persistence/Durability/DeadLetterManagement/MultiStoreDeadLetterAdminService.cs @@ -29,16 +29,6 @@ public async Task> SummarizeAllAsync(string return list; } - public async Task> SummarizeByDatabaseAsync(string serviceName, Uri database, TimeRange range, CancellationToken token) - { - if (_databases.TryFind(database, out var service)) - { - return await service.SummarizeAllAsync(serviceName, range, token); - } - - return []; - } - public async Task QueryAsync(DeadLetterEnvelopeQuery query, CancellationToken token) { if (query.Database != null) From 33167574e907c9e21c1da0a20667a75869fe6631 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Fri, 12 Sep 2025 15:09:45 -0500 Subject: [PATCH 06/25] Streamlined the signature of the IDeadLetterAdminService service --- .../PostgresqlMessageStore.cs | 6 + .../MessageDatabase.DeadLetterAdminService.cs | 47 ++------ .../Persistence/SqlServerMessageStore.cs | 23 ++++ .../DeadLetterAdminCompliance.cs | 4 +- .../DeadLetterEnvelopeQuery.cs | 16 ++- .../IDeadLetterAdminService.cs | 3 - .../MessageBatchRequest.cs | 3 - .../MultiStoreDeadLetterAdminService.cs | 111 ------------------ 8 files changed, 54 insertions(+), 159 deletions(-) delete mode 100644 src/Wolverine/Persistence/Durability/DeadLetterManagement/MessageBatchRequest.cs delete mode 100644 src/Wolverine/Persistence/Durability/DeadLetterManagement/MultiStoreDeadLetterAdminService.cs diff --git a/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs b/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs index c42319284..5a2b11798 100644 --- a/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs +++ b/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs @@ -576,4 +576,10 @@ public override DatabaseSagaSchema SagaSchemaFor() return storage; } + protected override void writeMessageIdArrayQueryList(DbCommandBuilder builder, Guid[] messageIds) + { + builder.Append($" and {DatabaseConstants.Id} = ANY("); + builder.AppendParameter(messageIds); + builder.Append(')'); + } } \ No newline at end of file diff --git a/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs b/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs index 13004acdc..c858042d2 100644 --- a/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs +++ b/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs @@ -113,7 +113,7 @@ public async Task QueryAsync(DeadLetterEnvelopeQuery return results; } - private static void writeDeadLetterWhereClause(DeadLetterEnvelopeQuery query, DbCommandBuilder builder) + private void writeDeadLetterWhereClause(DeadLetterEnvelopeQuery query, DbCommandBuilder builder) { if (query.Range.From.HasValue) { @@ -144,8 +144,15 @@ private static void writeDeadLetterWhereClause(DeadLetterEnvelopeQuery query, Db builder.Append($" and {DatabaseConstants.ReceivedAt} = "); builder.AppendParameter(query.ReceivedAt); } + + if (query.MessageIds != null && query.MessageIds.Any()) + { + writeMessageIdArrayQueryList(builder, query.MessageIds); + } } + protected abstract void writeMessageIdArrayQueryList(DbCommandBuilder builder, Guid[] messageIds); + protected abstract void writePagingAfter(DbCommandBuilder builder, int offset, int limit); public Task DiscardAsync(DeadLetterEnvelopeQuery query, CancellationToken token) @@ -177,42 +184,4 @@ public Task ReplayAsync(DeadLetterEnvelopeQuery query, CancellationToken token) return executeCommandBatch(builder, token); } - public Task DiscardAsync(MessageBatchRequest request, CancellationToken token) - { - var builder = ToCommandBuilder(); - - foreach (var id in request.Ids) - { - builder.Append($"delete from {SchemaName}.{DatabaseConstants.DeadLetterTable} where {DatabaseConstants.Id} = "); - builder.AppendParameter(id); - builder.Append(';'); - } - - new MoveReplayableErrorMessagesToIncomingOperation(this).ConfigureCommand(builder); - - return executeCommandBatch(builder, token); - } - - public Task ReplayAsync(MessageBatchRequest request, CancellationToken token) - { - var builder = ToCommandBuilder(); - - foreach (var id in request.Ids) - { - builder.Append( - $"update {SchemaName}.{DatabaseConstants.DeadLetterTable} set {DatabaseConstants.Replayable} = "); - builder.AppendParameter(true); - builder.Append($" where {DatabaseConstants.Id} = "); - builder.AppendParameter(id); - builder.Append(';'); - builder.Append( - $"delete from {SchemaName}.{DatabaseConstants.DeadLetterTable} where {DatabaseConstants.Replayable} = "); - builder.AppendParameter(true); - builder.Append(';'); - } - - new MoveReplayableErrorMessagesToIncomingOperation(this).ConfigureCommand(builder); - - return executeCommandBatch(builder, token); - } } \ No newline at end of file diff --git a/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs b/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs index 396614349..49cf127d5 100644 --- a/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs +++ b/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs @@ -51,6 +51,29 @@ public SqlServerMessageStore(DatabaseSettings database, DurabilitySettings setti } } + protected override void writeMessageIdArrayQueryList(DbCommandBuilder builder, Guid[] messageIds) + { + if (messageIds.Length == 1) + { + builder.Append($" and {DatabaseConstants.Id} = "); + builder.AppendParameter(messageIds.Single()); + } + else + { + builder.Append(" and ("); + builder.Append($"{DatabaseConstants.Id} = "); + builder.AppendParameter(messageIds[0]); + + for (int i = 1; i < messageIds.Length; i++) + { + builder.Append($" or {DatabaseConstants.Id} = "); + builder.AppendParameter(messageIds[i]); + } + + builder.Append(")"); + } + } + protected override INodeAgentPersistence? buildNodeStorage(DatabaseSettings databaseSettings, DbDataSource dataSource) { diff --git a/src/Testing/Wolverine.ComplianceTests/DeadLetterAdminCompliance.cs b/src/Testing/Wolverine.ComplianceTests/DeadLetterAdminCompliance.cs index ec5f01233..62eae5858 100644 --- a/src/Testing/Wolverine.ComplianceTests/DeadLetterAdminCompliance.cs +++ b/src/Testing/Wolverine.ComplianceTests/DeadLetterAdminCompliance.cs @@ -563,7 +563,7 @@ public async Task discard_by_message_batch() await loadAllEnvelopes(); var ids = allEnvelopes.Envelopes.Take(10).Select(x => x.Id).ToArray(); - await theDeadLetters.DiscardAsync(new MessageBatchRequest(ids), CancellationToken.None); + await theDeadLetters.DiscardAsync(new DeadLetterEnvelopeQuery{MessageIds = ids}, CancellationToken.None); // Reload await loadAllEnvelopes(); @@ -611,7 +611,7 @@ public async Task replay_by_message_batch() await loadAllEnvelopes(); var ids = allEnvelopes.Envelopes.Take(10).Select(x => x.Id).ToArray(); - await theDeadLetters.ReplayAsync(new MessageBatchRequest(ids), CancellationToken.None); + await theDeadLetters.ReplayAsync(new DeadLetterEnvelopeQuery(ids), CancellationToken.None); // Reload await loadAllEnvelopes(); diff --git a/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQuery.cs b/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQuery.cs index 2c027d152..160583e13 100644 --- a/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQuery.cs +++ b/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQuery.cs @@ -7,6 +7,15 @@ public DeadLetterEnvelopeQuery(TimeRange range) Range = range; } + public DeadLetterEnvelopeQuery(Guid[] messageIds) + { + MessageIds = messageIds; + } + + public DeadLetterEnvelopeQuery() + { + } + public int PageNumber { get; set; } public int PageSize { get; set; } = 100; public string? MessageType { get; set; } @@ -14,5 +23,10 @@ public DeadLetterEnvelopeQuery(TimeRange range) public string? ReceivedAt { get; set; } public Uri? Database { get; set; } - public TimeRange Range { get; set; } + public TimeRange Range { get; set; } = TimeRange.AllTime(); + + /// + /// If set, this takes precedence over all other options + /// + public Guid[] MessageIds { get; set; } = []; } \ No newline at end of file diff --git a/src/Wolverine/Persistence/Durability/DeadLetterManagement/IDeadLetterAdminService.cs b/src/Wolverine/Persistence/Durability/DeadLetterManagement/IDeadLetterAdminService.cs index 5ece75f6e..8d622eb97 100644 --- a/src/Wolverine/Persistence/Durability/DeadLetterManagement/IDeadLetterAdminService.cs +++ b/src/Wolverine/Persistence/Durability/DeadLetterManagement/IDeadLetterAdminService.cs @@ -13,8 +13,5 @@ Task> SummarizeAllAsync(string serviceName, Task DiscardAsync(DeadLetterEnvelopeQuery query, CancellationToken token); Task ReplayAsync(DeadLetterEnvelopeQuery query, CancellationToken token); - Task DiscardAsync(MessageBatchRequest request, CancellationToken token); - Task ReplayAsync(MessageBatchRequest request, CancellationToken token); - Uri Uri { get; } } \ No newline at end of file diff --git a/src/Wolverine/Persistence/Durability/DeadLetterManagement/MessageBatchRequest.cs b/src/Wolverine/Persistence/Durability/DeadLetterManagement/MessageBatchRequest.cs deleted file mode 100644 index 9f955bc08..000000000 --- a/src/Wolverine/Persistence/Durability/DeadLetterManagement/MessageBatchRequest.cs +++ /dev/null @@ -1,3 +0,0 @@ -namespace Wolverine.Persistence.Durability.DeadLetterManagement; - -public record MessageBatchRequest(Guid[] Ids, Uri? Database = null); \ No newline at end of file diff --git a/src/Wolverine/Persistence/Durability/DeadLetterManagement/MultiStoreDeadLetterAdminService.cs b/src/Wolverine/Persistence/Durability/DeadLetterManagement/MultiStoreDeadLetterAdminService.cs deleted file mode 100644 index 74ba97d0d..000000000 --- a/src/Wolverine/Persistence/Durability/DeadLetterManagement/MultiStoreDeadLetterAdminService.cs +++ /dev/null @@ -1,111 +0,0 @@ -using ImTools; -using JasperFx.Core; - -namespace Wolverine.Persistence.Durability.DeadLetterManagement; - -public class MultiStoreDeadLetterAdminService : IDeadLetterAdminService -{ - private ImHashMap _databases = ImHashMap.Empty; - - public void AddStore(IDeadLetterAdminService service) - { - _databases = _databases.AddOrUpdate(service.Uri, service); - } - - private async Task executeOnAll(Func step) - { - foreach (var entry in _databases.Enumerate().OrderBy(x => x.Key).ToArray()) - { - await step(entry.Value); - } - } - - public async Task> SummarizeAllAsync(string serviceName, TimeRange range, CancellationToken token) - { - var list = new List(); - - await executeOnAll(async service => list.AddRange(await service.SummarizeAllAsync(serviceName, range, token))); - - return list; - } - - public async Task QueryAsync(DeadLetterEnvelopeQuery query, CancellationToken token) - { - if (query.Database != null) - { - if (_databases.TryFind(query.Database!, out var service)) - { - return await service.QueryAsync(query, token); - } - } - - if (query.PageNumber > 0) throw new InvalidOperationException("Must specify a database to use paged querying"); - - var results = new DeadLetterEnvelopeResults { PageNumber = 0 }; - await executeOnAll(async service => - { - if (query.PageSize <= 0) return; - - var singleResults = await service.QueryAsync(query, token); - results.TotalCount += singleResults.TotalCount; - results.Envelopes.AddRange(singleResults.Envelopes); - query.PageSize -= singleResults.Envelopes.Count; - }); - - return results; - } - - public Task DiscardAsync(DeadLetterEnvelopeQuery query, CancellationToken token) - { - if (query.Database != null) - { - if (_databases.TryFind(query.Database!, out var service)) - { - return service.DiscardAsync(query, token); - } - } - - return executeOnAll(service => service.DiscardAsync(query, token)); - } - - public Task ReplayAsync(DeadLetterEnvelopeQuery query, CancellationToken token) - { - if (query.Database != null) - { - if (_databases.TryFind(query.Database!, out var service)) - { - return service.ReplayAsync(query, token); - } - } - - return executeOnAll(service => service.ReplayAsync(query, token)); - } - - public Task DiscardAsync(MessageBatchRequest request, CancellationToken token) - { - if (request.Database != null) - { - if (_databases.TryFind(request.Database!, out var service)) - { - return service.DiscardAsync(request, token); - } - } - - return executeOnAll(service => service.DiscardAsync(request, token)); - } - - public Task ReplayAsync(MessageBatchRequest request, CancellationToken token) - { - if (request.Database != null) - { - if (_databases.TryFind(request.Database!, out var service)) - { - return service.ReplayAsync(request, token); - } - } - - return executeOnAll(service => service.ReplayAsync(request, token)); - } - - public Uri Uri => new("wolverinedb://all"); -} \ No newline at end of file From 4474ee12cb1a5f348337248abf3a0ca4871629fc Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Mon, 15 Sep 2025 11:48:08 -0500 Subject: [PATCH 07/25] fixes on message storage while coding against CritterWatch --- .../Wolverine.Marten/MartenIntegration.cs | 15 ++++++++++++++- .../WolverineOptionsMartenExtensions.cs | 6 ++++++ src/Wolverine/OutgoingMessages.cs | 8 ++++++++ .../DeadLetterEnvelopeQuery.cs | 2 ++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/Persistence/Wolverine.Marten/MartenIntegration.cs b/src/Persistence/Wolverine.Marten/MartenIntegration.cs index 9e6c2fc60..7991e83b3 100644 --- a/src/Persistence/Wolverine.Marten/MartenIntegration.cs +++ b/src/Persistence/Wolverine.Marten/MartenIntegration.cs @@ -63,6 +63,19 @@ public void Configure(WolverineOptions options) options.Policies.Add(); } + + /// + /// In the case of Marten using a database per tenant, you may wish to + /// explicitly determine the master database for Wolverine where Wolverine will store node and envelope information. + /// This does not have to be one of the tenant databases + /// Wolverine will try to use the master database from the Marten configuration when possible + /// + [Obsolete("Prefer MainDatabaseConnectionString")] + public string? MasterDatabaseConnectionString + { + get => MainDatabaseConnectionString; + set => MainDatabaseConnectionString = value; + } /// /// In the case of Marten using a database per tenant, you may wish to @@ -70,7 +83,7 @@ public void Configure(WolverineOptions options) /// This does not have to be one of the tenant databases /// Wolverine will try to use the master database from the Marten configuration when possible /// - public string? MasterDatabaseConnectionString { get; set; } + public string? MainDatabaseConnectionString { get; set; } /// /// In the case of Marten using a database per tenant, you may wish to diff --git a/src/Persistence/Wolverine.Marten/WolverineOptionsMartenExtensions.cs b/src/Persistence/Wolverine.Marten/WolverineOptionsMartenExtensions.cs index 1763189fb..413c06c56 100644 --- a/src/Persistence/Wolverine.Marten/WolverineOptionsMartenExtensions.cs +++ b/src/Persistence/Wolverine.Marten/WolverineOptionsMartenExtensions.cs @@ -175,6 +175,12 @@ internal static IMessageStore BuildMultiTenantedMessageDatabase( IWolverineRuntime runtime, IServiceProvider serviceProvider) { + if (masterDataSource == null && masterDatabaseConnectionString.IsEmpty()) + { + throw new ArgumentOutOfRangeException(nameof(masterDatabaseConnectionString), + $"Wolverine requires a main message store database even if the current Marten tenancy model does not. You may need to explicitly configure that in the {nameof(IntegrateWithWolverine)}() configuration."); + } + var masterSettings = new DatabaseSettings { SchemaName = schemaName, diff --git a/src/Wolverine/OutgoingMessages.cs b/src/Wolverine/OutgoingMessages.cs index eb79932b3..6bc52e10d 100644 --- a/src/Wolverine/OutgoingMessages.cs +++ b/src/Wolverine/OutgoingMessages.cs @@ -12,6 +12,14 @@ namespace Wolverine; /// public class OutgoingMessages : List, IWolverineReturnType, INotToBeRouted { + public OutgoingMessages() + { + } + + public OutgoingMessages(IEnumerable collection) : base(collection) + { + } + /// /// Send a message back to the original sender /// diff --git a/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQuery.cs b/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQuery.cs index 160583e13..208ecd8cb 100644 --- a/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQuery.cs +++ b/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQuery.cs @@ -21,6 +21,8 @@ public DeadLetterEnvelopeQuery() public string? MessageType { get; set; } public string? ExceptionType { get; set; } public string? ReceivedAt { get; set; } + + [Obsolete("Remove this")] public Uri? Database { get; set; } public TimeRange Range { get; set; } = TimeRange.AllTime(); From d8ee284be57b09a0f8811b175ddb9fb15b28728d Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Mon, 15 Sep 2025 14:01:25 -0500 Subject: [PATCH 08/25] Adjustments to the test harness to bring in SignalR into the CritterWatch runtime --- .../SignalR/Wolverine.SignalR/SignalRWolverineExtensions.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Transports/SignalR/Wolverine.SignalR/SignalRWolverineExtensions.cs b/src/Transports/SignalR/Wolverine.SignalR/SignalRWolverineExtensions.cs index abf5fe367..30d430eeb 100644 --- a/src/Transports/SignalR/Wolverine.SignalR/SignalRWolverineExtensions.cs +++ b/src/Transports/SignalR/Wolverine.SignalR/SignalRWolverineExtensions.cs @@ -44,6 +44,8 @@ public static SignalRMessage ToWebSocketGroup(this T Message, string Group public static SignalRListenerConfiguration UseSignalR(this WolverineOptions options) { + options.Services.AddSignalR(); + var transport = options.SignalRTransport(); options.Services.AddSingleton(s => From 794dc79f09a2956e138b1ea9b81d2825d9fc46e6 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Mon, 15 Sep 2025 17:46:55 -0500 Subject: [PATCH 09/25] Added the new [EnlistInCurrentConnectionSaga] attribute and middleware --- .../WebSocketTestContext.cs | 2 ++ .../group_mechanics.cs | 33 ++++++++++++++++++- .../EnlistInCurrentConnectionSagaAttribute.cs | 22 +++++++++++++ .../Internals/EnlistmentOperations.cs | 18 ++++++++++ 4 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 src/Transports/SignalR/Wolverine.SignalR/EnlistInCurrentConnectionSagaAttribute.cs create mode 100644 src/Transports/SignalR/Wolverine.SignalR/Internals/EnlistmentOperations.cs diff --git a/src/Transports/SignalR/Wolverine.SignalR.Tests/WebSocketTestContext.cs b/src/Transports/SignalR/Wolverine.SignalR.Tests/WebSocketTestContext.cs index 0ab883965..991a27298 100644 --- a/src/Transports/SignalR/Wolverine.SignalR.Tests/WebSocketTestContext.cs +++ b/src/Transports/SignalR/Wolverine.SignalR.Tests/WebSocketTestContext.cs @@ -45,6 +45,8 @@ public async Task InitializeAsync() opts.PublishMessage().ToSignalR(); opts.PublishMessage().ToSignalR(); opts.PublishMessage().ToSignalR(); + + opts.PublishMessage().ToSignalR(); }); var app = builder.Build(); diff --git a/src/Transports/SignalR/Wolverine.SignalR.Tests/group_mechanics.cs b/src/Transports/SignalR/Wolverine.SignalR.Tests/group_mechanics.cs index 595f6769e..6d55f8982 100644 --- a/src/Transports/SignalR/Wolverine.SignalR.Tests/group_mechanics.cs +++ b/src/Transports/SignalR/Wolverine.SignalR.Tests/group_mechanics.cs @@ -34,6 +34,23 @@ await blue.TrackActivity().IncludeExternalTransports().AlsoTrack(theWebApp) .All(x => x.Message == "Hello") .ShouldBeTrue(); } + + [Fact] + public async Task take_advantage_of_the_enlist_in_current_connection_saga() + { + var green = await StartClientHost("green"); + var red = await StartClientHost("red"); + var blue = await StartClientHost("blue"); + + var tracked = await green.TrackActivity().IncludeExternalTransports().AlsoTrack(theWebApp) + .WaitForMessageToBeReceivedAt(green) + .SendMessageAndWaitAsync(new AddNumbers(5,6)); + + var record = tracked.Executed.SingleRecord(); + record.ServiceName.ShouldBe("green"); + record.Envelope.Message.ShouldBeOfType().Sum.ShouldBe(11); + + } } public record EnrollMe(string GroupName) : WebSocketMessage; @@ -64,4 +81,18 @@ public static object Handle(BroadCastToGroup msg) public static void Handle(Information msg) => Debug.WriteLine(msg.Message); -} \ No newline at end of file + + [EnlistInCurrentConnectionSaga] + public static DoMath Handle(AddNumbers numbers) + { + return new DoMath(numbers.X, numbers.Y); + } + + public static MathAnswer Handle(DoMath math) => new MathAnswer(math.X + math.Y); + + public static void Handle(MathAnswer msg) => Debug.WriteLine(msg.Sum.ToString()); +} + +public record AddNumbers(int X, int Y); +public record DoMath(int X, int Y); +public record MathAnswer(int Sum); \ No newline at end of file diff --git a/src/Transports/SignalR/Wolverine.SignalR/EnlistInCurrentConnectionSagaAttribute.cs b/src/Transports/SignalR/Wolverine.SignalR/EnlistInCurrentConnectionSagaAttribute.cs new file mode 100644 index 000000000..4411ac9a1 --- /dev/null +++ b/src/Transports/SignalR/Wolverine.SignalR/EnlistInCurrentConnectionSagaAttribute.cs @@ -0,0 +1,22 @@ +using JasperFx.CodeGeneration; +using JasperFx.CodeGeneration.Frames; +using Wolverine.Attributes; +using Wolverine.Runtime.Handlers; +using Wolverine.SignalR.Internals; + +namespace Wolverine.SignalR; + +/// +/// If a handler method or handler type is decorated with this attribute, Wolverine will track +/// any cascaded messages using its SagaId so that eventual responses back to the SignalR transport +/// will to the current connection +/// +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)] +public class EnlistInCurrentConnectionSagaAttribute : ModifyHandlerChainAttribute +{ + public override void Modify(HandlerChain chain, GenerationRules rules) + { + chain.Middleware.Insert(0, new MethodCall(typeof(EnlistmentOperations), nameof(EnlistmentOperations.EnlistInConnectionSaga))); + } +} + diff --git a/src/Transports/SignalR/Wolverine.SignalR/Internals/EnlistmentOperations.cs b/src/Transports/SignalR/Wolverine.SignalR/Internals/EnlistmentOperations.cs new file mode 100644 index 000000000..d606539e4 --- /dev/null +++ b/src/Transports/SignalR/Wolverine.SignalR/Internals/EnlistmentOperations.cs @@ -0,0 +1,18 @@ +namespace Wolverine.SignalR.Internals; + +public static class EnlistmentOperations +{ + /// + /// This silly thing just helps track any messages cascaded out from the current + /// message being handled to the saga id of the current connection to help send + /// responses back to the originating connection + /// + /// + public static void EnlistInConnectionSaga(Envelope envelope) + { + if (envelope is SignalREnvelope se) + { + se.SagaId = new WebSocketRouting.Connection(se.ConnectionId).ToString(); + } + } +} \ No newline at end of file From 02f218fc202b2dc28423c29ea2b74e8364eefe96 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Tue, 16 Sep 2025 08:45:54 -0500 Subject: [PATCH 10/25] Introducing MessageStoreCollection needed Source to be settable for CritterWatch testing Having IDeadLetterAdminService just implement IMessageStore, will make CW internals easier. Closes GH-1702 Introduced the new MessageStoreRole Tests on message store role assignment WIP: preliminary work on the new MessageStoreCollection wip: Using MessageStoreCollection inside of WolverineRuntime. Got the PersistenceTests passing wip: most of the postgresql tests pass --- .../bootstrap_with_no_persistence.cs | 3 +- ...ncillary_stores_use_different_databases.cs | 13 +- ..._ancillary_marten_stores_with_wolverine.cs | 5 +- .../MultiTenancy/MultiTenancyFixture.cs | 2 +- ...ootstrapping_and_database_configuration.cs | 18 +- .../MultiTenancy/multi_tenancy_queue_usage.cs | 3 + .../application_of_transaction_middleware.cs | 3 +- .../PersistenceTests/durability_with_local.cs | 3 + .../modular_monolith_usage.cs | 11 +- .../registration_of_message_stores.cs | 307 +++++++++++++++++ .../Agents/node_persistence.cs | 2 +- ..._store_initialization_and_configuration.cs | 10 +- .../SqlServerTests/Agents/node_persistence.cs | 2 +- ..._store_initialization_and_configuration.cs | 10 +- ...cillaryWolverineOptionsMartenExtensions.cs | 10 +- .../MartenMessageDatabaseSource.cs | 10 +- .../Publishing/OutboxedSessionFactory.cs | 2 +- .../WolverineOptionsMartenExtensions.cs | 4 +- .../PostgresqlBackedPersistence.cs | 23 +- .../PostgresqlConfigurationExtensions.cs | 13 +- .../PostgresqlMessageStore.cs | 2 +- .../PostgresqlTenantedMessageStore.cs | 23 +- .../Transport/PostgresqlTransport.cs | 59 ++-- .../Wolverine.RDBMS/DatabaseSettings.cs | 3 +- .../Durability/DeleteOldNodeEventRecords.cs | 3 +- .../Wolverine.RDBMS/DurabilityAgent.cs | 3 +- .../Wolverine.RDBMS/IMessageDatabase.cs | 2 - .../Wolverine.RDBMS/MessageDatabase.Admin.cs | 3 +- .../MessageDatabase.DeadLetterAdminService.cs | 2 +- .../Wolverine.RDBMS/MessageDatabase.cs | 20 +- .../MultiTenancy/IMessageDatabaseSource.cs | 15 - .../MultiTenancy/MessageDatabaseDiscovery.cs | 40 +-- .../RavenDbMessageStore.DeadLetters.cs | 20 ++ .../Internals/RavenDbMessageStore.cs | 7 + .../Persistence/SqlServerMessageStore.cs | 3 +- .../SqlServerBackedPersistence.cs | 28 +- .../SqlServerConfigurationExtensions.cs | 6 +- .../SqlServerTenantedMessageStore.cs | 10 +- .../Transport/SqlServerTransport.cs | 4 +- .../CoreTests/Runtime/MockWolverineRuntime.cs | 14 +- .../DeadLetterAdminCompliance.cs | 4 +- .../Configuration/EndpointCollection.cs | 8 +- src/Wolverine/Envelope.cs | 2 +- src/Wolverine/HostBuilderExtensions.cs | 7 +- .../IDeadLetterAdminService.cs | 17 - .../Durability/DurabilityAgentFamily.cs | 141 -------- .../Persistence/Durability/IDeadLetters.cs | 37 ++ .../Persistence/Durability/IMessageStore.cs | 32 ++ .../Durability/MultiTenantedMessageStore.cs | 26 ++ .../Durability/NullMessageStore.cs | 25 ++ .../Persistence/MessageStoreCollection.cs | 325 ++++++++++++++++++ .../Runtime/Agents/NodeAgentController.cs | 3 +- src/Wolverine/Runtime/IWolverineRuntime.cs | 12 +- .../Runtime/WolverineRuntime.Agents.cs | 4 +- .../Runtime/WolverineRuntime.HostService.cs | 11 +- src/Wolverine/Runtime/WolverineRuntime.cs | 32 +- src/Wolverine/WolverineSystemPart.cs | 27 +- 57 files changed, 1050 insertions(+), 384 deletions(-) create mode 100644 src/Persistence/PersistenceTests/registration_of_message_stores.cs delete mode 100644 src/Persistence/Wolverine.RDBMS/MultiTenancy/IMessageDatabaseSource.cs delete mode 100644 src/Wolverine/Persistence/Durability/DeadLetterManagement/IDeadLetterAdminService.cs delete mode 100644 src/Wolverine/Persistence/Durability/DurabilityAgentFamily.cs create mode 100644 src/Wolverine/Persistence/MessageStoreCollection.cs diff --git a/src/Http/Wolverine.Http.Tests/bootstrap_with_no_persistence.cs b/src/Http/Wolverine.Http.Tests/bootstrap_with_no_persistence.cs index 58cfa0842..8ab9c15e0 100644 --- a/src/Http/Wolverine.Http.Tests/bootstrap_with_no_persistence.cs +++ b/src/Http/Wolverine.Http.Tests/bootstrap_with_no_persistence.cs @@ -24,8 +24,7 @@ public async Task start_up_with_no_persistence() }); #endregion - - host.Services.GetRequiredService().ShouldBeOfType(); + host.GetRuntime().Storage.ShouldBeOfType(); } } \ No newline at end of file diff --git a/src/Persistence/MartenTests/AncillaryStores/ancillary_stores_use_different_databases.cs b/src/Persistence/MartenTests/AncillaryStores/ancillary_stores_use_different_databases.cs index 569074575..f4320721e 100644 --- a/src/Persistence/MartenTests/AncillaryStores/ancillary_stores_use_different_databases.cs +++ b/src/Persistence/MartenTests/AncillaryStores/ancillary_stores_use_different_databases.cs @@ -13,6 +13,7 @@ using Wolverine.Marten.Publishing; using Wolverine.Persistence.Durability; using Wolverine.Runtime; +using Wolverine.Runtime.Agents; using Wolverine.Tracking; namespace MartenTests.AncillaryStores; @@ -23,7 +24,7 @@ public class ancillary_stores_use_different_databases : IAsyncLifetime private string playersConnectionString; private string thingsConnectionString; - private DurabilityAgentFamily theFamily; + private IAgentFamily theStores; public async Task InitializeAsync() { @@ -64,8 +65,8 @@ public async Task InitializeAsync() opts.Services.AddResourceSetupOnStartup(); }).StartAsync(); - - theFamily = new DurabilityAgentFamily(theHost.GetRuntime()); + + theStores = theHost.GetRuntime().Stores; } public async Task DisposeAsync() @@ -106,10 +107,10 @@ public async Task create_transaction_for_separate_store_on_different_database() [Fact] public async Task have_durability_agents_for_other_databases() { - var uris = await theFamily.AllKnownAgentsAsync(); - uris.ShouldBe([ - new Uri("wolverinedb://postgresql/localhost/postgres/wolverine"), + var uris = await theStores.AllKnownAgentsAsync(); + uris.OrderBy(x => x.ToString()).ShouldBe([ new Uri("wolverinedb://postgresql/localhost/players/wolverine"), + new Uri("wolverinedb://postgresql/localhost/postgres/wolverine"), new Uri("wolverinedb://postgresql/localhost/things/wolverine"), ]); } diff --git a/src/Persistence/MartenTests/AncillaryStores/bootstrapping_ancillary_marten_stores_with_wolverine.cs b/src/Persistence/MartenTests/AncillaryStores/bootstrapping_ancillary_marten_stores_with_wolverine.cs index 329d05bc3..dd8d2fade 100644 --- a/src/Persistence/MartenTests/AncillaryStores/bootstrapping_ancillary_marten_stores_with_wolverine.cs +++ b/src/Persistence/MartenTests/AncillaryStores/bootstrapping_ancillary_marten_stores_with_wolverine.cs @@ -19,6 +19,7 @@ using Wolverine.Postgresql; using Wolverine.RDBMS; using Wolverine.RDBMS.MultiTenancy; +using Wolverine.Runtime.Agents; using Wolverine.Tracking; using Xunit.Abstractions; @@ -30,7 +31,7 @@ public class bootstrapping_ancillary_marten_stores_with_wolverine : IAsyncLifeti private string tenant1ConnectionString; private string tenant2ConnectionString; private string tenant3ConnectionString; - private DurabilityAgentFamily theFamily; + private IAgentFamily theFamily; private IHost theHost; public bootstrapping_ancillary_marten_stores_with_wolverine(ITestOutputHelper output) @@ -101,7 +102,7 @@ public async Task InitializeAsync() #endregion - theFamily = new DurabilityAgentFamily(theHost.GetRuntime()); + theFamily = theHost.GetRuntime().Stores; } public async Task DisposeAsync() diff --git a/src/Persistence/MartenTests/MultiTenancy/MultiTenancyFixture.cs b/src/Persistence/MartenTests/MultiTenancy/MultiTenancyFixture.cs index c7894f0d2..21e003f5c 100644 --- a/src/Persistence/MartenTests/MultiTenancy/MultiTenancyFixture.cs +++ b/src/Persistence/MartenTests/MultiTenancy/MultiTenancyFixture.cs @@ -23,7 +23,7 @@ public MultiTenancyContext(MultiTenancyFixture fixture) { Fixture = fixture; Runtime = fixture.Host.GetRuntime(); - Stores = Runtime.Storage.ShouldBeOfType(); + Stores = Runtime.Stores.MultiTenanted.Single(); } public MultiTenantedMessageStore Stores { get; } diff --git a/src/Persistence/MartenTests/MultiTenancy/basic_bootstrapping_and_database_configuration.cs b/src/Persistence/MartenTests/MultiTenancy/basic_bootstrapping_and_database_configuration.cs index 53825c73f..2077acde7 100644 --- a/src/Persistence/MartenTests/MultiTenancy/basic_bootstrapping_and_database_configuration.cs +++ b/src/Persistence/MartenTests/MultiTenancy/basic_bootstrapping_and_database_configuration.cs @@ -3,6 +3,7 @@ using Npgsql; using Shouldly; using Weasel.Postgresql; +using Wolverine.Persistence.Durability; using Wolverine.RDBMS; using Wolverine.Transports; @@ -30,6 +31,19 @@ public void should_have_the_specified_master_database_as_master() .Database.ShouldBe("postgres"); } + [Fact] + public async Task store_roles() + { + Stores.Main.Role.ShouldBe(MessageStoreRole.Main); + foreach (var activeDatabase in Stores.ActiveDatabases()) + { + if (activeDatabase != Stores.Main) + { + activeDatabase.Role.ShouldBe(MessageStoreRole.Tenant); + } + } + } + [Fact] public void knows_about_tenant_databases() { @@ -104,9 +118,9 @@ public void only_the_master_database_is_the_master() { foreach (var database in Stores.ActiveDatabases().OfType().Where(x => x.Name != StorageConstants.Main)) { - database.IsMain.ShouldBeFalse(); + database.Role.ShouldBe(MessageStoreRole.Tenant); } - Stores.Main.As().IsMain.ShouldBeTrue(); + Stores.Main.As().Role.ShouldBe(MessageStoreRole.Main); } } \ No newline at end of file diff --git a/src/Persistence/MartenTests/MultiTenancy/multi_tenancy_queue_usage.cs b/src/Persistence/MartenTests/MultiTenancy/multi_tenancy_queue_usage.cs index fd1841be4..6f3d5669d 100644 --- a/src/Persistence/MartenTests/MultiTenancy/multi_tenancy_queue_usage.cs +++ b/src/Persistence/MartenTests/MultiTenancy/multi_tenancy_queue_usage.cs @@ -84,6 +84,8 @@ public async Task InitializeAsync() opts.Services.AddMarten(o => { + o.DisableNpgsqlLogging = true; + // This is a new strategy for configuring tenant databases with Marten // In this usage, Marten is tracking the tenant databases in a single table in the "master" // database by tenant @@ -191,6 +193,7 @@ public async Task send_message_through_tenant() var message = new CreateTenantDoc("Blue", 10); var tracked = await _sender.TrackActivity() .AlsoTrack(_receiver) + .Timeout(20.Seconds()) .IncludeExternalTransports() .SendMessageAndWaitAsync(message, new DeliveryOptions { TenantId = "tenant3" }); diff --git a/src/Persistence/PersistenceTests/application_of_transaction_middleware.cs b/src/Persistence/PersistenceTests/application_of_transaction_middleware.cs index b2bd73119..2defcb28e 100644 --- a/src/Persistence/PersistenceTests/application_of_transaction_middleware.cs +++ b/src/Persistence/PersistenceTests/application_of_transaction_middleware.cs @@ -12,6 +12,7 @@ using Wolverine.EntityFrameworkCore; using Wolverine.EntityFrameworkCore.Codegen; using Wolverine.Marten.Persistence.Sagas; +using Wolverine.Persistence.Durability; using Wolverine.Postgresql; using Wolverine.Runtime; using Wolverine.Runtime.Handlers; @@ -37,7 +38,7 @@ public async Task InitializeAsync() opts.Services.AddDbContextWithWolverineIntegration(x => x.UseSqlServer(Servers.SqlServerConnectionString)); - opts.PersistMessagesWithSqlServer(Servers.SqlServerConnectionString); + opts.PersistMessagesWithSqlServer(Servers.SqlServerConnectionString, role:MessageStoreRole.Ancillary); opts.PersistMessagesWithPostgresql(Servers.PostgresConnectionString); opts.Policies.AutoApplyTransactions(); diff --git a/src/Persistence/PersistenceTests/durability_with_local.cs b/src/Persistence/PersistenceTests/durability_with_local.cs index 002cbd390..1c8e73e30 100644 --- a/src/Persistence/PersistenceTests/durability_with_local.cs +++ b/src/Persistence/PersistenceTests/durability_with_local.cs @@ -11,6 +11,7 @@ using Wolverine.Marten; using Wolverine.Persistence.Durability; using Wolverine.Runtime.Handlers; +using Wolverine.Tracking; using Xunit; namespace PersistenceTests; @@ -21,6 +22,8 @@ public class durability_with_local : PostgresqlContext public async Task should_recover_persisted_messages() { using var host1 = await WolverineHost.ForAsync(opts => opts.ConfigureDurableSender(true, true)); + await host1.GetRuntime().Storage.Admin.RebuildAsync(); + await host1.SendAsync(new ReceivedMessage()); var counts = await host1.Get().Admin.FetchCountsAsync(); diff --git a/src/Persistence/PersistenceTests/modular_monolith_usage.cs b/src/Persistence/PersistenceTests/modular_monolith_usage.cs index 94552f1a7..e2bcc4a26 100644 --- a/src/Persistence/PersistenceTests/modular_monolith_usage.cs +++ b/src/Persistence/PersistenceTests/modular_monolith_usage.cs @@ -40,7 +40,7 @@ public async Task set_the_default_message_store_schema_name() }).StartAsync(); var runtime = host.GetRuntime(); - var stores = runtime.AncillaryStores.OfType(); + var stores = (await runtime.Stores.FindAllAsync()).OfType().ToArray(); stores.Any().ShouldBeTrue(); foreach (var store in stores) @@ -72,7 +72,7 @@ public async Task set_the_default_message_store_schema_name_2() }).StartAsync(); var runtime = host.GetRuntime(); - var stores = runtime.AncillaryStores.OfType(); + var stores = (await runtime.Stores.FindAllAsync()).OfType().ToArray(); stores.Any().ShouldBeTrue(); foreach (var store in stores) @@ -103,7 +103,7 @@ public async Task do_not_override_when_the_schema_name_is_explicitly_set() }).StartAsync(); var runtime = host.GetRuntime(); - var stores = runtime.AncillaryStores.OfType(); + var stores = (await runtime.Stores.FindAllAsync()).OfType().ToArray(); stores.Any().ShouldBeTrue(); foreach (var store in stores) @@ -120,6 +120,9 @@ public async Task using_the_marten_schema_name_with_no_other_settings() { //opts.Durability.MessageStorageSchemaName = "wolverine"; + // Gotta have this for the nodes + opts.PersistMessagesWithPostgresql(Servers.PostgresConnectionString, "main"); + opts.Services.AddMartenStore(m => { m.Connection(Servers.PostgresConnectionString); @@ -134,7 +137,7 @@ public async Task using_the_marten_schema_name_with_no_other_settings() }).StartAsync(); var runtime = host.GetRuntime(); - var stores = runtime.AncillaryStores.OfType(); + var stores = (await runtime.Stores.FindAllAsync()).OfType().ToArray(); stores.OfType>().Single().As() .Settings.SchemaName.ShouldBe("players"); diff --git a/src/Persistence/PersistenceTests/registration_of_message_stores.cs b/src/Persistence/PersistenceTests/registration_of_message_stores.cs new file mode 100644 index 000000000..8e306610a --- /dev/null +++ b/src/Persistence/PersistenceTests/registration_of_message_stores.cs @@ -0,0 +1,307 @@ +using IntegrationTests; +using JasperFx.Descriptors; +using JasperFx.Resources; +using Marten; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Npgsql; +using Shouldly; +using Weasel.Core.MultiTenancy; +using Weasel.Postgresql; +using Weasel.Postgresql.Migrations; +using Wolverine; +using Wolverine.Marten; +using Wolverine.Persistence; +using Wolverine.Persistence.Durability; +using Wolverine.Postgresql; +using Wolverine.Tracking; +using Xunit; +using Xunit.Abstractions; + +namespace PersistenceTests; + +/* + * TODO -- register multiple postgresql outside of Marten + * TODO -- register multiple Sql Server + * TODO -- make all main, see assertion, postgresql + * TODO -- make all main, see assertion, sql server + * TODO -- make none, "Service" should be Nullo + * TODO -- make one main, other ancillary, postgresql + * TODO -- make one main, other ancillary, sql server + * Move on to the durability agent family + * + * + * + */ + +public class registration_of_message_stores(ITestOutputHelper Output) : IAsyncLifetime +{ + private IHost _host; + private string connectionString1; + private string connectionString2; + private string connectionString3; + private string connectionString4; + + private async Task CreateDatabaseIfNotExists(NpgsqlConnection conn, string databaseName) + { + var builder = new NpgsqlConnectionStringBuilder(Servers.PostgresConnectionString); + + var exists = await conn.DatabaseExists(databaseName); + if (!exists) + { + await new DatabaseSpecification().BuildDatabase(conn, databaseName); + } + + builder.Database = databaseName; + + return builder.ConnectionString; + } + + public async Task InitializeAsync() + { + using var conn = new NpgsqlConnection(Servers.PostgresConnectionString); + await conn.OpenAsync(); + + connectionString1 = await CreateDatabaseIfNotExists(conn, "database1"); + connectionString2 = await CreateDatabaseIfNotExists(conn, "database2"); + connectionString3 = await CreateDatabaseIfNotExists(conn, "database3"); + connectionString4 = await CreateDatabaseIfNotExists(conn, "database4"); + } + + async Task IAsyncLifetime.DisposeAsync() + { + if (_host != null) + { + await _host.StopAsync(); + } + } + + + private async Task startHost(Action configure) + { + _host = await Host.CreateDefaultBuilder() + .UseWolverine(configure).ConfigureServices(services => services.AddResourceSetupOnStartup()).StartAsync(); + + var collection = new MessageStoreCollection(_host.GetRuntime(), _host.Services.GetServices(), _host.Services.GetServices()); + + await collection.InitializeAsync(); + + return collection; + } + + [Fact] + public async Task for_single_database() + { + // Not much to it + var collection = await startHost(opts => + { + opts.PersistMessagesWithPostgresql(Servers.PostgresConnectionString); + }); + + var expected = new Uri("wolverinedb://postgresql/localhost/postgres/wolverine"); + (await collection.FindAllAsync()).Single().Uri.ShouldBe(expected); + + (await collection.FindDatabasesAsync([expected])).Single().Uri.ShouldBe(expected); + + (await collection.FindDatabaseAsync(expected)).Uri.ShouldBe(expected); + + collection.Cardinality().ShouldBe(DatabaseCardinality.Single); + } + + [Fact] + public async Task for_several_ancillary_marten_databases() + { + var collection = await startHost(opts => + { + opts.Durability.MessageStorageSchemaName = "wolverine"; + + opts.Services.AddMarten(m => + { + m.Connection(Servers.PostgresConnectionString); + }).IntegrateWithWolverine(); + + opts.Services.AddMartenStore(m => + { + m.Connection(Servers.PostgresConnectionString); + m.DatabaseSchemaName = "first"; + }).IntegrateWithWolverine(); + + opts.Services.AddMartenStore(m => + { + m.Connection(Servers.PostgresConnectionString); + m.DatabaseSchemaName = "second"; + }).IntegrateWithWolverine(); + }); + + // STILL only one expected message store here + var expected = new Uri("wolverinedb://postgresql/localhost/postgres/wolverine"); + var services = await collection.FindAllAsync(); + + services.Single().Uri.ShouldBe(expected); + + (await collection.FindDatabasesAsync([expected])).Single().Uri.ShouldBe(expected); + + (await collection.FindDatabaseAsync(expected)).Uri.ShouldBe(expected); + + collection.Cardinality().ShouldBe(DatabaseCardinality.Single); + } + + [Fact] + public async Task ancillary_marten_databases_using_different_databases() + { + var collection = await startHost(opts => + { + opts.Durability.MessageStorageSchemaName = "wolverine"; + + opts.Services.AddMarten(m => + { + m.Connection(Servers.PostgresConnectionString); + }).IntegrateWithWolverine(); + + opts.Services.AddMartenStore(m => + { + m.Connection(connectionString1); + m.DatabaseSchemaName = "first"; + }).IntegrateWithWolverine(); + + opts.Services.AddMartenStore(m => + { + m.Connection(connectionString2); + m.DatabaseSchemaName = "second"; + }).IntegrateWithWolverine(); + }); + + // STILL only one expected message store here + var main = new Uri("wolverinedb://postgresql/localhost/postgres/wolverine"); + var first = new Uri("wolverinedb://postgresql/localhost/database1/wolverine"); + var second = new Uri("wolverinedb://postgresql/localhost/database2/wolverine"); + + var services = await collection.FindAllAsync(); + + services.Select(x => x.Uri).OrderBy(x => x.ToString()) + .ShouldBe([first, second, main]); + + + + (await collection.FindDatabasesAsync([first, main])).Select(x => x.Uri).ShouldBe([first, main]); + + (await collection.FindDatabaseAsync(second)).Uri.ShouldBe(second); + + collection.Cardinality().ShouldBe(DatabaseCardinality.Single); + } + + [Fact] + public async Task using_static_multi_tenancy() + { + var collection = await startHost(opts => + { + opts.Durability.MessageStorageSchemaName = "wolverine"; + opts.Services.AddMarten(m => + { + m.DisableNpgsqlLogging = true; + m.MultiTenantedDatabases(t => + { + t.AddSingleTenantDatabase(connectionString1, "t1"); + t.AddSingleTenantDatabase(connectionString2, "t2"); + t.AddSingleTenantDatabase(connectionString3, "t3"); + t.AddSingleTenantDatabase(connectionString4, "t4"); + }); + }).IntegrateWithWolverine(w => w.MasterDatabaseConnectionString = Servers.PostgresConnectionString); + }); + + var main = new Uri("wolverinedb://postgresql/localhost/postgres/wolverine"); + var db1 = new Uri("wolverinedb://postgresql/localhost/database1/wolverine"); + var db2 = new Uri("wolverinedb://postgresql/localhost/database2/wolverine"); + var db3 = new Uri("wolverinedb://postgresql/localhost/database3/wolverine"); + var db4 = new Uri("wolverinedb://postgresql/localhost/database4/wolverine"); + + + var services = await collection.FindAllAsync(); + + services.Select(x => x.Uri).OrderBy(x => x.ToString()) + .ShouldBe([db1, db2, db3, db4, main]); + + (await collection.FindDatabaseAsync(db3)).Uri.ShouldBe(db3); + (await collection.FindDatabaseAsync(main)).Uri.ShouldBe(main); + + (await collection.FindDatabasesAsync([db1, main])).Select(x => x.Uri).ShouldBe([db1, main]); + + collection.Cardinality().ShouldBe(DatabaseCardinality.StaticMultiple); + } + + [Fact] + public async Task using_dynamic_multi_tenancy() + { + // THIS SHOULD TAKE CARE OF THE TEST RERUNS, but for some reason doesn't + await dropWolverineSchema(); + + var collection = await startHost(opts => + { + opts.Durability.MessageStorageSchemaName = "wolverine"; + opts.Services.AddMarten(m => + { + m.MultiTenantedDatabasesWithMasterDatabaseTable(Servers.PostgresConnectionString); + + + }).IntegrateWithWolverine(w => w.MasterDatabaseConnectionString = Servers.PostgresConnectionString); + }); + + await _host.ClearAllTenantDatabaseRecordsAsync(); + + await _host.AddTenantDatabaseAsync("t1", connectionString1); + await _host.AddTenantDatabaseAsync("t2", connectionString2); + + + var main = new Uri("wolverinedb://postgresql/localhost/postgres/wolverine"); + var db1 = new Uri("wolverinedb://postgresql/localhost/database1/wolverine"); + var db2 = new Uri("wolverinedb://postgresql/localhost/database2/wolverine"); + var db3 = new Uri("wolverinedb://postgresql/localhost/database3/wolverine"); + var db4 = new Uri("wolverinedb://postgresql/localhost/database4/wolverine"); + + + var services = await collection.FindAllAsync(); + + services.Select(x => x.Uri).OrderBy(x => x.ToString()) + .ShouldBe([db1, db2, main]); + + (await collection.FindDatabaseAsync(db2)).Uri.ShouldBe(db2); + (await collection.FindDatabaseAsync(main)).Uri.ShouldBe(main); + + await _host.AddTenantDatabaseAsync("t3", connectionString3); + await _host.AddTenantDatabaseAsync("t4", connectionString4); + + services = await collection.FindAllAsync(); + + services.Select(x => x.Uri).OrderBy(x => x.ToString()) + .ShouldBe([db1, db2, db3, db4, main]); + + (await collection.FindDatabasesAsync([db1, main])).Select(x => x.Uri).ShouldBe([db1, main]); + + collection.Cardinality().ShouldBe(DatabaseCardinality.DynamicMultiple); + } + + private static async Task dropWolverineSchema() + { + using var conn = new NpgsqlConnection(Servers.PostgresConnectionString); + await conn.OpenAsync(); + await conn.DropSchemaAsync("wolverine"); + await conn.CloseAsync(); + } + + [Fact] + public async Task no_message_stores_still_works() + { + var collection = await startHost(opts => { }); + collection.Main.ShouldBeOfType(); + + (await collection.FindAllAsync()).Any().ShouldBeFalse(); + + collection.Cardinality().ShouldBe(DatabaseCardinality.None); + } +} + + + +public interface IFirstStore : IDocumentStore{} +public interface ISecondStore : IDocumentStore{} + diff --git a/src/Persistence/PostgresqlTests/Agents/node_persistence.cs b/src/Persistence/PostgresqlTests/Agents/node_persistence.cs index 0a091924d..22ec7a73d 100644 --- a/src/Persistence/PostgresqlTests/Agents/node_persistence.cs +++ b/src/Persistence/PostgresqlTests/Agents/node_persistence.cs @@ -28,7 +28,7 @@ protected override async Task buildCleanMessageStore() { ConnectionString = Servers.PostgresConnectionString, SchemaName = "nodes", - IsMain = true + Role = MessageStoreRole.Main }; var database = new PostgresqlMessageStore(settings, new DurabilitySettings(), diff --git a/src/Persistence/PostgresqlTests/message_store_initialization_and_configuration.cs b/src/Persistence/PostgresqlTests/message_store_initialization_and_configuration.cs index 164e2808b..2d2aa79cb 100644 --- a/src/Persistence/PostgresqlTests/message_store_initialization_and_configuration.cs +++ b/src/Persistence/PostgresqlTests/message_store_initialization_and_configuration.cs @@ -8,10 +8,12 @@ using Shouldly; using Weasel.Postgresql; using Wolverine; +using Wolverine.Persistence.Durability; using Wolverine.Postgresql; using Wolverine.RDBMS; using Wolverine.RDBMS.Transport; using Wolverine.Runtime; +using Wolverine.Tracking; namespace PostgresqlTests; @@ -49,6 +51,12 @@ public async Task DisposeAsync() } } + [Fact] + public void main_store_role() + { + _host.GetRuntime().Storage.Role.ShouldBe(MessageStoreRole.Main); + } + [Fact] public async Task builds_the_node_and_control_queue_tables() { @@ -99,7 +107,7 @@ public async Task deletes_the_node_on_shutdown() var settings = new DatabaseSettings { ConnectionString = Servers.PostgresConnectionString, - IsMain = false, + Role = MessageStoreRole.Tenant, SchemaName = "registry" }; diff --git a/src/Persistence/SqlServerTests/Agents/node_persistence.cs b/src/Persistence/SqlServerTests/Agents/node_persistence.cs index a911cac58..b5e5725ec 100644 --- a/src/Persistence/SqlServerTests/Agents/node_persistence.cs +++ b/src/Persistence/SqlServerTests/Agents/node_persistence.cs @@ -26,7 +26,7 @@ protected override async Task buildCleanMessageStore() { ConnectionString = Servers.SqlServerConnectionString, SchemaName = "nodes", - IsMain = true + Role = MessageStoreRole.Main }; var database = new SqlServerMessageStore(settings, new DurabilitySettings(), diff --git a/src/Persistence/SqlServerTests/message_store_initialization_and_configuration.cs b/src/Persistence/SqlServerTests/message_store_initialization_and_configuration.cs index 7868366d1..9423dd93b 100644 --- a/src/Persistence/SqlServerTests/message_store_initialization_and_configuration.cs +++ b/src/Persistence/SqlServerTests/message_store_initialization_and_configuration.cs @@ -8,12 +8,14 @@ using Shouldly; using Weasel.SqlServer; using Wolverine; +using Wolverine.Persistence.Durability; using Wolverine.RDBMS; using Wolverine.RDBMS.Sagas; using Wolverine.RDBMS.Transport; using Wolverine.Runtime; using Wolverine.SqlServer; using Wolverine.SqlServer.Persistence; +using Wolverine.Tracking; namespace SqlServerTests; @@ -50,6 +52,12 @@ public async Task DisposeAsync() _host.Dispose(); } } + + [Fact] + public void main_store_role() + { + _host.GetRuntime().Storage.Role.ShouldBe(MessageStoreRole.Main); + } [Fact] public async Task builds_the_node_and_control_queue_tables() @@ -101,7 +109,7 @@ public async Task deletes_the_node_on_shutdown() var settings = new DatabaseSettings { ConnectionString = Servers.SqlServerConnectionString, - IsMain = false, + Role = MessageStoreRole.Tenant, SchemaName = "registry" }; diff --git a/src/Persistence/Wolverine.Marten/AncillaryWolverineOptionsMartenExtensions.cs b/src/Persistence/Wolverine.Marten/AncillaryWolverineOptionsMartenExtensions.cs index a7dd25993..f6d1a02a2 100644 --- a/src/Persistence/Wolverine.Marten/AncillaryWolverineOptionsMartenExtensions.cs +++ b/src/Persistence/Wolverine.Marten/AncillaryWolverineOptionsMartenExtensions.cs @@ -110,18 +110,18 @@ internal static IAncillaryMessageStore BuildMultiTenantedMessageDatabase(s DocumentStore store, IWolverineRuntime runtime) where T : IDocumentStore { - var masterSettings = new DatabaseSettings + var mainSettings = new DatabaseSettings { ConnectionString = masterDatabaseConnectionString, SchemaName = schemaName, AutoCreate = autoCreate ?? store.Options.AutoCreateSchemaObjects, - IsMain = true, + Role = MessageStoreRole.Ancillary, CommandQueuesEnabled = true, DataSource = masterDataSource }; - var dataSource = findMasterDataSource(store, masterSettings); - var master = new PostgresqlMessageStore(masterSettings, runtime.Options.Durability, dataSource, + var dataSource = findMasterDataSource(store, mainSettings); + var master = new PostgresqlMessageStore(mainSettings, runtime.Options.Durability, dataSource, runtime.LoggerFactory.CreateLogger()) { Name = "Master", @@ -145,7 +145,7 @@ internal static IAncillaryMessageStore BuildSinglePostgresqlMessageStore( { SchemaName = schemaName, AutoCreate = autoCreate ?? store.Options.AutoCreateSchemaObjects, - IsMain = true, + Role = MessageStoreRole.Ancillary, ScheduledJobLockId = $"{schemaName ?? "public"}:scheduled-jobs".GetDeterministicHashCode() }; diff --git a/src/Persistence/Wolverine.Marten/MartenMessageDatabaseSource.cs b/src/Persistence/Wolverine.Marten/MartenMessageDatabaseSource.cs index bcb012679..3384f9ca8 100644 --- a/src/Persistence/Wolverine.Marten/MartenMessageDatabaseSource.cs +++ b/src/Persistence/Wolverine.Marten/MartenMessageDatabaseSource.cs @@ -28,7 +28,7 @@ public MartenMessageDatabaseSource(string schemaName, AutoCreate autoCreate, T s } } -internal class MartenMessageDatabaseSource : IMessageDatabaseSource +internal class MartenMessageDatabaseSource : ITenantedMessageSource { private readonly AutoCreate _autoCreate; @@ -92,7 +92,7 @@ public async ValueTask FindAsync(string tenantId) return store; } - store = createWolverineStore(database); + store = createTenantWolverineStore(database); store.Initialize(_runtime); _stores = _stores.AddOrUpdate(tenantId, store); @@ -117,7 +117,7 @@ public async Task RefreshAsync() { if (!_databases.Contains(martenDatabase.Identifier)) { - var wolverineStore = createWolverineStore(martenDatabase); + var wolverineStore = createTenantWolverineStore(martenDatabase); if (_runtime.Options.AutoBuildMessageStorageOnStartup != AutoCreate.None) { await wolverineStore.Admin.MigrateAsync(); @@ -145,12 +145,12 @@ public async ValueTask ConfigureDatabaseAsync(Func _configurations.Add(configureDatabase); } - private PostgresqlMessageStore createWolverineStore(IMartenDatabase database) + private PostgresqlMessageStore createTenantWolverineStore(IMartenDatabase database) { var settings = new DatabaseSettings { SchemaName = _schemaName, - IsMain = false, + Role = MessageStoreRole.Tenant, AutoCreate = _autoCreate, CommandQueuesEnabled = false, DataSource = database.As().DataSource diff --git a/src/Persistence/Wolverine.Marten/Publishing/OutboxedSessionFactory.cs b/src/Persistence/Wolverine.Marten/Publishing/OutboxedSessionFactory.cs index 633680e31..d4dfae252 100644 --- a/src/Persistence/Wolverine.Marten/Publishing/OutboxedSessionFactory.cs +++ b/src/Persistence/Wolverine.Marten/Publishing/OutboxedSessionFactory.cs @@ -16,7 +16,7 @@ public OutboxedSessionFactory(IWolverineRuntime runtime, T store) : base(new Ses _store = store; _factory = this; - MessageStore = runtime.AncillaryStores.OfType>().Single(); + MessageStore = runtime.FindAncillaryStoreForMarkerType(typeof(T)); } public IQuerySession QuerySession() diff --git a/src/Persistence/Wolverine.Marten/WolverineOptionsMartenExtensions.cs b/src/Persistence/Wolverine.Marten/WolverineOptionsMartenExtensions.cs index 413c06c56..578220596 100644 --- a/src/Persistence/Wolverine.Marten/WolverineOptionsMartenExtensions.cs +++ b/src/Persistence/Wolverine.Marten/WolverineOptionsMartenExtensions.cs @@ -185,7 +185,7 @@ internal static IMessageStore BuildMultiTenantedMessageDatabase( { SchemaName = schemaName, AutoCreate = autoCreate ?? store.Options.AutoCreateSchemaObjects, - IsMain = true, + Role = MessageStoreRole.Main, CommandQueuesEnabled = true, DataSource = masterDataSource ?? NpgsqlDataSource.Create(masterDatabaseConnectionString) }; @@ -217,7 +217,7 @@ internal static IMessageStore BuildSinglePostgresqlMessageStore( { SchemaName = schemaName, AutoCreate = autoCreate ?? store.Options.AutoCreateSchemaObjects, - IsMain = true, + Role = MessageStoreRole.Main, ScheduledJobLockId = $"{schemaName ?? "public"}:scheduled-jobs".GetDeterministicHashCode() }; diff --git a/src/Persistence/Wolverine.Postgresql/PostgresqlBackedPersistence.cs b/src/Persistence/Wolverine.Postgresql/PostgresqlBackedPersistence.cs index a0e98c126..9317f86e3 100644 --- a/src/Persistence/Wolverine.Postgresql/PostgresqlBackedPersistence.cs +++ b/src/Persistence/Wolverine.Postgresql/PostgresqlBackedPersistence.cs @@ -190,32 +190,41 @@ public IMessageStore BuildMessageStore(IWolverineRuntime runtime) var mainSource = DataSource ?? NpgsqlDataSource.Create(ConnectionString); var logger = runtime.LoggerFactory.CreateLogger(); - - var defaultStore = new PostgresqlMessageStore(settings, runtime.DurabilitySettings, mainSource, - logger, sagaTables); if (UseMasterTableTenancy) { + var defaultStore = new PostgresqlMessageStore(settings, runtime.DurabilitySettings, mainSource, + logger, sagaTables); + ConnectionStringTenancy = new MasterTenantSource(defaultStore, runtime.Options); return new MultiTenantedMessageStore(defaultStore, runtime, - new PostgresqlTenantedMessageStore(runtime, this, sagaTables)); + new PostgresqlTenantedMessageStore(runtime, this, sagaTables){}); } - else if (ConnectionStringTenancy != null || DataSourceTenancy != null) + + if (ConnectionStringTenancy != null || DataSourceTenancy != null) { + var defaultStore = new PostgresqlMessageStore(settings, runtime.DurabilitySettings, mainSource, + logger, sagaTables); + return new MultiTenantedMessageStore(defaultStore, runtime, new PostgresqlTenantedMessageStore(runtime, this, sagaTables)); } + + settings.Role = Role; - return defaultStore; + return new PostgresqlMessageStore(settings, runtime.DurabilitySettings, mainSource, + logger, sagaTables); } + + public MessageStoreRole Role { get; set; } = MessageStoreRole.Main; private DatabaseSettings buildMainDatabaseSettings() { var settings = new DatabaseSettings { CommandQueuesEnabled = CommandQueuesEnabled, - IsMain = true, + Role = MessageStoreRole.Main, ConnectionString = ConnectionString, DataSource = DataSource, ScheduledJobLockId = ScheduledJobLockId, diff --git a/src/Persistence/Wolverine.Postgresql/PostgresqlConfigurationExtensions.cs b/src/Persistence/Wolverine.Postgresql/PostgresqlConfigurationExtensions.cs index df7b24905..d2fdde1f7 100644 --- a/src/Persistence/Wolverine.Postgresql/PostgresqlConfigurationExtensions.cs +++ b/src/Persistence/Wolverine.Postgresql/PostgresqlConfigurationExtensions.cs @@ -2,6 +2,7 @@ using Npgsql; using JasperFx.Core.Reflection; using Wolverine.Configuration; +using Wolverine.Persistence.Durability; using Wolverine.Postgresql.Transport; @@ -34,13 +35,15 @@ internal static void AssertValidSchemaName(this string schemaName) /// /// /// Optional schema name for the Wolverine envelope storage + /// Default is Main. Use this to mark some stores as Ancillary to disambiguate the main storage for Wolverine public static IPostgresqlBackedPersistence PersistMessagesWithPostgresql(this WolverineOptions options, string connectionString, - string? schemaName = null) + string? schemaName = null, MessageStoreRole role = MessageStoreRole.Main) { var persistence = new PostgresqlBackedPersistence(options.Durability, options) { ConnectionString = connectionString, - AlreadyIncluded = true + AlreadyIncluded = true, + Role = role }; if (schemaName.IsNotEmpty()) @@ -60,12 +63,14 @@ public static IPostgresqlBackedPersistence PersistMessagesWithPostgresql(this Wo /// /// /// Optional schema name for the Wolverine envelope storage + /// Default is Main. Use this to mark some stores as Ancillary to disambiguate the main storage for Wolverine public static IPostgresqlBackedPersistence PersistMessagesWithPostgresql(this WolverineOptions options, NpgsqlDataSource dataSource, - string? schemaName = null) + string? schemaName = null, MessageStoreRole role = MessageStoreRole.Main) { var persistence = new PostgresqlBackedPersistence(options.Durability, options) { - DataSource = dataSource + DataSource = dataSource, + Role = role }; if (schemaName.IsNotEmpty()) diff --git a/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs b/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs index 5a2b11798..bc83cb0c7 100644 --- a/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs +++ b/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs @@ -480,7 +480,7 @@ public override IEnumerable AllObjects() yield return table; } - if (_settings.IsMain) + if (_settings.Role == MessageStoreRole.Main) { var nodeTable = new Table(new DbObjectName(SchemaName, DatabaseConstants.NodeTableName)); nodeTable.AddColumn("id").AsPrimaryKey(); diff --git a/src/Persistence/Wolverine.Postgresql/PostgresqlTenantedMessageStore.cs b/src/Persistence/Wolverine.Postgresql/PostgresqlTenantedMessageStore.cs index 5df2a92ee..47342e26f 100644 --- a/src/Persistence/Wolverine.Postgresql/PostgresqlTenantedMessageStore.cs +++ b/src/Persistence/Wolverine.Postgresql/PostgresqlTenantedMessageStore.cs @@ -12,7 +12,7 @@ namespace Wolverine.Postgresql; -internal class PostgresqlTenantedMessageStore : ITenantedMessageSource, IMessageDatabaseSource +internal class PostgresqlTenantedMessageStore : ITenantedMessageSource { private readonly PostgresqlBackedPersistence _persistence; private readonly SagaTableDefinition[] _sagaTables; @@ -27,9 +27,8 @@ public PostgresqlTenantedMessageStore(IWolverineRuntime runtime, PostgresqlBacke _runtime = runtime; } - public ITenantedSource DataSource { get; set; } - - public DatabaseCardinality Cardinality => DataSource.Cardinality; + public DatabaseCardinality Cardinality => _persistence.DataSourceTenancy?.Cardinality ?? + _persistence.ConnectionStringTenancy.Cardinality; public async ValueTask FindAsync(string tenantId) { if (_stores.TryFind(tenantId, out var store)) @@ -40,12 +39,12 @@ public async ValueTask FindAsync(string tenantId) if (_persistence.DataSourceTenancy != null) { var source = await _persistence.DataSourceTenancy.FindAsync(tenantId); - store = buildStoreForDataSource(source); + store = buildTenantStoreForDataSource(source); } else { var connectionString = await _persistence.ConnectionStringTenancy.FindAsync(tenantId); - store = buildStoreForConnectionString(connectionString); + store = buildTenantStoreForConnectionString(connectionString); } if (_runtime.Options.AutoBuildMessageStorageOnStartup != AutoCreate.None) @@ -57,7 +56,7 @@ public async ValueTask FindAsync(string tenantId) return store; } - private PostgresqlMessageStore buildStoreForConnectionString(string connectionString) + private PostgresqlMessageStore buildTenantStoreForConnectionString(string connectionString) { PostgresqlMessageStore store; // TODO -- do some idempotency so that you don't build two or more stores for the same tenant id @@ -69,7 +68,7 @@ private PostgresqlMessageStore buildStoreForConnectionString(string connectionSt // TODO -- set the AutoCreate here DataSource = npgsqlDataSource, ConnectionString = connectionString, - IsMain = false, + Role = MessageStoreRole.Tenant, ScheduledJobLockId = _persistence.ScheduledJobLockId, SchemaName = _persistence.EnvelopeStorageSchemaName }; @@ -79,7 +78,7 @@ private PostgresqlMessageStore buildStoreForConnectionString(string connectionSt return store; } - private PostgresqlMessageStore buildStoreForDataSource(NpgsqlDataSource source) + private PostgresqlMessageStore buildTenantStoreForDataSource(NpgsqlDataSource source) { PostgresqlMessageStore store; // TODO -- do some idempotency so that you don't build two or more stores for the same tenant id @@ -89,7 +88,7 @@ private PostgresqlMessageStore buildStoreForDataSource(NpgsqlDataSource source) CommandQueuesEnabled = false, // TODO -- set the AutoCreate here DataSource = source, - IsMain = false, + Role = MessageStoreRole.Tenant, ScheduledJobLockId = _persistence.ScheduledJobLockId, SchemaName = _persistence.EnvelopeStorageSchemaName }; @@ -110,7 +109,7 @@ public async Task RefreshAsync() // TODO -- some idempotency if (!_stores.Contains(assignment.TenantId)) { - var store = buildStoreForConnectionString(assignment.Value); + var store = buildTenantStoreForConnectionString(assignment.Value); if (_runtime.Options.AutoBuildMessageStorageOnStartup != AutoCreate.None) { @@ -130,7 +129,7 @@ public async Task RefreshAsync() // TODO -- some idempotency if (!_stores.Contains(assignment.TenantId)) { - var store = buildStoreForDataSource(assignment.Value); + var store = buildTenantStoreForDataSource(assignment.Value); if (_runtime.Options.AutoBuildMessageStorageOnStartup != AutoCreate.None) { diff --git a/src/Persistence/Wolverine.Postgresql/Transport/PostgresqlTransport.cs b/src/Persistence/Wolverine.Postgresql/Transport/PostgresqlTransport.cs index 505f3511d..dc9e1f37b 100644 --- a/src/Persistence/Wolverine.Postgresql/Transport/PostgresqlTransport.cs +++ b/src/Persistence/Wolverine.Postgresql/Transport/PostgresqlTransport.cs @@ -37,7 +37,14 @@ public async ValueTask ConfigureAsync(IWolverineRuntime runtime) { // This is important, let the Postgres queues get built automatically AutoProvision = AutoProvision || runtime.Options.AutoBuildMessageStorageOnStartup != AutoCreate.None; - if (runtime.Storage is PostgresqlMessageStore store) + var stores = await runtime.Stores.FindAllAsync(); + if (!stores.Any()) + { + throw new InvalidOperationException( + $"The PostgreSQL transport is configured for usage, but the envelope storage is incompatible: {runtime.Storage}"); + } + + foreach (var store in stores) { foreach (var queue in Queues) { @@ -46,38 +53,22 @@ public async ValueTask ConfigureAsync(IWolverineRuntime runtime) } MessageStorageSchemaName = store.SchemaName; + } - Store = store; + if (runtime.Storage is PostgresqlMessageStore s) + { + Store = s; } - else if (runtime.Storage is MultiTenantedMessageStore tenants) + else if (runtime.Storage is MultiTenantedMessageStore mt) { - Store = tenants.Main as PostgresqlMessageStore; - - if (tenants.Source is IMessageDatabaseSource source) + if (mt.Main is PostgresqlMessageStore p) { - await source.ConfigureDatabaseAsync(messageStore => - { - if (messageStore is PostgresqlMessageStore s) - { - MessageStorageSchemaName = s.SchemaName; - - foreach (var queue in Queues) - { - s.AddTable(queue.QueueTable); - s.AddTable(queue.ScheduledTable); - } - } - else - { - throw new InvalidOperationException( - "The PostgreSQL backed transport can only be used with PostgreSQL is the active envelope storage mechanism"); - } - - return new ValueTask(); - }); + Store = p; + Databases = mt; } } - else + + if (Store == null) { throw new InvalidOperationException( $"The PostgreSQL transport is configured for usage, but the envelope storage is incompatible: {runtime.Storage}"); @@ -108,13 +99,15 @@ public override async ValueTask ConnectAsync(IWolverineRuntime runtime) { Store = store; } - else if (runtime.Storage is MultiTenantedMessageStore tenants) + else if (runtime.Storage is MultiTenantedMessageStore mt && mt.Main is PostgresqlMessageStore s) { - Store = tenants.Main as PostgresqlMessageStore ?? - throw new ArgumentOutOfRangeException( - "The PostgreSQL transport can only be used if PostgreSQL is the backing message store"); - - Databases = tenants; + Store = s; + Databases = mt; + } + else + { + throw new ArgumentOutOfRangeException( + "The PostgreSQL transport can only be used if PostgreSQL is the backing message store"); } // This is de facto a little environment test diff --git a/src/Persistence/Wolverine.RDBMS/DatabaseSettings.cs b/src/Persistence/Wolverine.RDBMS/DatabaseSettings.cs index 955de2a60..3a7b1d009 100644 --- a/src/Persistence/Wolverine.RDBMS/DatabaseSettings.cs +++ b/src/Persistence/Wolverine.RDBMS/DatabaseSettings.cs @@ -2,6 +2,7 @@ using JasperFx; using JasperFx.MultiTenancy; using Weasel.Core; +using Wolverine.Persistence.Durability; namespace Wolverine.RDBMS; @@ -16,7 +17,7 @@ public class DatabaseSettings /// /// Is this database the master database for node storage and any kind of command queueing? /// - public bool IsMain { get; set; } + public MessageStoreRole Role { get; set; } = MessageStoreRole.Ancillary; /// /// If the main database, add a tenant lookup table diff --git a/src/Persistence/Wolverine.RDBMS/Durability/DeleteOldNodeEventRecords.cs b/src/Persistence/Wolverine.RDBMS/Durability/DeleteOldNodeEventRecords.cs index aec350ca8..eb4525d57 100644 --- a/src/Persistence/Wolverine.RDBMS/Durability/DeleteOldNodeEventRecords.cs +++ b/src/Persistence/Wolverine.RDBMS/Durability/DeleteOldNodeEventRecords.cs @@ -1,4 +1,5 @@ using System.Data.Common; +using Wolverine.Persistence.Durability; using Wolverine.RDBMS.Polling; using Wolverine.Runtime.Agents; using DbCommandBuilder = Weasel.Core.DbCommandBuilder; @@ -13,7 +14,7 @@ internal class DeleteOldNodeEventRecords : IDatabaseOperation, IDoNotReturnData public DeleteOldNodeEventRecords(IMessageDatabase database, DurabilitySettings settings) { _database = database ?? throw new ArgumentNullException(nameof(database)); - if (!_database.Settings.IsMain) + if (_database.Settings.Role != MessageStoreRole.Main) { throw new ArgumentOutOfRangeException(nameof(database), "This operation is only valid on 'Main' databases"); } diff --git a/src/Persistence/Wolverine.RDBMS/DurabilityAgent.cs b/src/Persistence/Wolverine.RDBMS/DurabilityAgent.cs index a06ca559f..e3814ec90 100644 --- a/src/Persistence/Wolverine.RDBMS/DurabilityAgent.cs +++ b/src/Persistence/Wolverine.RDBMS/DurabilityAgent.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging; using Weasel.Core; using Wolverine.Persistence; +using Wolverine.Persistence.Durability; using Wolverine.RDBMS.Durability; using Wolverine.RDBMS.Polling; using Wolverine.Runtime; @@ -158,7 +159,7 @@ private bool isTimeToPruneNodeEventRecords() private IDatabaseOperation[] buildOperationBatch() { - if (_database.Settings.IsMain && isTimeToPruneNodeEventRecords()) + if (_database.Settings.Role == MessageStoreRole.Main && isTimeToPruneNodeEventRecords()) { return [ diff --git a/src/Persistence/Wolverine.RDBMS/IMessageDatabase.cs b/src/Persistence/Wolverine.RDBMS/IMessageDatabase.cs index dbc883a13..e5d9b0a07 100644 --- a/src/Persistence/Wolverine.RDBMS/IMessageDatabase.cs +++ b/src/Persistence/Wolverine.RDBMS/IMessageDatabase.cs @@ -51,8 +51,6 @@ public static bool TryFindMessageDatabase(this MessageContext context, out IMess public interface IMessageDatabase : IMessageStoreWithAgentSupport, ITenantDatabaseRegistry { public DatabaseSettings Settings {get; } - - bool IsMain { get; } string SchemaName { get; set; } diff --git a/src/Persistence/Wolverine.RDBMS/MessageDatabase.Admin.cs b/src/Persistence/Wolverine.RDBMS/MessageDatabase.Admin.cs index 8e407b999..e69af1d26 100644 --- a/src/Persistence/Wolverine.RDBMS/MessageDatabase.Admin.cs +++ b/src/Persistence/Wolverine.RDBMS/MessageDatabase.Admin.cs @@ -2,6 +2,7 @@ using JasperFx.Core; using Weasel.Core; using Wolverine.Logging; +using Wolverine.Persistence.Durability; namespace Wolverine.RDBMS; @@ -134,7 +135,7 @@ await tx.CreateCommand($"delete from {SchemaName}.{DatabaseConstants.IncomingTab await tx.CreateCommand($"delete from {SchemaName}.{DatabaseConstants.DeadLetterTable}") .ExecuteNonQueryAsync(_cancellation); - if (_settings.IsMain) + if (_settings.Role == MessageStoreRole.Main) { await tx.CreateCommand($"delete from {SchemaName}.{DatabaseConstants.AgentRestrictionsTableName}") .ExecuteNonQueryAsync(_cancellation); diff --git a/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs b/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs index c858042d2..48fc635a0 100644 --- a/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs +++ b/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs @@ -7,7 +7,7 @@ namespace Wolverine.RDBMS; -public abstract partial class MessageDatabase : IDeadLetterAdminService +public abstract partial class MessageDatabase { public async Task> SummarizeAllAsync(string serviceName, TimeRange range, CancellationToken token) diff --git a/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs b/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs index 5fb7b7728..522db186b 100644 --- a/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs +++ b/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs @@ -70,7 +70,9 @@ protected MessageDatabase(DatabaseSettings databaseSettings, DbDataSource dataSo _schemaName }; - if (databaseSettings.IsMain) + Role = databaseSettings.Role; + + if (databaseSettings.Role == MessageStoreRole.Main) { SubjectUri = new Uri("wolverine://messages/main"); } @@ -78,6 +80,8 @@ protected MessageDatabase(DatabaseSettings databaseSettings, DbDataSource dataSo Uri = new Uri($"{PersistenceConstants.AgentScheme}://{parts.Where(x => x.IsNotEmpty()).Join("/")}"); } + public MessageStoreRole Role { get; private set; } + public override string ToString() { return $"{Uri} ({Name})"; @@ -104,9 +108,12 @@ public IAgent BuildAgent(IWolverineRuntime runtime) public DurabilitySettings Durability { get; } - public bool IsMain => Settings.IsMain; - public string Name { get; set; } = TransportConstants.Default; + public void PromoteToMain(IWolverineRuntime runtime) + { + Role = MessageStoreRole.Main; + Initialize(runtime); + } public DatabaseSettings Settings => _settings; @@ -153,7 +160,7 @@ public void Initialize(IWolverineRuntime runtime) _batcher = new DatabaseBatcher(this, runtime, runtime.Options.Durability.Cancellation); - if (Settings.IsMain && runtime.Options.Transports.NodeControlEndpoint == null && runtime.Options.Durability.Mode == DurabilityMode.Balanced) + if (Role == MessageStoreRole.Main && runtime.Options.Transports.NodeControlEndpoint == null && runtime.Options.Durability.Mode == DurabilityMode.Balanced) { var transport = new DatabaseControlTransport(this, runtime.Options); runtime.Options.Transports.Add(transport); @@ -264,11 +271,6 @@ public IAgent StartScheduledJobs(IWolverineRuntime runtime) return agent; } - public IAgentFamily? BuildAgentFamily(IWolverineRuntime runtime) - { - return new DurabilityAgentFamily(runtime); - } - public async ValueTask> EnrollAndFetchSagaStorage(MessageContext context) where TSaga : Saga { var conn = CreateConnection(); diff --git a/src/Persistence/Wolverine.RDBMS/MultiTenancy/IMessageDatabaseSource.cs b/src/Persistence/Wolverine.RDBMS/MultiTenancy/IMessageDatabaseSource.cs deleted file mode 100644 index 660542cb6..000000000 --- a/src/Persistence/Wolverine.RDBMS/MultiTenancy/IMessageDatabaseSource.cs +++ /dev/null @@ -1,15 +0,0 @@ -using Wolverine.Persistence.Durability; - -namespace Wolverine.RDBMS.MultiTenancy; - -/// -/// Source of known tenant databases -/// -public interface IMessageDatabaseSource : ITenantedMessageSource -{ - /// - /// Add extra configuration to every actively used tenant database - /// - /// - public ValueTask ConfigureDatabaseAsync(Func configureDatabase); -} \ No newline at end of file diff --git a/src/Persistence/Wolverine.RDBMS/MultiTenancy/MessageDatabaseDiscovery.cs b/src/Persistence/Wolverine.RDBMS/MultiTenancy/MessageDatabaseDiscovery.cs index 45e40d7cc..e4e49f3f9 100644 --- a/src/Persistence/Wolverine.RDBMS/MultiTenancy/MessageDatabaseDiscovery.cs +++ b/src/Persistence/Wolverine.RDBMS/MultiTenancy/MessageDatabaseDiscovery.cs @@ -7,11 +7,12 @@ namespace Wolverine.RDBMS.MultiTenancy; public class MessageDatabaseDiscovery : IDatabaseSource { - private readonly IWolverineRuntime _runtime; + private readonly WolverineRuntime _runtime; public MessageDatabaseDiscovery(IWolverineRuntime runtime) { - _runtime = runtime; + // Yeah, I know, this is awful. But also working just fine + _runtime = (WolverineRuntime)runtime; } public async ValueTask DescribeDatabasesAsync(CancellationToken token) @@ -41,40 +42,9 @@ public async ValueTask DescribeDatabasesAsync(CancellationToken t return usage; } - public async ValueTask> BuildDatabases() + public ValueTask> BuildDatabases() { - var list = new List(); - - if (_runtime.Storage is IMessageDatabase database) - { - list.Add(database); - } - else if (_runtime.Storage is MultiTenantedMessageStore tenants) - { - await tenants.InitializeAsync(_runtime); - list.AddRange(tenants.ActiveDatabases().OfType()); - } - - foreach (var ancillaryStore in _runtime.AncillaryStores) - { - if (ancillaryStore is IMessageDatabase db) - { - list.Add(db); - } - else if (ancillaryStore is MultiTenantedMessageStore tenants) - { - await tenants.InitializeAsync(_runtime); - list.AddRange(tenants.ActiveDatabases().OfType()); - } - } - - var groups = list.GroupBy(x => x.Uri); - return groups.Select(group => - { - // It's important to use a "main" version because that has extra tables - var master = group.FirstOrDefault(x => x.IsMain); - return master ?? group.First(); - }).OfType().ToList(); + return _runtime.Stores.FindAllAsync(); } public DatabaseCardinality Cardinality diff --git a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.DeadLetters.cs b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.DeadLetters.cs index e207438d9..033337a55 100644 --- a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.DeadLetters.cs +++ b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.DeadLetters.cs @@ -136,4 +136,24 @@ public async Task DeleteDeadLetterEnvelopesAsync(Guid[] ids, string? tenantId = await session.SaveChangesAsync(); } + + public Task> SummarizeAllAsync(string serviceName, TimeRange range, CancellationToken token) + { + throw new NotImplementedException(); + } + + public Task QueryAsync(DeadLetterEnvelopeQuery query, CancellationToken token) + { + throw new NotImplementedException(); + } + + public Task DiscardAsync(DeadLetterEnvelopeQuery query, CancellationToken token) + { + throw new NotImplementedException(); + } + + public Task ReplayAsync(DeadLetterEnvelopeQuery query, CancellationToken token) + { + throw new NotImplementedException(); + } } \ No newline at end of file diff --git a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.cs b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.cs index f7a03046f..a5d7716d6 100644 --- a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.cs +++ b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.cs @@ -28,6 +28,13 @@ public RavenDbMessageStore(IDocumentStore store, WolverineOptions options) _scheduledLockId = "wolverine/scheduled"; } + public MessageStoreRole Role { get; set; } = MessageStoreRole.Main; + + public void PromoteToMain(IWolverineRuntime runtime) + { + Role = MessageStoreRole.Main; + } + public string Name => _store.Identifier; public Uri Uri => new("ravendb://durability"); diff --git a/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs b/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs index 49cf127d5..424897dd9 100644 --- a/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs +++ b/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs @@ -10,6 +10,7 @@ using Weasel.Core; using Weasel.SqlServer; using Wolverine.Logging; +using Wolverine.Persistence.Durability; using Wolverine.Persistence.Durability.DeadLetterManagement; using Wolverine.RDBMS; using Wolverine.RDBMS.Sagas; @@ -427,7 +428,7 @@ public override IEnumerable AllObjects() yield return table; } - if (_settings.IsMain) + if (_settings.Role == MessageStoreRole.Main) { var nodeTable = new Table(new DbObjectName(SchemaName, DatabaseConstants.NodeTableName)); nodeTable.AddColumn("id").AsPrimaryKey(); diff --git a/src/Persistence/Wolverine.SqlServer/SqlServerBackedPersistence.cs b/src/Persistence/Wolverine.SqlServer/SqlServerBackedPersistence.cs index 2e9937a9d..09e5028d0 100644 --- a/src/Persistence/Wolverine.SqlServer/SqlServerBackedPersistence.cs +++ b/src/Persistence/Wolverine.SqlServer/SqlServerBackedPersistence.cs @@ -170,38 +170,44 @@ public IMessageStore BuildMessageStore(IWolverineRuntime runtime) var logger = runtime.LoggerFactory.CreateLogger(); - var defaultStore = new SqlServerMessageStore(settings, runtime.DurabilitySettings, - logger, sagaTables); - if (UseMasterTableTenancy) { + var defaultStore = new SqlServerMessageStore(settings, runtime.DurabilitySettings, + logger, sagaTables); + ConnectionStringTenancy = new MasterTenantSource(defaultStore, runtime.Options); return new MultiTenantedMessageStore(defaultStore, runtime, - new SqlServerTenantedMessageStore(runtime, this, sagaTables)); + new SqlServerTenantedMessageStore(runtime, this, sagaTables){DataSource = ConnectionStringTenancy}); } - else if (ConnectionStringTenancy != null) + + if (ConnectionStringTenancy != null) { + var defaultStore = new SqlServerMessageStore(settings, runtime.DurabilitySettings, + logger, sagaTables); + return new MultiTenantedMessageStore(defaultStore, runtime, - new SqlServerTenantedMessageStore(runtime, this, sagaTables)); + new SqlServerTenantedMessageStore(runtime, this, sagaTables){DataSource = ConnectionStringTenancy}); } - return defaultStore; + settings.Role = Role; + + return new SqlServerMessageStore(settings, runtime.DurabilitySettings, + logger, sagaTables); } private DatabaseSettings buildMainDatabaseSettings() { - var settings = new DatabaseSettings + return new DatabaseSettings { CommandQueuesEnabled = CommandQueuesEnabled, - IsMain = true, + Role = MessageStoreRole.Main, ConnectionString = ConnectionString, ScheduledJobLockId = ScheduledJobLockId, SchemaName = EnvelopeStorageSchemaName, AddTenantLookupTable = UseMasterTableTenancy, TenantConnections = TenantConnections }; - return settings; } private List> _transportConfigurations = new(); @@ -265,4 +271,6 @@ ISqlServerBackedPersistence ISqlServerBackedPersistence.UseMasterTableTenancy(Ac /// This is any default connection strings by tenant that should be loaded at start up time /// public StaticConnectionStringSource? TenantConnections { get; set; } + + public MessageStoreRole Role { get; set; } = MessageStoreRole.Main; } \ No newline at end of file diff --git a/src/Persistence/Wolverine.SqlServer/SqlServerConfigurationExtensions.cs b/src/Persistence/Wolverine.SqlServer/SqlServerConfigurationExtensions.cs index 624091469..6ea759f05 100644 --- a/src/Persistence/Wolverine.SqlServer/SqlServerConfigurationExtensions.cs +++ b/src/Persistence/Wolverine.SqlServer/SqlServerConfigurationExtensions.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.DependencyInjection; using Weasel.Core.Migrations; using Wolverine.Configuration; +using Wolverine.Persistence.Durability; using Wolverine.RDBMS; using Wolverine.SqlServer.Transport; @@ -17,11 +18,12 @@ public static class SqlServerConfigurationExtensions /// /// Potentially override the schema name for Wolverine envelope storage. Default is to use WolverineOptions.Durability.MessageStorageSchemaName ?? "dbo" public static ISqlServerBackedPersistence PersistMessagesWithSqlServer(this WolverineOptions options, string connectionString, - string? schema = null) + string? schema = null, MessageStoreRole role = MessageStoreRole.Main) { var extension = new SqlServerBackedPersistence { - ConnectionString = connectionString + ConnectionString = connectionString, + Role = role }; if (schema.IsNotEmpty()) diff --git a/src/Persistence/Wolverine.SqlServer/SqlServerTenantedMessageStore.cs b/src/Persistence/Wolverine.SqlServer/SqlServerTenantedMessageStore.cs index 455380e4b..c7a08f6bd 100644 --- a/src/Persistence/Wolverine.SqlServer/SqlServerTenantedMessageStore.cs +++ b/src/Persistence/Wolverine.SqlServer/SqlServerTenantedMessageStore.cs @@ -12,7 +12,7 @@ namespace Wolverine.SqlServer; -internal class SqlServerTenantedMessageStore : ITenantedMessageSource, IMessageDatabaseSource +internal class SqlServerTenantedMessageStore : ITenantedMessageSource { private ImHashMap _values = ImHashMap.Empty; private readonly SqlServerBackedPersistence _persistence; @@ -44,7 +44,7 @@ public async ValueTask FindAsync(string tenantId) } var connectionString = await _persistence.ConnectionStringTenancy!.FindAsync(tenantId); - store = buildStoreForConnectionString(connectionString); + store = buildTenantStoreForConnectionString(connectionString); if (_runtime.Options.AutoBuildMessageStorageOnStartup != AutoCreate.None) { @@ -55,7 +55,7 @@ public async ValueTask FindAsync(string tenantId) return store; } - private SqlServerMessageStore buildStoreForConnectionString(string connectionString) + private SqlServerMessageStore buildTenantStoreForConnectionString(string connectionString) { SqlServerMessageStore store; @@ -66,7 +66,7 @@ private SqlServerMessageStore buildStoreForConnectionString(string connectionStr CommandQueuesEnabled = false, // TODO -- set the AutoCreate here ConnectionString = connectionString, - IsMain = false, + Role = MessageStoreRole.Tenant, ScheduledJobLockId = _persistence.ScheduledJobLockId, SchemaName = _persistence.EnvelopeStorageSchemaName }; @@ -85,7 +85,7 @@ public async Task RefreshAsync() // TODO -- some idempotency if (!_stores.Contains(assignment.TenantId)) { - var store = buildStoreForConnectionString(assignment.Value); + var store = buildTenantStoreForConnectionString(assignment.Value); if (_runtime.Options.AutoBuildMessageStorageOnStartup != AutoCreate.None) { diff --git a/src/Persistence/Wolverine.SqlServer/Transport/SqlServerTransport.cs b/src/Persistence/Wolverine.SqlServer/Transport/SqlServerTransport.cs index 848a12e2e..841caae44 100644 --- a/src/Persistence/Wolverine.SqlServer/Transport/SqlServerTransport.cs +++ b/src/Persistence/Wolverine.SqlServer/Transport/SqlServerTransport.cs @@ -72,8 +72,8 @@ protected override SqlServerQueue findEndpointByUri(Uri uri) public override async ValueTask ConnectAsync(IWolverineRuntime runtime) { AutoProvision = AutoProvision || runtime.Options.AutoBuildMessageStorageOnStartup != AutoCreate.None; - - var storage = runtime.Storage as SqlServerMessageStore; + + var storage = await runtime.TryFindMainMessageStore(); Storage = storage ?? throw new InvalidOperationException( "The Sql Server Transport can only be used if the message persistence is also Sql Server backed"); diff --git a/src/Testing/CoreTests/Runtime/MockWolverineRuntime.cs b/src/Testing/CoreTests/Runtime/MockWolverineRuntime.cs index 054da9852..59b4620a0 100644 --- a/src/Testing/CoreTests/Runtime/MockWolverineRuntime.cs +++ b/src/Testing/CoreTests/Runtime/MockWolverineRuntime.cs @@ -7,6 +7,7 @@ using Wolverine.ComplianceTests; using Wolverine.Configuration; using Wolverine.Logging; +using Wolverine.Persistence; using Wolverine.Persistence.Durability; using Wolverine.Runtime; using Wolverine.Runtime.Agents; @@ -97,7 +98,18 @@ public IMessageRouter RoutingFor(Type messageType) public IMessageStore Storage { get; } = Substitute.For(); public ILogger Logger { get; } = Substitute.For(); - public IReadOnlyList AncillaryStores { get; } = new List(); + + public MessageStoreCollection Stores => throw new NotSupportedException(); + + public Task TryFindMainMessageStore() where T : class + { + throw new NotImplementedException(); + } + + public IAncillaryMessageStore FindAncillaryStoreForMarkerType(Type markerType) + { + throw new NotImplementedException(); + } public void ScheduleLocalExecutionInMemory(DateTimeOffset executionTime, Envelope envelope) { diff --git a/src/Testing/Wolverine.ComplianceTests/DeadLetterAdminCompliance.cs b/src/Testing/Wolverine.ComplianceTests/DeadLetterAdminCompliance.cs index 62eae5858..c76b53809 100644 --- a/src/Testing/Wolverine.ComplianceTests/DeadLetterAdminCompliance.cs +++ b/src/Testing/Wolverine.ComplianceTests/DeadLetterAdminCompliance.cs @@ -29,7 +29,7 @@ public abstract class DeadLetterAdminCompliance : IAsyncLifetime private DateTimeOffset FourHoursAgo; private DateTimeOffset SevenHoursAgo; private DateTimeOffset SixHoursAgo; - protected IDeadLetterAdminService theDeadLetters; + protected IDeadLetters theDeadLetters; protected EnvelopeGenerator theGenerator; protected IMessageStore thePersistence; private IReadOnlyList theSummaries; @@ -48,7 +48,7 @@ public async Task InitializeAsync() await theHost.ResetResourceState(); thePersistence = theHost.Services.GetRequiredService(); - theDeadLetters = (IDeadLetterAdminService)thePersistence.DeadLetters; + theDeadLetters = thePersistence.DeadLetters; theGenerator = new EnvelopeGenerator(); theGenerator.MessageSource = BuildRandomMessage; diff --git a/src/Wolverine/Configuration/EndpointCollection.cs b/src/Wolverine/Configuration/EndpointCollection.cs index db07ebd43..4cb70eb7b 100644 --- a/src/Wolverine/Configuration/EndpointCollection.cs +++ b/src/Wolverine/Configuration/EndpointCollection.cs @@ -310,12 +310,18 @@ private ISendingAgent buildSendingAgent(ISender sender, Endpoint endpoint) return a; } + IMessageStore store = _runtime.Storage; + if (_runtime.Stores.TryFindMultiTenantedForMainStore(store, out var tenanted)) + { + store = tenanted; + } + switch (endpoint.Mode) { case EndpointMode.Durable: return new DurableSendingAgent(sender, _options.Durability, _runtime.LoggerFactory.CreateLogger(), _runtime.MessageTracking, - _runtime.Storage, endpoint); + store, endpoint); case EndpointMode.BufferedInMemory: return new BufferedSendingAgent(_runtime.LoggerFactory.CreateLogger(), diff --git a/src/Wolverine/Envelope.cs b/src/Wolverine/Envelope.cs index 0625c2083..401c25510 100644 --- a/src/Wolverine/Envelope.cs +++ b/src/Wolverine/Envelope.cs @@ -219,7 +219,7 @@ public object? Message /// /// The name of the service that sent this envelope /// - public string? Source { get; internal set; } + public string? Source { get; set; } /// /// Message type alias for the contents of this Envelope diff --git a/src/Wolverine/HostBuilderExtensions.cs b/src/Wolverine/HostBuilderExtensions.cs index ee1635321..1ccae6629 100644 --- a/src/Wolverine/HostBuilderExtensions.cs +++ b/src/Wolverine/HostBuilderExtensions.cs @@ -16,6 +16,7 @@ using JasperFx.Resources; using JasperFx.RuntimeCompiler; using Wolverine.Configuration; +using Wolverine.Persistence; using Wolverine.Persistence.Durability; using Wolverine.Persistence.Sagas; using Wolverine.Runtime; @@ -95,9 +96,9 @@ internal static IServiceCollection AddWolverine(this IServiceCollection services } services.AddJasperFx(); - + services.AddSingleton(); services.AddSingleton(); - + services.AddSingleton(services); services.AddSingleton(); @@ -152,7 +153,6 @@ internal static IServiceCollection AddWolverine(this IServiceCollection services services.MessagingRootService(x => x.MessageTracking); - services.TryAddSingleton(); services.AddSingleton(); services.MessagingRootService(x => x.Pipeline); @@ -426,7 +426,6 @@ public static IServiceCollection DisableAllExternalWolverineTransports(this ISer /// public static IServiceCollection DisableAllWolverineMessagePersistence(this IServiceCollection services) { - services.AddSingleton(); services.AddSingleton(); return services; } diff --git a/src/Wolverine/Persistence/Durability/DeadLetterManagement/IDeadLetterAdminService.cs b/src/Wolverine/Persistence/Durability/DeadLetterManagement/IDeadLetterAdminService.cs deleted file mode 100644 index 8d622eb97..000000000 --- a/src/Wolverine/Persistence/Durability/DeadLetterManagement/IDeadLetterAdminService.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace Wolverine.Persistence.Durability.DeadLetterManagement; - -/// -/// This is the dead letter service that is meant for CritterWatch usage -/// -public interface IDeadLetterAdminService -{ - Task> SummarizeAllAsync(string serviceName, TimeRange range, - CancellationToken token); - - Task QueryAsync(DeadLetterEnvelopeQuery query, CancellationToken token); - - Task DiscardAsync(DeadLetterEnvelopeQuery query, CancellationToken token); - Task ReplayAsync(DeadLetterEnvelopeQuery query, CancellationToken token); - - Uri Uri { get; } -} \ No newline at end of file diff --git a/src/Wolverine/Persistence/Durability/DurabilityAgentFamily.cs b/src/Wolverine/Persistence/Durability/DurabilityAgentFamily.cs deleted file mode 100644 index 6dcdfbb1c..000000000 --- a/src/Wolverine/Persistence/Durability/DurabilityAgentFamily.cs +++ /dev/null @@ -1,141 +0,0 @@ -using Wolverine.Persistence.Durability.DeadLetterManagement; -using Wolverine.Runtime; -using Wolverine.Runtime.Agents; - -namespace Wolverine.Persistence.Durability; - -public class DurabilityAgentFamily : IAgentFamily -{ - private readonly Dictionary _storeWithAgents = new(); - private readonly List _tenantedStores = new(); - - public DurabilityAgentFamily(IWolverineRuntime runtime) - { - addStore(runtime.Storage); - foreach (var ancillaryStore in runtime.AncillaryStores) - { - addStore(ancillaryStore); - } - } - - private void addStore(IMessageStore store) - { - if (store is IMessageStoreWithAgentSupport agentSupport) - { - _storeWithAgents.TryAdd(agentSupport.Uri, agentSupport); - } - else if (store is MultiTenantedMessageStore tenantedMessageStore) - { - if (tenantedMessageStore.Main is IMessageStoreWithAgentSupport master) - { - _storeWithAgents.TryAdd(master.Uri, master); - } - - _tenantedStores.Add(tenantedMessageStore.Source); - } - } - - internal static async Task StartScheduledJobProcessing(IWolverineRuntime runtime) - { - var family = new DurabilityAgentFamily(runtime); - - // First, find all unique message stores - var stores = await family.findUniqueMessageStores(); - var agents = stores.Select(x => x.StartScheduledJobs(runtime)); - return new CompositeAgent(new Uri("internal://scheduledjobs"), agents); - } - - private async Task> findUniqueMessageStores() - { - var stores = new Dictionary(); - foreach (var agentSupport in _storeWithAgents) - { - stores.TryAdd(agentSupport.Key, agentSupport.Value); - } - - foreach (var tenantedStore in _tenantedStores) - { - if (!tenantedStore.AllActive().Any()) - { - await tenantedStore.RefreshAsync(); - } - - foreach (var store in tenantedStore.AllActive()) - { - stores.TryAdd(store.Uri, store); - } - } - - return stores.Values.ToList(); - } - - public string Scheme => PersistenceConstants.AgentScheme; - - public async ValueTask> AllKnownAgentsAsync() - { - foreach (var tenantedStore in _tenantedStores) - { - await tenantedStore.RefreshAsync(); - } - - var fromTenantedSources = _tenantedStores.SelectMany(x => - x.AllActive().OfType().Select(mas => mas.Uri)); - - return _storeWithAgents.Select(x => x.Value.Uri) - .Concat(fromTenantedSources) - .Distinct().ToList(); - - - } - - public async ValueTask BuildAgentAsync(Uri uri, IWolverineRuntime wolverineRuntime) - { - var store = await findStoreAsync(uri); - if (store == null) - { - throw new InvalidAgentException($"Unknown durability agent '{uri}'"); - } - - return store.BuildAgent(wolverineRuntime); - } - - private async ValueTask findStoreAsync(Uri uri) - { - if (_storeWithAgents.TryGetValue(uri, out var store)) - { - return store; - } - - // Do a first pass trying to find it w/o refreshing the multi-tenanted stores - foreach (var tenantedStore in _tenantedStores) - { - store = tenantedStore.AllActive().OfType() - .FirstOrDefault(x => x.Uri == uri); - - if (store != null) return store; - } - - // Do a 2nd pass, but this time force a refresh - foreach (var tenantedStore in _tenantedStores) - { - await tenantedStore.RefreshAsync(); - store = tenantedStore.AllActive().OfType() - .FirstOrDefault(x => x.Uri == uri); - - if (store != null) return store; - } - - return null; - } - - public ValueTask> SupportedAgentsAsync() - { - return AllKnownAgentsAsync(); - } - - public ValueTask EvaluateAssignmentsAsync(AssignmentGrid assignments) - { - assignments.DistributeEvenly(Scheme); - return ValueTask.CompletedTask; - } -} \ No newline at end of file diff --git a/src/Wolverine/Persistence/Durability/IDeadLetters.cs b/src/Wolverine/Persistence/Durability/IDeadLetters.cs index 80ed72e16..196dd6c34 100644 --- a/src/Wolverine/Persistence/Durability/IDeadLetters.cs +++ b/src/Wolverine/Persistence/Durability/IDeadLetters.cs @@ -18,6 +18,7 @@ public interface IDeadLetters /// /// Exception Type that should be marked. Default is any. /// Number of envelopes marked. + [Obsolete("Prefer ReplayAsync")] Task MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType = ""); /// @@ -26,6 +27,7 @@ public interface IDeadLetters /// /// /// Leaving tenantId null will query all tenants + [Obsolete("Prefer ReplayAsync")] Task MarkDeadLetterEnvelopesAsReplayableAsync(Guid[] ids, string? tenantId = null); /// @@ -33,5 +35,40 @@ public interface IDeadLetters /// /// /// Leaving tenantId null will query all tenants + [Obsolete("Prefer DiscardAsync")] Task DeleteDeadLetterEnvelopesAsync(Guid[] ids, string? tenantId = null); + + /// + /// Fetch a summary of the persisted dead letter queue envelopes + /// + /// + /// + /// + /// + Task> SummarizeAllAsync(string serviceName, TimeRange range, + CancellationToken token); + + /// + /// Query for detailed results of dead letter queued envelopes matching the specified query + /// + /// + /// + /// + Task QueryAsync(DeadLetterEnvelopeQuery query, CancellationToken token); + + /// + /// Deletes all dead letter envelopes matching the query + /// + /// + /// + /// + Task DiscardAsync(DeadLetterEnvelopeQuery query, CancellationToken token); + + /// + /// Marks all dead letter envelopes matching the query as replayable + /// + /// + /// + /// + Task ReplayAsync(DeadLetterEnvelopeQuery query, CancellationToken token); } \ No newline at end of file diff --git a/src/Wolverine/Persistence/Durability/IMessageStore.cs b/src/Wolverine/Persistence/Durability/IMessageStore.cs index 4e51111f8..d33614f06 100644 --- a/src/Wolverine/Persistence/Durability/IMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/IMessageStore.cs @@ -5,6 +5,31 @@ namespace Wolverine.Persistence.Durability; +public enum MessageStoreRole +{ + /// + /// Denotes that this message store is the main message store for the application and where + /// node information is stored + /// + Main, + + /// + /// Denotes that this message store is an additional message store for the application, but + /// does not store the node information + /// + Ancillary, + + /// + /// This message store is strictly for one or more tenants + /// + Tenant, + + /// + /// This message store is a multi-tenanted composite of other message stores + /// + Composite +} + public interface IMessageInbox { Task ScheduleExecutionAsync(Envelope envelope); @@ -56,6 +81,11 @@ public interface IMessageStoreWithAgentSupport : IMessageStore public interface IMessageStore : IAsyncDisposable { + /// + /// What is the role of this message store within the application? + /// + MessageStoreRole Role { get; } + /// /// Unique identifier for a message store in case of systems that use multiple message /// store databases. Must use the "messagedb" scheme, and reflect the database connection @@ -102,6 +132,8 @@ public interface IMessageStore : IAsyncDisposable /// Descriptive name for cases of multiple message stores /// string Name { get; } + + void PromoteToMain(IWolverineRuntime runtime); } public record IncomingCount(Uri Destination, int Count); diff --git a/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs b/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs index 0bd7cdc64..4933550c6 100644 --- a/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs @@ -48,6 +48,8 @@ public MultiTenantedMessageStore(IMessageStore main, IWolverineRuntime runtime, Main = main; } + public MessageStoreRole Role => MessageStoreRole.Composite; + public ITenantedMessageSource Source { get; } public Uri Uri => new($"{PersistenceConstants.AgentScheme}://multitenanted"); @@ -98,6 +100,26 @@ public async Task DeleteDeadLetterEnvelopesAsync(Guid[] ids, string? tenantId = foreach (var database in databases()) await database.DeadLetters.DeleteDeadLetterEnvelopesAsync(ids); } + public Task> SummarizeAllAsync(string serviceName, TimeRange range, CancellationToken token) + { + throw new NotImplementedException(); + } + + public Task QueryAsync(DeadLetterEnvelopeQuery query, CancellationToken token) + { + throw new NotImplementedException(); + } + + public Task DiscardAsync(DeadLetterEnvelopeQuery query, CancellationToken token) + { + throw new NotImplementedException(); + } + + public Task ReplayAsync(DeadLetterEnvelopeQuery query, CancellationToken token) + { + throw new NotImplementedException(); + } + public async Task QueryDeadLetterEnvelopesAsync( DeadLetterEnvelopeQueryParameters queryParameters, string? tenantId) { @@ -335,6 +357,10 @@ public async Task ReassignIncomingAsync(int ownerId, IReadOnlyList inc } public string Name { get; } + public void PromoteToMain(IWolverineRuntime runtime) + { + // Nothing here. + } public async Task> LoadPageOfGloballyOwnedIncomingAsync(Uri listenerAddress, int limit) { diff --git a/src/Wolverine/Persistence/Durability/NullMessageStore.cs b/src/Wolverine/Persistence/Durability/NullMessageStore.cs index 46fcd7c3f..d235b359e 100644 --- a/src/Wolverine/Persistence/Durability/NullMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/NullMessageStore.cs @@ -14,6 +14,8 @@ public class NullMessageStore : IMessageStore, IMessageInbox, IMessageOutbox, IM { internal IScheduledJobProcessor? ScheduledJobs { get; set; } + + public MessageStoreRole Role => MessageStoreRole.Main; public Uri Uri => new Uri($"{PersistenceConstants.AgentScheme}://null"); public Task MarkIncomingEnvelopeAsHandledAsync(Envelope envelope) @@ -22,6 +24,10 @@ public Task MarkIncomingEnvelopeAsHandledAsync(Envelope envelope) } public string Name => "Nullo"; + public void PromoteToMain(IWolverineRuntime runtime) + { + + } public Task MarkIncomingEnvelopeAsHandledAsync(IReadOnlyList envelopes) { @@ -206,6 +212,25 @@ public Task MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType) public Task MarkDeadLetterEnvelopesAsReplayableAsync(Guid[] ids, string? tenantId = null) => Task.CompletedTask; public Task DeleteDeadLetterEnvelopesAsync(Guid[] ids, string? tenantId = null) => Task.CompletedTask; + public Task> SummarizeAllAsync(string serviceName, TimeRange range, CancellationToken token) + { + return Task.FromResult>(new List()); + } + + public Task QueryAsync(DeadLetterEnvelopeQuery query, CancellationToken token) + { + return Task.FromResult(new DeadLetterEnvelopeResults()); + } + + public Task DiscardAsync(DeadLetterEnvelopeQuery query, CancellationToken token) + { + return Task.CompletedTask; + } + + public Task ReplayAsync(DeadLetterEnvelopeQuery query, CancellationToken token) + { + return Task.CompletedTask; + } public Task RebuildAsync() { diff --git a/src/Wolverine/Persistence/MessageStoreCollection.cs b/src/Wolverine/Persistence/MessageStoreCollection.cs new file mode 100644 index 000000000..0456bebea --- /dev/null +++ b/src/Wolverine/Persistence/MessageStoreCollection.cs @@ -0,0 +1,325 @@ +using ImTools; +using JasperFx.Core; +using JasperFx.Core.Reflection; +using JasperFx.Descriptors; +using Wolverine.Persistence.Durability; +using Wolverine.Runtime; +using Wolverine.Runtime.Agents; + +namespace Wolverine.Persistence; + +public class MessageStoreCollection : IAgentFamily +{ + private readonly IWolverineRuntime _runtime; + private readonly List _multiTenanted = new(); + private ImHashMap _services = ImHashMap.Empty; + private ImHashMap _ancillaryStores = ImHashMap.Empty; + private bool _onlyOneDatabase; + + public MessageStoreCollection(IWolverineRuntime runtime, IEnumerable stores, IEnumerable ancillaryMessageStores) + { + _runtime = runtime; + + foreach (var store in stores.Concat(ancillaryMessageStores)) + { + if (store is MultiTenantedMessageStore multiTenanted) + { + _multiTenanted.Add(multiTenanted); + categorizeStore(multiTenanted.Main); + } + else + { + categorizeStore(store); + } + } + + foreach (var ancillaryMessageStore in ancillaryMessageStores) + { + _ancillaryStores = _ancillaryStores.AddOrUpdate(ancillaryMessageStore.MarkerType, ancillaryMessageStore); + } + + if (!_runtime.Options.Durability.DurabilityAgentEnabled) + { + Main = new NullMessageStore(); + } + else if (_services.Count() == 1) + { + _onlyOneDatabase = !_multiTenanted.Any(); + + // Make sure in this case that the one, single store is really + // the "Main" store. And do it early so that this happens + // before we get to storage building + var messageStore = _services.Enumerate().Single().Value; + messageStore.PromoteToMain(_runtime); + Main = messageStore; + } + else + { + var mains = _services.Enumerate().Select(x => x.Value).Where(x => x.Role == MessageStoreRole.Main) + .ToArray(); + if (mains.Length == 1) + { + if (TryFindMultiTenantedForMainStore(mains[0], out var tenanted)) + { + Main = tenanted; + } + else + { + Main = mains[0]; + } + } + } + + } + + public IReadOnlyList MultiTenanted => _multiTenanted; + + private void categorizeStore(IMessageStore store) + { + if (_services.TryFind(store.Uri, out var existing)) + { + if (store.Role == MessageStoreRole.Main && existing.Role != MessageStoreRole.Main) + { + _services = _services.AddOrUpdate(store.Uri, store); + } + } + else + { + _services = _services.AddOrUpdate(store.Uri, store); + } + } + + private bool _hasInitialized; + internal async ValueTask InitializeAsync() + { + if (_hasInitialized) return; + _hasInitialized = true; + + if (!_runtime.Options.Durability.DurabilityAgentEnabled) + { + Main = new NullMessageStore(); + return; + } + + foreach (var tenantedMessageStore in _multiTenanted) + { + await tenantedMessageStore.InitializeAsync(_runtime); + await tenantedMessageStore.Source.RefreshAsync(); + foreach (var store in tenantedMessageStore.Source.AllActive()) + { + categorizeStore(store); + } + } + + _onlyOneDatabase = _services.Count() == 1 && !_multiTenanted.Any(); + + var mains = _services.Enumerate().Select(x => x.Value) + .Where(x => x.Role == MessageStoreRole.Main).ToArray(); + + if (mains.Length > 1) + { + throw new InvalidWolverineStorageConfigurationException( + $"There must be exactly one message store tagged as the 'main' store, you may need to mark all but one message store as 'ancillary'. Found multiples: {mains.Select(x => x.Uri.ToString()).Join(", ")}"); + } + + if (mains.Length == 1) + { + if (TryFindMultiTenantedForMainStore(mains[0], out var tenanted)) + { + Main = tenanted; + } + else + { + Main = mains[0]; + } + + return; + } + + if (!_services.IsEmpty || _multiTenanted.Any()) + { + throw new InvalidWolverineStorageConfigurationException( + "There are valid message stores for this Wolverine system, but none has been designated as the 'Main' store"); + } + + Main = new NullMessageStore(); + } + + public IMessageStore Main { get; private set; } = new NullMessageStore(); + + public DatabaseCardinality Cardinality() + { + if (_services.IsEmpty && !_multiTenanted.Any()) return DatabaseCardinality.None; + + if (_onlyOneDatabase) return DatabaseCardinality.Single; + + if (!_multiTenanted.Any()) return DatabaseCardinality.Single; + + if (_multiTenanted.Any(x => x.Source.Cardinality == DatabaseCardinality.DynamicMultiple)) + return DatabaseCardinality.DynamicMultiple; + + return DatabaseCardinality.StaticMultiple; + } + + public async ValueTask> FindAllAsync() + { + if (_onlyOneDatabase) return [Main]; + + foreach (var tenantedMessageStore in _multiTenanted) + { + if (tenantedMessageStore.Source.Cardinality == DatabaseCardinality.DynamicMultiple) + { + await refreshTenantedDatabaseList(tenantedMessageStore); + } + } + + return new List(_services.Enumerate().Select(x => x.Value)); + } + + /// + /// Find all message stores that can be cast to the type T + /// + /// + /// + public async ValueTask> FindAllAsync() + { + foreach (var tenantedMessageStore in _multiTenanted) + { + if (tenantedMessageStore.Source.Cardinality == DatabaseCardinality.DynamicMultiple) + { + await refreshTenantedDatabaseList(tenantedMessageStore); + } + } + + return _services.Enumerate().Select(x => x.Value).OfType().ToList(); + } + + private async ValueTask refreshTenantedDatabaseList(MultiTenantedMessageStore tenantedMessageStore) + { + await tenantedMessageStore.Source.RefreshAsync(); + + foreach (var store in tenantedMessageStore.Source.AllActive()) + { + categorizeStore(store); + } + } + + public async ValueTask FindDatabaseAsync(Uri uri) + { + if (_services.TryFind(uri, out var service)) + { + return service; + } + + // Force dynamic tenanted databases to refresh + foreach (var tenantedMessageStore in _multiTenanted) + { + if (tenantedMessageStore.Source.Cardinality == DatabaseCardinality.DynamicMultiple) + { + await refreshTenantedDatabaseList(tenantedMessageStore); + } + } + + // Try the lookup again + if (_services.TryFind(uri, out service)) + { + return service; + } + + // We're going to force it to probe for missing DBs every time instead + // of using a cached null in case it really does get added back later + return null; + } + + public async ValueTask> FindDatabasesAsync(Uri[] uris) + { + if (_onlyOneDatabase) return [Main]; + + var list = new List(); + foreach (var uri in uris) + { + var db = await FindDatabaseAsync(uri); + if (db != null) + { + list.Add(db); + } + } + + return list; + } + + public IAncillaryMessageStore FindAncillaryStore(Type markerType) + { + if (_ancillaryStores.TryFind(markerType, out var store)) return store; + + throw new ArgumentOutOfRangeException(nameof(markerType), + $"No known ancillary store for type {markerType.FullNameInCode()}. Known stores exist for {_ancillaryStores.Enumerate().Select(x => x.Key.FullNameInCode()).Join(", ")}"); + } + + public async Task DrainAsync() + { + foreach (var entry in _services.Enumerate()) + { + await entry.Value.DrainAsync(); + } + } + + public async Task MigrateAsync() + { + var stores = await FindAllAsync(); + foreach (var store in stores) + { + await store.Admin.MigrateAsync(); + } + } + + public string Scheme => PersistenceConstants.AgentScheme; + public async ValueTask> AllKnownAgentsAsync() + { + var stores = await FindAllAsync(); + return stores.Select(x => x.Uri).ToList(); + } + + public async ValueTask BuildAgentAsync(Uri uri, IWolverineRuntime wolverineRuntime) + { + var database = await FindDatabaseAsync(uri); + if (database is IMessageStoreWithAgentSupport agentSupport) + { + return agentSupport.BuildAgent(wolverineRuntime); + } + + throw new ArgumentOutOfRangeException(nameof(uri), $"No database with Uri {uri} supports a durability agent"); + } + + public ValueTask> SupportedAgentsAsync() + { + return AllKnownAgentsAsync(); + } + + public ValueTask EvaluateAssignmentsAsync(AssignmentGrid assignments) + { + assignments.DistributeEvenly(Scheme); + return ValueTask.CompletedTask; + } + + internal async Task StartScheduledJobProcessing(IWolverineRuntime runtime) + { + // First, find all unique message stores + var stores = await FindAllAsync(); + var agents = stores.Select(x => x.StartScheduledJobs(runtime)); + return new CompositeAgent(new Uri("internal://scheduledjobs"), agents); + } + + public bool TryFindMultiTenantedForMainStore(IMessageStore store, out MultiTenantedMessageStore multiTenanted) + { + multiTenanted = _multiTenanted.FirstOrDefault(x => x.Main.Uri == store.Uri); + return multiTenanted != null; + } +} + +public class InvalidWolverineStorageConfigurationException : Exception +{ + public InvalidWolverineStorageConfigurationException(string? message) : base(message) + { + } +} \ No newline at end of file diff --git a/src/Wolverine/Runtime/Agents/NodeAgentController.cs b/src/Wolverine/Runtime/Agents/NodeAgentController.cs index a9ce03ef6..5f2361929 100644 --- a/src/Wolverine/Runtime/Agents/NodeAgentController.cs +++ b/src/Wolverine/Runtime/Agents/NodeAgentController.cs @@ -46,8 +46,7 @@ internal NodeAgentController(IWolverineRuntime runtime, if (runtime.Options.Durability.DurabilityAgentEnabled) { - var family = new DurabilityAgentFamily(runtime); - _agentFamilies[family.Scheme] = family; + _agentFamilies[_runtime.Stores.Scheme] = _runtime.Stores; } foreach (var family in runtime.Options.Transports.OfType().SelectMany(x => x.BuildAgentFamilySources(runtime))) diff --git a/src/Wolverine/Runtime/IWolverineRuntime.cs b/src/Wolverine/Runtime/IWolverineRuntime.cs index ef2b29bdb..89c105a86 100644 --- a/src/Wolverine/Runtime/IWolverineRuntime.cs +++ b/src/Wolverine/Runtime/IWolverineRuntime.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging; using Wolverine.Configuration; using Wolverine.Logging; +using Wolverine.Persistence; using Wolverine.Persistence.Durability; using Wolverine.Runtime.Agents; using Wolverine.Runtime.Handlers; @@ -28,10 +29,19 @@ public interface IWolverineRuntime ILoggerFactory LoggerFactory { get; } IAgentRuntime Agents { get; } - IReadOnlyList AncillaryStores { get; } IServiceProvider Services { get; } IWolverineObserver Observer { get; set; } + MessageStoreCollection Stores { get; } + + /// + /// Try to find the main message store in a completely initialized state and safely cast to the type "T" + /// + /// + /// + Task TryFindMainMessageStore() where T : class; + + IAncillaryMessageStore FindAncillaryStoreForMarkerType(Type markerType); /// /// Schedule an envelope for later execution in memory diff --git a/src/Wolverine/Runtime/WolverineRuntime.Agents.cs b/src/Wolverine/Runtime/WolverineRuntime.Agents.cs index f52aab91d..5b140bad0 100644 --- a/src/Wolverine/Runtime/WolverineRuntime.Agents.cs +++ b/src/Wolverine/Runtime/WolverineRuntime.Agents.cs @@ -168,7 +168,7 @@ private async Task startAgentsAsync() private async Task startDurableScheduledJobs() { - DurableScheduledJobs = await DurabilityAgentFamily.StartScheduledJobProcessing(this); + DurableScheduledJobs = await _stores.Value.StartScheduledJobProcessing(this); } internal IAgent? DurableScheduledJobs { get; private set; } @@ -263,7 +263,7 @@ internal async Task DisableAgentsAsync(DateTimeOffset lastHeartbeatTime) if (NodeController != null) { await NodeController.DisableAgentsAsync(); - await _persistence.Value.Nodes.OverwriteHealthCheckTimeAsync(Options.UniqueNodeId, lastHeartbeatTime); + await Storage.Nodes.OverwriteHealthCheckTimeAsync(Options.UniqueNodeId, lastHeartbeatTime); } NodeController = null; diff --git a/src/Wolverine/Runtime/WolverineRuntime.HostService.cs b/src/Wolverine/Runtime/WolverineRuntime.HostService.cs index 4ccc9b64c..e2dfa52b9 100644 --- a/src/Wolverine/Runtime/WolverineRuntime.HostService.cs +++ b/src/Wolverine/Runtime/WolverineRuntime.HostService.cs @@ -29,6 +29,8 @@ public async Task StartAsync(CancellationToken cancellationToken) await ApplyAsyncExtensions(); + await _stores.Value.InitializeAsync(); + if (!Options.ExternalTransportsAreStubbed) { foreach (var configuresRuntime in Options.Transports.OfType().ToArray()) @@ -114,10 +116,7 @@ private async Task tryMigrateStorage() if (Options.AutoBuildMessageStorageOnStartup != AutoCreate.None) { - foreach (var ancillaryStore in AncillaryStores) - { - await ancillaryStore.Admin.MigrateAsync(); - } + await _stores.Value.MigrateAsync(); } _hasMigratedStorage = true; @@ -186,11 +185,11 @@ public async Task StopAsync(CancellationToken cancellationToken) // Latch health checks ASAP DisableHealthChecks(); - if (_persistence.IsValueCreated && StopMode == StopMode.Normal) + if (_stores.IsValueCreated && StopMode == StopMode.Normal) { try { - await Storage.DrainAsync(); + await _stores.Value.DrainAsync(); } catch (TaskCanceledException) { diff --git a/src/Wolverine/Runtime/WolverineRuntime.cs b/src/Wolverine/Runtime/WolverineRuntime.cs index d3ee1492d..3156f47ab 100644 --- a/src/Wolverine/Runtime/WolverineRuntime.cs +++ b/src/Wolverine/Runtime/WolverineRuntime.cs @@ -3,11 +3,13 @@ using JasperFx; using JasperFx.Core; using JasperFx.Core.Reflection; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.ObjectPool; using Wolverine.Configuration; using Wolverine.Logging; +using Wolverine.Persistence; using Wolverine.Persistence.Durability; using Wolverine.Runtime.Agents; using Wolverine.Runtime.Handlers; @@ -23,15 +25,13 @@ public sealed partial class WolverineRuntime : IWolverineRuntime, IHostedService private readonly EndpointCollection _endpoints; private readonly LightweightCache _invokers; - private readonly Lazy _persistence; - private readonly string _serviceName; private readonly Guid _uniqueNodeId; private ImHashMap _extensions = ImHashMap.Empty; private bool _hasStopped; - private readonly Lazy> _ancillaryStores; + private readonly Lazy _stores; public WolverineRuntime(WolverineOptions options, IServiceContainer container, @@ -41,6 +41,9 @@ public WolverineRuntime(WolverineOptions options, Options = options; Handlers = options.HandlerGraph; + _stores = + new Lazy(() => container.Services.GetRequiredService()); + LoggerFactory = loggers; Logger = loggers.CreateLogger(); @@ -58,8 +61,6 @@ public WolverineRuntime(WolverineOptions options, Pipeline = new HandlerPipeline(this, this); - _persistence = new Lazy(container.GetInstance); - _container = container; Cancellation = DurabilitySettings.Cancellation; @@ -92,9 +93,6 @@ public WolverineRuntime(WolverineOptions options, _invokers = new LightweightCache(findInvoker); - _ancillaryStores = - new Lazy>(() => _container.GetAllInstances()); - var activators = container.GetAllInstances(); foreach (var activator in activators) { @@ -106,8 +104,6 @@ public WolverineRuntime(WolverineOptions options, public IServiceProvider Services => _container.Services; - public IReadOnlyList AncillaryStores => _ancillaryStores.Value; - public ObjectPool ExecutionPool { get; } internal HandlerGraph Handlers { get; } @@ -193,12 +189,24 @@ public void ScheduleLocalExecutionInMemory(DateTimeOffset executionTime, Envelop ScheduledJobs.Enqueue(executionTime, envelope); } + public IAncillaryMessageStore FindAncillaryStoreForMarkerType(Type markerType) + { + return _stores.Value.FindAncillaryStore(markerType); + } + + public MessageStoreCollection Stores => _stores.Value; + + public async Task TryFindMainMessageStore() where T : class + { + await _stores.Value.InitializeAsync(); + return _stores.Value.Main as T; + } + public IHandlerPipeline Pipeline { get; } public IMessageTracker MessageTracking => this; - - public IMessageStore Storage => _persistence.Value; + public IMessageStore Storage => _stores.Value.Main; private IMessageInvoker findInvoker(Type messageType) { diff --git a/src/Wolverine/WolverineSystemPart.cs b/src/Wolverine/WolverineSystemPart.cs index a2c4b16c2..bb06dea89 100644 --- a/src/Wolverine/WolverineSystemPart.cs +++ b/src/Wolverine/WolverineSystemPart.cs @@ -173,34 +173,29 @@ public void WriteErrorHandling() } - public override ValueTask> FindResources() + public override async ValueTask> FindResources() { var list = new List(); // These have to run first. Right now, the only options are for building multi-tenanted // databases with EF Core list.AddRange(_runtime.Services.GetServices()); - - if (_runtime.Options.ExternalTransportsAreStubbed) return new ValueTask>(list); - foreach (var transport in _runtime.Options.Transports) + if (!_runtime.Options.ExternalTransportsAreStubbed) { - if (transport.TryBuildStatefulResource(_runtime, out var resource)) + foreach (var transport in _runtime.Options.Transports) { - list.Add(resource!); + if (transport.TryBuildStatefulResource(_runtime, out var resource)) + { + list.Add(resource!); + } } } - if (_runtime.Storage is not NullMessageStore) - { - list.Add(new MessageStoreResource(_runtime.Options, _runtime.Storage)); - } - - foreach (var store in _runtime.AncillaryStores) - { - list.Add(new MessageStoreResource(_runtime.Options, store)); - } + var stores = await _runtime.Stores.FindAllAsync(); + + list.AddRange(stores.Select(store => new MessageStoreResource(_runtime.Options, store))); - return new ValueTask>(list); + return list; } } \ No newline at end of file From 7e986a0bda9d780a14ac7a42311cfb622ca7f327 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Thu, 18 Sep 2025 15:53:16 -0500 Subject: [PATCH 11/25] More validation on adding ancillary sql server or postgresql message stores --- .../registration_of_message_stores.cs | 52 +++++++++++++++++++ .../PostgresqlConfigurationExtensions.cs | 2 +- .../SqlServerConfigurationExtensions.cs | 2 +- 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/src/Persistence/PersistenceTests/registration_of_message_stores.cs b/src/Persistence/PersistenceTests/registration_of_message_stores.cs index 8e306610a..9687af7a2 100644 --- a/src/Persistence/PersistenceTests/registration_of_message_stores.cs +++ b/src/Persistence/PersistenceTests/registration_of_message_stores.cs @@ -14,6 +14,7 @@ using Wolverine.Persistence; using Wolverine.Persistence.Durability; using Wolverine.Postgresql; +using Wolverine.SqlServer; using Wolverine.Tracking; using Xunit; using Xunit.Abstractions; @@ -298,6 +299,57 @@ public async Task no_message_stores_still_works() collection.Cardinality().ShouldBe(DatabaseCardinality.None); } + + [Fact] + public async Task register_two_postgresql_both_main_throws() + { + await Should.ThrowAsync(async () => + { + var collection = await startHost(opts => + { + opts.PersistMessagesWithPostgresql(Servers.PostgresConnectionString); + opts.PersistMessagesWithPostgresql(connectionString1); + }); + }); + } + + [Fact] + public async Task register_two_sql_server_both_main_throws() + { + await Should.ThrowAsync(async () => + { + var collection = await startHost(opts => + { + opts.PersistMessagesWithSqlServer(Servers.SqlServerConnectionString, "one"); + opts.PersistMessagesWithSqlServer(Servers.SqlServerConnectionString, "two"); + + }); + }); + } + + [Fact] + public async Task register_one_main_and_one_ancillary_postgresql() + { + var collection = await startHost(opts => + { + opts.PersistMessagesWithPostgresql(Servers.PostgresConnectionString); + opts.PersistMessagesWithPostgresql(connectionString1, role:MessageStoreRole.Ancillary); + }); + + collection.Main.ShouldBeOfType(); + (await collection.FindAllAsync()).Count.ShouldBe(2); + } + + [Fact] + public async Task register_one_main_and_one_ancillary_sql_server() + { + var collection = await startHost(opts => + { + opts.PersistMessagesWithSqlServer(Servers.SqlServerConnectionString, "one"); + opts.PersistMessagesWithSqlServer(Servers.SqlServerConnectionString, "two", role:MessageStoreRole.Ancillary); + + }); + } } diff --git a/src/Persistence/Wolverine.Postgresql/PostgresqlConfigurationExtensions.cs b/src/Persistence/Wolverine.Postgresql/PostgresqlConfigurationExtensions.cs index d2fdde1f7..8fd8906f0 100644 --- a/src/Persistence/Wolverine.Postgresql/PostgresqlConfigurationExtensions.cs +++ b/src/Persistence/Wolverine.Postgresql/PostgresqlConfigurationExtensions.cs @@ -52,7 +52,7 @@ public static IPostgresqlBackedPersistence PersistMessagesWithPostgresql(this Wo persistence.EnvelopeStorageSchemaName = schemaName; } - options.Include(persistence); + persistence.Configure(options); return persistence; } diff --git a/src/Persistence/Wolverine.SqlServer/SqlServerConfigurationExtensions.cs b/src/Persistence/Wolverine.SqlServer/SqlServerConfigurationExtensions.cs index 6ea759f05..446f58976 100644 --- a/src/Persistence/Wolverine.SqlServer/SqlServerConfigurationExtensions.cs +++ b/src/Persistence/Wolverine.SqlServer/SqlServerConfigurationExtensions.cs @@ -35,7 +35,7 @@ public static ISqlServerBackedPersistence PersistMessagesWithSqlServer(this Wolv extension.EnvelopeStorageSchemaName = options.Durability.MessageStorageSchemaName ?? "dbo"; } - options.Include(extension); + extension.Configure(options); return extension; } From 7bee218e5c2d358d4df0402bcbbd96d7512390ed Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Thu, 18 Sep 2025 16:05:52 -0500 Subject: [PATCH 12/25] Made the StorageCommand aware of the MessageStoreCollection and added IHost extensions for test helpers --- src/Wolverine/Persistence/StorageCommand.cs | 8 ++-- src/Wolverine/Runtime/StorageExtensions.cs | 48 +++++++++++++++++++++ src/Wolverine/Wolverine.csproj | 4 -- 3 files changed, 52 insertions(+), 8 deletions(-) create mode 100644 src/Wolverine/Runtime/StorageExtensions.cs diff --git a/src/Wolverine/Persistence/StorageCommand.cs b/src/Wolverine/Persistence/StorageCommand.cs index ea5ca93c6..1c03f946e 100644 --- a/src/Wolverine/Persistence/StorageCommand.cs +++ b/src/Wolverine/Persistence/StorageCommand.cs @@ -4,6 +4,7 @@ using JasperFx.CommandLine; using Spectre.Console; using Wolverine.Persistence.Durability; +using Wolverine.Runtime; namespace Wolverine.Persistence; @@ -63,19 +64,18 @@ public override async Task Execute(StorageInput input) case StorageCommandAction.clear: await persistence.Admin.ClearAllAsync(); - await persistence.Nodes.ClearAllAsync(CancellationToken.None); + await host.ClearAllPersistedWolverineDataAsync(); AnsiConsole.Write("[green]Successfully deleted all persisted envelopes and existing node records[/]"); break; case StorageCommandAction.rebuild: - await persistence.Admin.RebuildAsync(); + await host.RebuildAllEnvelopeStorageAsync(); AnsiConsole.Write("[green]Successfully rebuilt the envelope storage[/]"); break; case StorageCommandAction.release: - await persistence.Admin.RebuildAsync(); Console.WriteLine("Releasing all ownership of persisted envelopes"); - await persistence.Admin.ReleaseAllOwnershipAsync(); + await host.ReleaseAllOwnershipAsync(); break; diff --git a/src/Wolverine/Runtime/StorageExtensions.cs b/src/Wolverine/Runtime/StorageExtensions.cs new file mode 100644 index 000000000..7733c88ee --- /dev/null +++ b/src/Wolverine/Runtime/StorageExtensions.cs @@ -0,0 +1,48 @@ +using Microsoft.Extensions.Hosting; +using Wolverine.Tracking; + +namespace Wolverine.Runtime; + +public static class StorageExtensions +{ + /// + /// Clear out all the envelope storage for all known message stories in this application + /// + /// + public static async Task ClearAllEnvelopeStorageAsync(this IHost host) + { + var runtime = host.GetRuntime(); + foreach (var store in await runtime.Stores.FindAllAsync()) + { + await store.Admin.ClearAllAsync(); + } + } + + /// + /// Rebuild the envelope storage for all message stores in this application + /// + /// + public static async Task RebuildAllEnvelopeStorageAsync(this IHost host) + { + var runtime = host.GetRuntime(); + foreach (var store in await runtime.Stores.FindAllAsync()) + { + await store.Admin.RebuildAsync(); + } + } + + /// + /// Release the ownership of all persisted envelopes by the current node + /// + /// + public static async Task ReleaseAllOwnershipAsync(this IHost host) + { + var runtime = host.GetRuntime(); + foreach (var store in await runtime.Stores.FindAllAsync()) + { + await store.Admin.ReleaseAllOwnershipAsync(); + } + } + + +} \ No newline at end of file diff --git a/src/Wolverine/Wolverine.csproj b/src/Wolverine/Wolverine.csproj index 49aec928e..3359d2429 100644 --- a/src/Wolverine/Wolverine.csproj +++ b/src/Wolverine/Wolverine.csproj @@ -34,10 +34,6 @@ - - - - From e28990d8df79550f16cf5d867dffea3aa52bec20 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Thu, 18 Sep 2025 16:19:35 -0500 Subject: [PATCH 13/25] Review of usages WolverineRuntime.Storage --- .../Durability/NodeRecoveryOperation.cs | 25 ------------------- .../MultiTenancy/MessageDatabaseDiscovery.cs | 17 +++---------- .../DurableSendingAndReceivingCompliance.cs | 4 +-- .../Configuration/EndpointCollection.cs | 8 +----- .../Persistence/MessageStoreCollection.cs | 16 +++++++++++- .../Runtime/WolverineRuntime.HostService.cs | 22 ++++------------ 6 files changed, 26 insertions(+), 66 deletions(-) delete mode 100644 src/Persistence/Wolverine.RDBMS/Durability/NodeRecoveryOperation.cs diff --git a/src/Persistence/Wolverine.RDBMS/Durability/NodeRecoveryOperation.cs b/src/Persistence/Wolverine.RDBMS/Durability/NodeRecoveryOperation.cs deleted file mode 100644 index beec85581..000000000 --- a/src/Persistence/Wolverine.RDBMS/Durability/NodeRecoveryOperation.cs +++ /dev/null @@ -1,25 +0,0 @@ -using Microsoft.Extensions.Logging; -using Wolverine.Runtime; -using Wolverine.Runtime.Agents; - -namespace Wolverine.RDBMS.Durability; - -internal class NodeRecoveryOperation : IAgentCommand -{ - private readonly int _ownerNodeId; - - public NodeRecoveryOperation(int ownerNodeId) - { - _ownerNodeId = ownerNodeId; - } - - public async Task ExecuteAsync(IWolverineRuntime runtime, - CancellationToken cancellationToken) - { - runtime.Logger.LogInformation("Releasing node ownership in the inbox/outbox from dormant node {Node}", - _ownerNodeId); - await runtime.Storage.Admin.ReleaseAllOwnershipAsync(); - - return AgentCommands.Empty; - } -} \ No newline at end of file diff --git a/src/Persistence/Wolverine.RDBMS/MultiTenancy/MessageDatabaseDiscovery.cs b/src/Persistence/Wolverine.RDBMS/MultiTenancy/MessageDatabaseDiscovery.cs index e4e49f3f9..4caa4c515 100644 --- a/src/Persistence/Wolverine.RDBMS/MultiTenancy/MessageDatabaseDiscovery.cs +++ b/src/Persistence/Wolverine.RDBMS/MultiTenancy/MessageDatabaseDiscovery.cs @@ -39,6 +39,8 @@ public async ValueTask DescribeDatabasesAsync(CancellationToken t usage.Cardinality = DatabaseCardinality.None; } + usage.Cardinality = _runtime.Stores.Cardinality(); + return usage; } @@ -47,18 +49,5 @@ public ValueTask> BuildDatabases() return _runtime.Stores.FindAllAsync(); } - public DatabaseCardinality Cardinality - { - get - { - if (_runtime.Storage is MultiTenantedMessageStore tenantedMessageStore) - { - return tenantedMessageStore.Source.Cardinality; - } - else - { - return DatabaseCardinality.Single; - } - } - } + public DatabaseCardinality Cardinality => _runtime.Stores.Cardinality(); } \ No newline at end of file diff --git a/src/Transports/AWS/Wolverine.AmazonSqs.Tests/DurableSendingAndReceivingCompliance.cs b/src/Transports/AWS/Wolverine.AmazonSqs.Tests/DurableSendingAndReceivingCompliance.cs index adbb1daef..e819dcb99 100644 --- a/src/Transports/AWS/Wolverine.AmazonSqs.Tests/DurableSendingAndReceivingCompliance.cs +++ b/src/Transports/AWS/Wolverine.AmazonSqs.Tests/DurableSendingAndReceivingCompliance.cs @@ -63,8 +63,8 @@ await ReceiverIs(opts => opts.ListenToSqsQueue("receiver-" + number); }); - await Sender.Services.GetRequiredService().Storage.Admin.RebuildAsync(); - await Receiver.Services.GetRequiredService().Storage.Admin.RebuildAsync(); + await Sender.RebuildAllEnvelopeStorageAsync(); + await Receiver.RebuildAllEnvelopeStorageAsync(); } public async Task DisposeAsync() diff --git a/src/Wolverine/Configuration/EndpointCollection.cs b/src/Wolverine/Configuration/EndpointCollection.cs index 4cb70eb7b..db07ebd43 100644 --- a/src/Wolverine/Configuration/EndpointCollection.cs +++ b/src/Wolverine/Configuration/EndpointCollection.cs @@ -310,18 +310,12 @@ private ISendingAgent buildSendingAgent(ISender sender, Endpoint endpoint) return a; } - IMessageStore store = _runtime.Storage; - if (_runtime.Stores.TryFindMultiTenantedForMainStore(store, out var tenanted)) - { - store = tenanted; - } - switch (endpoint.Mode) { case EndpointMode.Durable: return new DurableSendingAgent(sender, _options.Durability, _runtime.LoggerFactory.CreateLogger(), _runtime.MessageTracking, - store, endpoint); + _runtime.Storage, endpoint); case EndpointMode.BufferedInMemory: return new BufferedSendingAgent(_runtime.LoggerFactory.CreateLogger(), diff --git a/src/Wolverine/Persistence/MessageStoreCollection.cs b/src/Wolverine/Persistence/MessageStoreCollection.cs index 0456bebea..5e9507bb1 100644 --- a/src/Wolverine/Persistence/MessageStoreCollection.cs +++ b/src/Wolverine/Persistence/MessageStoreCollection.cs @@ -8,7 +8,7 @@ namespace Wolverine.Persistence; -public class MessageStoreCollection : IAgentFamily +public class MessageStoreCollection : IAgentFamily, IAsyncDisposable { private readonly IWolverineRuntime _runtime; private readonly List _multiTenanted = new(); @@ -315,6 +315,20 @@ public bool TryFindMultiTenantedForMainStore(IMessageStore store, out MultiTenan multiTenanted = _multiTenanted.FirstOrDefault(x => x.Main.Uri == store.Uri); return multiTenanted != null; } + + public async Task ReleaseAllOwnershipAsync(int nodeNumber) + { + foreach (var store in _services.Enumerate().Select(x => x.Value)) + { + await store.Admin.ReleaseAllOwnershipAsync(nodeNumber); + } + } + + public ValueTask DisposeAsync() + { + var stores = _services.Enumerate().Select(x => x.Value).ToArray(); + return stores.MaybeDisposeAllAsync(); + } } public class InvalidWolverineStorageConfigurationException : Exception diff --git a/src/Wolverine/Runtime/WolverineRuntime.HostService.cs b/src/Wolverine/Runtime/WolverineRuntime.HostService.cs index e2dfa52b9..047e2a336 100644 --- a/src/Wolverine/Runtime/WolverineRuntime.HostService.cs +++ b/src/Wolverine/Runtime/WolverineRuntime.HostService.cs @@ -49,8 +49,10 @@ public async Task StartAsync(CancellationToken cancellationToken) if (Options.Durability.DurabilityAgentEnabled) { - // TODO -- this needs to be async! - Storage.Initialize(this); + foreach (var store in await _stores.Value.FindAllAsync()) + { + store.Initialize(this); + } } // This MUST be done before the messaging transports are started up @@ -110,11 +112,6 @@ private async Task tryMigrateStorage() if (!Options.Durability.DurabilityAgentEnabled) return; if (Options.AutoBuildMessageStorageOnStartup != AutoCreate.None && Storage is not NullMessageStore) - { - await Storage.Admin.MigrateAsync(); - } - - if (Options.AutoBuildMessageStorageOnStartup != AutoCreate.None) { await _stores.Value.MigrateAsync(); } @@ -199,7 +196,7 @@ public async Task StopAsync(CancellationToken cancellationToken) try { // New to 3.0, try to release any ownership on the way out. Do this *after* the drain - await Storage.Admin.ReleaseAllOwnershipAsync(DurabilitySettings.AssignedNodeNumber); + await _stores.Value.ReleaseAllOwnershipAsync(DurabilitySettings.AssignedNodeNumber); } catch (ObjectDisposedException) { @@ -216,15 +213,6 @@ public async Task StopAsync(CancellationToken cancellationToken) } DurabilitySettings.Cancel(); - - try - { - // Do this to release pooled connections in Npgsql just in case - await Storage.DisposeAsync(); - } - catch (Exception) - { - } } private void startInMemoryScheduledJobs() From 0d7578b960194f2bec189d42bd6004ff89300799 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Fri, 19 Sep 2025 07:56:20 -0500 Subject: [PATCH 14/25] Removed an API in IMessageInbox that was not used --- .../cross_database_message_storage.cs | 2 -- .../Wolverine.RDBMS/MessageDatabase.cs | 13 ---------- .../Internals/RavenDbMessageStore.Inbox.cs | 25 ------------------- src/Wolverine/Envelope.cs | 8 ++++++ .../Persistence/Durability/IMessageStore.cs | 10 +++++--- .../Durability/MultiTenantedMessageStore.cs | 5 ---- .../Durability/NullMessageStore.cs | 5 ---- 7 files changed, 14 insertions(+), 54 deletions(-) diff --git a/src/Persistence/MartenTests/MultiTenancy/cross_database_message_storage.cs b/src/Persistence/MartenTests/MultiTenancy/cross_database_message_storage.cs index 8e3e4f058..6c1ea0022 100644 --- a/src/Persistence/MartenTests/MultiTenancy/cross_database_message_storage.cs +++ b/src/Persistence/MartenTests/MultiTenancy/cross_database_message_storage.cs @@ -536,8 +536,6 @@ public async Task clear_all_spans_all_databases() [Fact] public async Task release_ownership_smoke_test() { - await Stores.Inbox.ReleaseIncomingAsync(3); - await Stores.Inbox.ReleaseIncomingAsync(4, TransportConstants.LocalUri); } diff --git a/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs b/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs index 522db186b..b17f4851e 100644 --- a/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs +++ b/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs @@ -217,19 +217,6 @@ public async ValueTask DisposeAsync() public abstract DbCommandBuilder ToCommandBuilder(); - public async Task ReleaseIncomingAsync(int ownerId) - { - if (_cancellation.IsCancellationRequested) return; - - var count = await _dataSource - .CreateCommand( - $"update {SchemaName}.{DatabaseConstants.IncomingTable} set owner_id = 0 where owner_id = @owner") - .With("owner", ownerId) - .ExecuteNonQueryAsync(_cancellation); - - Logger.LogInformation("Reassigned {Count} incoming messages from {Owner} to any node in the durable inbox", count, ownerId); - } - public async Task ReleaseIncomingAsync(int ownerId, Uri receivedAt) { if (HasDisposed) return; diff --git a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.Inbox.cs b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.Inbox.cs index 2402f408b..69eda90b4 100644 --- a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.Inbox.cs +++ b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.Inbox.cs @@ -168,31 +168,6 @@ where id() in ($ids) await op.WaitForCompletionAsync(); } - public async Task ReleaseIncomingAsync(int ownerId) - { - using var session = _store.OpenAsyncSession(); - - var query = new IndexQuery - { - Query = $@" -from IncomingMessages as m -where m.OwnerId = $owner -update -{{ - m.OwnerId = 0 -}}", - WaitForNonStaleResults = true, - WaitForNonStaleResultsTimeout = 10.Seconds(), - QueryParameters = new() - { - {"owner", ownerId} - } - }; - - var op = await _store.Operations.SendAsync(new PatchByQueryOperation(query)); - await op.WaitForCompletionAsync(); - } - public async Task ReleaseIncomingAsync(int ownerId, Uri receivedAt) { using var session = _store.OpenAsyncSession(); diff --git a/src/Wolverine/Envelope.cs b/src/Wolverine/Envelope.cs index 401c25510..bde7e27f7 100644 --- a/src/Wolverine/Envelope.cs +++ b/src/Wolverine/Envelope.cs @@ -4,6 +4,7 @@ using JasperFx.MultiTenancy; using MassTransit; using Wolverine.Attributes; +using Wolverine.Persistence.Durability; using Wolverine.Runtime.Serialization; using Wolverine.Util; @@ -443,4 +444,11 @@ internal string GetMessageTypeName() /// For stream based transports (Kafka/RedPanda, this will reflect the message offset. This is strictly informational /// public long Offset { get; set; } + + + /// + /// For some forms of modular monoliths, Wolverine needs to track what message store + /// persisted this envelope for later tracking + /// + public IMessageStore? Store { get; private set; } } \ No newline at end of file diff --git a/src/Wolverine/Persistence/Durability/IMessageStore.cs b/src/Wolverine/Persistence/Durability/IMessageStore.cs index d33614f06..b0503530b 100644 --- a/src/Wolverine/Persistence/Durability/IMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/IMessageStore.cs @@ -32,19 +32,21 @@ public enum MessageStoreRole public interface IMessageInbox { + // This is *moving* an existing, persisted envelope in the inbox to being + // scheduled for a retry + Task ScheduleJobAsync(Envelope envelope); Task ScheduleExecutionAsync(Envelope envelope); Task MoveToDeadLetterStorageAsync(Envelope envelope, Exception? exception); Task IncrementIncomingEnvelopeAttemptsAsync(Envelope envelope); Task StoreIncomingAsync(Envelope envelope); Task StoreIncomingAsync(IReadOnlyList envelopes); - Task ScheduleJobAsync(Envelope envelope); + Task MarkIncomingEnvelopeAsHandledAsync(Envelope envelope); + + // Only called by DurableReceiver Task MarkIncomingEnvelopeAsHandledAsync(IReadOnlyList envelopes); - // Good as is - Task ReleaseIncomingAsync(int ownerId); - // Good as is Task ReleaseIncomingAsync(int ownerId, Uri receivedAt); } diff --git a/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs b/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs index 4933550c6..f7ee48fbb 100644 --- a/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs @@ -236,11 +236,6 @@ public async Task MarkIncomingEnvelopeAsHandledAsync(IReadOnlyList env } } - Task IMessageInbox.ReleaseIncomingAsync(int ownerId) - { - return executeOnAllAsync(d => d.Inbox.ReleaseIncomingAsync(ownerId)); - } - Task IMessageInbox.ReleaseIncomingAsync(int ownerId, Uri receivedAt) { return executeOnAllAsync(d => d.Inbox.ReleaseIncomingAsync(ownerId, receivedAt)); diff --git a/src/Wolverine/Persistence/Durability/NullMessageStore.cs b/src/Wolverine/Persistence/Durability/NullMessageStore.cs index d235b359e..3da457531 100644 --- a/src/Wolverine/Persistence/Durability/NullMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/NullMessageStore.cs @@ -96,11 +96,6 @@ public Task ScheduleJobAsync(Envelope envelope) return Task.CompletedTask; } - public Task ReleaseIncomingAsync(int ownerId) - { - return Task.CompletedTask; - } - public Task DeleteOutgoingAsync(Envelope[] envelopes) { return Task.CompletedTask; From c0fcb3d317241f1f597116e1c4e3920b32064780 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Fri, 19 Sep 2025 08:01:50 -0500 Subject: [PATCH 15/25] Renamed an API method on IMessageInbox that wasn't clear --- .../Persistence/MartenBackedMessagePersistenceTests.cs | 2 +- .../Persistence/SqlServerBackedMessageStoreTests.cs | 2 +- src/Persistence/Wolverine.RDBMS/MessageDatabase.Scheduled.cs | 2 +- .../Wolverine.RavenDb/Internals/RavenDbMessageStore.Inbox.cs | 2 +- src/Testing/CoreTests/Runtime/MessageContextTests.cs | 4 ++-- src/Wolverine/Persistence/Durability/IMessageStore.cs | 4 +--- .../Persistence/Durability/MultiTenantedMessageStore.cs | 4 ++-- src/Wolverine/Persistence/Durability/NullMessageStore.cs | 2 +- src/Wolverine/Runtime/MessageContext.cs | 4 ++-- 9 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/Persistence/MartenTests/Persistence/MartenBackedMessagePersistenceTests.cs b/src/Persistence/MartenTests/Persistence/MartenBackedMessagePersistenceTests.cs index 72211fb81..226b8ba60 100644 --- a/src/Persistence/MartenTests/Persistence/MartenBackedMessagePersistenceTests.cs +++ b/src/Persistence/MartenTests/Persistence/MartenBackedMessagePersistenceTests.cs @@ -50,7 +50,7 @@ public async Task InitializeAsync() await persistence.Admin.RebuildAsync(); - persistence.Inbox.ScheduleJobAsync(theEnvelope).Wait(3.Seconds()); + persistence.Inbox.RescheduleExistingEnvelopeForRetryAsync(theEnvelope).Wait(3.Seconds()); persisted = (await persistence.Admin .AllIncomingAsync()) diff --git a/src/Persistence/SqlServerTests/Persistence/SqlServerBackedMessageStoreTests.cs b/src/Persistence/SqlServerTests/Persistence/SqlServerBackedMessageStoreTests.cs index 9a9bda78c..f75b032cc 100644 --- a/src/Persistence/SqlServerTests/Persistence/SqlServerBackedMessageStoreTests.cs +++ b/src/Persistence/SqlServerTests/Persistence/SqlServerBackedMessageStoreTests.cs @@ -41,7 +41,7 @@ protected override async Task initialize() theEnvelope.ConversationId = Guid.NewGuid(); theEnvelope.ParentId = Guid.NewGuid().ToString(); - theHost.Get().Inbox.ScheduleJobAsync(theEnvelope).Wait(3.Seconds()); + theHost.Get().Inbox.RescheduleExistingEnvelopeForRetryAsync(theEnvelope).Wait(3.Seconds()); var persistor = theHost.GetRuntime().Storage.As(); diff --git a/src/Persistence/Wolverine.RDBMS/MessageDatabase.Scheduled.cs b/src/Persistence/Wolverine.RDBMS/MessageDatabase.Scheduled.cs index d8d307504..1e12797d3 100644 --- a/src/Persistence/Wolverine.RDBMS/MessageDatabase.Scheduled.cs +++ b/src/Persistence/Wolverine.RDBMS/MessageDatabase.Scheduled.cs @@ -18,7 +18,7 @@ public Task ScheduleExecutionAsync(Envelope envelope) .ExecuteNonQueryAsync(_cancellation); } - public Task ScheduleJobAsync(Envelope envelope) + public Task RescheduleExistingEnvelopeForRetryAsync(Envelope envelope) { envelope.Status = EnvelopeStatus.Scheduled; envelope.OwnerId = TransportConstants.AnyNode; diff --git a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.Inbox.cs b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.Inbox.cs index 69eda90b4..3016bb20e 100644 --- a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.Inbox.cs +++ b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.Inbox.cs @@ -99,7 +99,7 @@ public async Task StoreIncomingAsync(IReadOnlyList envelopes) await session.SaveChangesAsync(); } - public Task ScheduleJobAsync(Envelope envelope) + public Task RescheduleExistingEnvelopeForRetryAsync(Envelope envelope) { envelope.Status = EnvelopeStatus.Scheduled; envelope.OwnerId = TransportConstants.AnyNode; diff --git a/src/Testing/CoreTests/Runtime/MessageContextTests.cs b/src/Testing/CoreTests/Runtime/MessageContextTests.cs index ced4549e2..0f1f8036e 100644 --- a/src/Testing/CoreTests/Runtime/MessageContextTests.cs +++ b/src/Testing/CoreTests/Runtime/MessageContextTests.cs @@ -115,7 +115,7 @@ public async Task reschedule_without_native_scheduling() theEnvelope.ScheduledTime.ShouldBe(scheduledTime); - await theContext.Storage.Inbox.Received().ScheduleJobAsync(theEnvelope); + await theContext.Storage.Inbox.Received().RescheduleExistingEnvelopeForRetryAsync(theEnvelope); } [Fact] @@ -130,7 +130,7 @@ public async Task reschedule_with_native_scheduling() theEnvelope.ScheduledTime.ShouldBe(scheduledTime); - await theContext.Storage.Inbox.DidNotReceive().ScheduleJobAsync(theEnvelope); + await theContext.Storage.Inbox.DidNotReceive().RescheduleExistingEnvelopeForRetryAsync(theEnvelope); await callback.As().Received() .MoveToScheduledUntilAsync(theEnvelope, scheduledTime); } diff --git a/src/Wolverine/Persistence/Durability/IMessageStore.cs b/src/Wolverine/Persistence/Durability/IMessageStore.cs index b0503530b..5e47c111d 100644 --- a/src/Wolverine/Persistence/Durability/IMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/IMessageStore.cs @@ -32,9 +32,7 @@ public enum MessageStoreRole public interface IMessageInbox { - // This is *moving* an existing, persisted envelope in the inbox to being - // scheduled for a retry - Task ScheduleJobAsync(Envelope envelope); + Task RescheduleExistingEnvelopeForRetryAsync(Envelope envelope); Task ScheduleExecutionAsync(Envelope envelope); Task MoveToDeadLetterStorageAsync(Envelope envelope, Exception? exception); Task IncrementIncomingEnvelopeAttemptsAsync(Envelope envelope); diff --git a/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs b/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs index f7ee48fbb..964f67ec1 100644 --- a/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs @@ -198,10 +198,10 @@ async Task IMessageInbox.StoreIncomingAsync(IReadOnlyList envelopes) } } - async Task IMessageInbox.ScheduleJobAsync(Envelope envelope) + async Task IMessageInbox.RescheduleExistingEnvelopeForRetryAsync(Envelope envelope) { var database = await GetDatabaseAsync(envelope.TenantId); - await database.Inbox.ScheduleJobAsync(envelope); + await database.Inbox.RescheduleExistingEnvelopeForRetryAsync(envelope); } async Task IMessageInbox.MarkIncomingEnvelopeAsHandledAsync(Envelope envelope) diff --git a/src/Wolverine/Persistence/Durability/NullMessageStore.cs b/src/Wolverine/Persistence/Durability/NullMessageStore.cs index 3da457531..546eba7ad 100644 --- a/src/Wolverine/Persistence/Durability/NullMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/NullMessageStore.cs @@ -83,7 +83,7 @@ public Task StoreIncomingAsync(IReadOnlyList envelopes) return Task.CompletedTask; } - public Task ScheduleJobAsync(Envelope envelope) + public Task RescheduleExistingEnvelopeForRetryAsync(Envelope envelope) { if (!envelope.ScheduledTime.HasValue) { diff --git a/src/Wolverine/Runtime/MessageContext.cs b/src/Wolverine/Runtime/MessageContext.cs index daab783f6..451049327 100644 --- a/src/Wolverine/Runtime/MessageContext.cs +++ b/src/Wolverine/Runtime/MessageContext.cs @@ -158,7 +158,7 @@ public async Task ReScheduleAsync(DateTimeOffset scheduledTime) } else { - await Storage.Inbox.ScheduleJobAsync(Envelope); + await Storage.Inbox.RescheduleExistingEnvelopeForRetryAsync(Envelope); } } @@ -505,7 +505,7 @@ private async Task flushScheduledMessagesAsync() } else { - foreach (var envelope in Scheduled) await Storage.Inbox.ScheduleJobAsync(envelope); + foreach (var envelope in Scheduled) await Storage.Inbox.RescheduleExistingEnvelopeForRetryAsync(envelope); } Scheduled.Clear(); From f87141484b13f4c9efa0a78628f5b4354e0d9d1d Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Fri, 19 Sep 2025 08:06:47 -0500 Subject: [PATCH 16/25] Pruned IMessageStore a bit more --- .../Wolverine.Postgresql/PostgresqlMessageStore.cs | 5 ----- src/Persistence/Wolverine.RDBMS/MessageDatabase.cs | 2 -- .../Wolverine.RavenDb/Internals/RavenDbMessageStore.cs | 5 ----- .../Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs | 5 ----- src/Wolverine/Persistence/Durability/IMessageStore.cs | 3 --- .../Persistence/Durability/MultiTenantedMessageStore.cs | 5 ----- src/Wolverine/Persistence/Durability/NullMessageStore.cs | 5 ----- src/Wolverine/Persistence/StorageCommand.cs | 2 -- 8 files changed, 32 deletions(-) diff --git a/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs b/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs index bc83cb0c7..aeec51973 100644 --- a/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs +++ b/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs @@ -268,11 +268,6 @@ public override async Task DeleteDeadLetterEnvelopesAsync(Guid[] ids, string? te } } - public override void Describe(TextWriter writer) - { - writer.WriteLine($"Persistent Envelope storage using Postgresql in schema '{SchemaName}'"); - } - public override async Task DiscardAndReassignOutgoingAsync(Envelope[] discards, Envelope[] reassigned, int nodeId) { await using var cmd = CreateCommand(_discardAndReassignOutgoingSql) diff --git a/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs b/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs index b17f4851e..ecabc53ff 100644 --- a/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs +++ b/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs @@ -173,8 +173,6 @@ public void Initialize(IWolverineRuntime runtime) public IMessageStoreAdmin Admin => this; - public abstract void Describe(TextWriter writer); - public async Task DrainAsync() { if (_batcher != null) diff --git a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.cs b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.cs index a5d7716d6..08e2ff946 100644 --- a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.cs +++ b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.cs @@ -59,11 +59,6 @@ public void Initialize(IWolverineRuntime runtime) // NOTHING YET } - public void Describe(TextWriter writer) - { - writer.WriteLine("RavenDb backed Wolverine envelope storage"); - } - public DatabaseDescriptor Describe() { return new DatabaseDescriptor(this) diff --git a/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs b/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs index 424897dd9..9964e7333 100644 --- a/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs +++ b/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs @@ -195,11 +195,6 @@ public override Task DeleteDeadLetterEnvelopesAsync(Guid[] ids, string? tenantId return command.ExecuteNonQueryAsync(_cancellation); } - public override void Describe(TextWriter writer) - { - writer.WriteLine($"Sql Server Envelope Storage in Schema '{SchemaName}'"); - } - protected override string determineOutgoingEnvelopeSql(DurabilitySettings settings) { return diff --git a/src/Wolverine/Persistence/Durability/IMessageStore.cs b/src/Wolverine/Persistence/Durability/IMessageStore.cs index 5e47c111d..aceafc4d8 100644 --- a/src/Wolverine/Persistence/Durability/IMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/IMessageStore.cs @@ -115,9 +115,6 @@ public interface IMessageStore : IAsyncDisposable /// /// void Initialize(IWolverineRuntime runtime); - - [Obsolete("Eliminate this in 4.0")] - void Describe(TextWriter writer); DatabaseDescriptor Describe(); diff --git a/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs b/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs index 964f67ec1..ea3a1b549 100644 --- a/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs @@ -401,11 +401,6 @@ public void Initialize(IWolverineRuntime runtime) public INodeAgentPersistence Nodes => this; public IMessageStoreAdmin Admin => this; - public void Describe(TextWriter writer) - { - Main.Describe(writer); - } - public DatabaseDescriptor Describe() { return new DatabaseDescriptor(this) diff --git a/src/Wolverine/Persistence/Durability/NullMessageStore.cs b/src/Wolverine/Persistence/Durability/NullMessageStore.cs index 546eba7ad..7e7b9a795 100644 --- a/src/Wolverine/Persistence/Durability/NullMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/NullMessageStore.cs @@ -133,11 +133,6 @@ public void Initialize(IWolverineRuntime runtime) public IMessageStoreAdmin Admin => this; - public void Describe(TextWriter writer) - { - writer.WriteLine("No persistent envelope storage"); - } - public DatabaseDescriptor Describe() { return new DatabaseDescriptor(this); diff --git a/src/Wolverine/Persistence/StorageCommand.cs b/src/Wolverine/Persistence/StorageCommand.cs index 1c03f946e..34539c2c8 100644 --- a/src/Wolverine/Persistence/StorageCommand.cs +++ b/src/Wolverine/Persistence/StorageCommand.cs @@ -42,8 +42,6 @@ public override async Task Execute(StorageInput input) using var host = input.BuildHost(); var persistence = host.Services.GetRequiredService(); - persistence.Describe(Console.Out); - switch (input.Action) { case StorageCommandAction.counts: From 542e41b12197033285f5a641da037ed226c8f586 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Fri, 19 Sep 2025 08:33:46 -0500 Subject: [PATCH 17/25] Whipped up the Delegating message inbox & outbox for later --- .../Durability/DelegatingMessageInbox.cs | 79 +++++++++++++++++++ .../Durability/DelegatingMessageOutbox.cs | 49 ++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 src/Wolverine/Persistence/Durability/DelegatingMessageInbox.cs create mode 100644 src/Wolverine/Persistence/Durability/DelegatingMessageOutbox.cs diff --git a/src/Wolverine/Persistence/Durability/DelegatingMessageInbox.cs b/src/Wolverine/Persistence/Durability/DelegatingMessageInbox.cs new file mode 100644 index 000000000..2bf6f4fcc --- /dev/null +++ b/src/Wolverine/Persistence/Durability/DelegatingMessageInbox.cs @@ -0,0 +1,79 @@ +namespace Wolverine.Persistence.Durability; + +/// +/// This is necessary when the Wolverine system has "ancillary" message +/// stores that are in separate databases +/// +internal class DelegatingMessageInbox : IMessageInbox +{ + private readonly IMessageInbox _inner; + private readonly MessageStoreCollection _stores; + + public DelegatingMessageInbox(IMessageInbox inner, MessageStoreCollection stores) + { + _inner = inner; + _stores = stores; + } + + public Task RescheduleExistingEnvelopeForRetryAsync(Envelope envelope) + { + return (envelope.Store?.Inbox ?? _inner).RescheduleExistingEnvelopeForRetryAsync(envelope); + } + + public Task ScheduleExecutionAsync(Envelope envelope) + { + return (envelope.Store?.Inbox ?? _inner).ScheduleExecutionAsync(envelope); + } + + public Task MoveToDeadLetterStorageAsync(Envelope envelope, Exception? exception) + { + return (envelope.Store?.Inbox ?? _inner).MoveToDeadLetterStorageAsync(envelope, exception); + } + + public Task IncrementIncomingEnvelopeAttemptsAsync(Envelope envelope) + { + return (envelope.Store?.Inbox ?? _inner).IncrementIncomingEnvelopeAttemptsAsync(envelope); + } + + public Task StoreIncomingAsync(Envelope envelope) + { + return (envelope.Store?.Inbox ?? _inner).StoreIncomingAsync(envelope); + } + + // This would only be coming from a batch receipt and not from any kind of + // local queueing + public Task StoreIncomingAsync(IReadOnlyList envelopes) + { + return _inner.StoreIncomingAsync(envelopes); + } + + public Task MarkIncomingEnvelopeAsHandledAsync(Envelope envelope) + { + return (envelope.Store?.Inbox ?? _inner).MarkIncomingEnvelopeAsHandledAsync(envelope); + } + + public async Task MarkIncomingEnvelopeAsHandledAsync(IReadOnlyList envelopes) + { + // Going to purposely leave this naive and let the whole thing be retried + var groups = envelopes.GroupBy(x => x.Store).ToList(); + if (groups.Count == 1) + { + await (groups[0].Key?.Inbox ?? _inner).MarkIncomingEnvelopeAsHandledAsync(envelopes); + return; + } + + foreach (var group in groups) + { + await (group.Key?.Inbox ?? _inner).MarkIncomingEnvelopeAsHandledAsync(group.ToList()); + } + } + + public async Task ReleaseIncomingAsync(int ownerId, Uri receivedAt) + { + var databases = await _stores.FindAllAsync(); + foreach (var database in databases) + { + await database.Inbox.ReleaseIncomingAsync(ownerId, receivedAt); + } + } +} \ No newline at end of file diff --git a/src/Wolverine/Persistence/Durability/DelegatingMessageOutbox.cs b/src/Wolverine/Persistence/Durability/DelegatingMessageOutbox.cs new file mode 100644 index 000000000..289f966aa --- /dev/null +++ b/src/Wolverine/Persistence/Durability/DelegatingMessageOutbox.cs @@ -0,0 +1,49 @@ +namespace Wolverine.Persistence.Durability; + +internal class DelegatingMessageOutbox : IMessageOutbox +{ + private readonly IMessageOutbox _inner; + private readonly MessageStoreCollection _stores; + + public DelegatingMessageOutbox(IMessageOutbox inner, MessageStoreCollection stores) + { + _inner = inner; + _stores = stores; + } + + public Task> LoadOutgoingAsync(Uri destination) + { + return _inner.LoadOutgoingAsync(destination); + } + + public Task StoreOutgoingAsync(Envelope envelope, int ownerId) + { + return (envelope.Store?.Outbox ?? _inner).StoreOutgoingAsync(envelope, ownerId); + } + + public async Task DeleteOutgoingAsync(Envelope[] envelopes) + { + // Going to purposely leave this naive and let the whole thing be retried + var groups = envelopes.GroupBy(x => x.Store).ToList(); + if (groups.Count == 1) + { + await (groups[0].Key?.Outbox ?? _inner).DeleteOutgoingAsync(envelopes); + return; + } + + foreach (var group in groups) + { + await (group.Key?.Outbox ?? _inner).DeleteOutgoingAsync(group.ToArray()); + } + } + + public Task DeleteOutgoingAsync(Envelope envelope) + { + return (envelope.Store?.Outbox ?? _inner).DeleteOutgoingAsync(envelope); + } + + public Task DiscardAndReassignOutgoingAsync(Envelope[] discards, Envelope[] reassigned, int nodeId) + { + return _inner.DiscardAndReassignOutgoingAsync(discards, reassigned, nodeId); + } +} \ No newline at end of file From 9defcfe4b9765881fd27434484f63754e37fb260 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Fri, 19 Sep 2025 09:13:01 -0500 Subject: [PATCH 18/25] Little preparation for more support for ancillary message stores --- src/Wolverine/Envelope.cs | 2 +- src/Wolverine/Runtime/MessageBus.cs | 1 + src/Wolverine/Runtime/WorkerQueues/DurableReceiver.cs | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Wolverine/Envelope.cs b/src/Wolverine/Envelope.cs index bde7e27f7..9a79c2600 100644 --- a/src/Wolverine/Envelope.cs +++ b/src/Wolverine/Envelope.cs @@ -450,5 +450,5 @@ internal string GetMessageTypeName() /// For some forms of modular monoliths, Wolverine needs to track what message store /// persisted this envelope for later tracking /// - public IMessageStore? Store { get; private set; } + internal IMessageStore? Store { get; set; } } \ No newline at end of file diff --git a/src/Wolverine/Runtime/MessageBus.cs b/src/Wolverine/Runtime/MessageBus.cs index 7bfdf4a7f..c5c41dfe9 100644 --- a/src/Wolverine/Runtime/MessageBus.cs +++ b/src/Wolverine/Runtime/MessageBus.cs @@ -273,6 +273,7 @@ internal virtual void TrackEnvelopeCorrelation(Envelope outbound, Activity? acti outbound.ConversationId = outbound.Id; // the message chain originates here outbound.TenantId ??= TenantId; // don't override a tenant id that's specifically set on the envelope itself outbound.ParentId = activity?.Id; + outbound.Store = Storage; } internal async ValueTask PersistOrSendAsync(params Envelope[] outgoing) diff --git a/src/Wolverine/Runtime/WorkerQueues/DurableReceiver.cs b/src/Wolverine/Runtime/WorkerQueues/DurableReceiver.cs index 1d3c457f3..d2bda4ca5 100644 --- a/src/Wolverine/Runtime/WorkerQueues/DurableReceiver.cs +++ b/src/Wolverine/Runtime/WorkerQueues/DurableReceiver.cs @@ -37,6 +37,9 @@ public DurableReceiver(Endpoint endpoint, IWolverineRuntime runtime, IHandlerPip { _endpoint = endpoint; _settings = runtime.DurabilitySettings; + + // HERE, HERE, HERE + _inbox = runtime.Storage.Inbox; _logger = runtime.LoggerFactory.CreateLogger(); From 491f30ba303c299857848c625d7f364ef339eea3 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Fri, 19 Sep 2025 09:35:36 -0500 Subject: [PATCH 19/25] Simplifying IDeadLetters before doing more work --- .../PostgresqlTests/PostgresqlMessageStoreTests.cs | 3 +-- ...stgresqlMessageStore_with_IdAndDestination_Identity.cs | 3 +-- .../Persistence/SqlServerMessageStoreTests.cs | 3 +-- ...qlServerMessageStore_with_IdAndDestination_Identity.cs | 5 ++--- .../Wolverine.RDBMS/MessageDatabase.DeadLetters.cs | 4 ++-- .../Internals/RavenDbMessageStore.DeadLetters.cs | 4 +--- src/Wolverine/Persistence/Durability/IDeadLetters.cs | 2 +- .../Persistence/Durability/MultiTenantedMessageStore.cs | 8 ++------ src/Wolverine/Persistence/Durability/NullMessageStore.cs | 2 +- src/Wolverine/Persistence/StorageCommand.cs | 5 ++--- 10 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/Persistence/PostgresqlTests/PostgresqlMessageStoreTests.cs b/src/Persistence/PostgresqlTests/PostgresqlMessageStoreTests.cs index 5a992f104..8592a9b0e 100644 --- a/src/Persistence/PostgresqlTests/PostgresqlMessageStoreTests.cs +++ b/src/Persistence/PostgresqlTests/PostgresqlMessageStoreTests.cs @@ -136,7 +136,7 @@ public async Task move_replayable_error_messages_to_incoming() await thePersistence.Inbox.MoveToDeadLetterStorageAsync(replayableEnvelope, applicationException); // make one of the messages(DivideByZeroException) replayable - var replayableErrorMessagesCountAfterMakingReplayable = await thePersistence + await thePersistence .DeadLetters .MarkDeadLetterEnvelopesAsReplayableAsync(divideByZeroException.GetType().FullName!); @@ -147,7 +147,6 @@ public async Task move_replayable_error_messages_to_incoming() var counts = await thePersistence.Admin.FetchCountsAsync(); - replayableErrorMessagesCountAfterMakingReplayable.ShouldBe(1); counts.DeadLetter.ShouldBe(1); counts.Incoming.ShouldBe(1); counts.Scheduled.ShouldBe(0); diff --git a/src/Persistence/PostgresqlTests/PostgresqlMessageStore_with_IdAndDestination_Identity.cs b/src/Persistence/PostgresqlTests/PostgresqlMessageStore_with_IdAndDestination_Identity.cs index ba6cba0a5..60714a318 100644 --- a/src/Persistence/PostgresqlTests/PostgresqlMessageStore_with_IdAndDestination_Identity.cs +++ b/src/Persistence/PostgresqlTests/PostgresqlMessageStore_with_IdAndDestination_Identity.cs @@ -106,7 +106,7 @@ public async Task move_replayable_error_messages_to_incoming() await thePersistence.Inbox.MoveToDeadLetterStorageAsync(replayableEnvelope, applicationException); // make one of the messages(DivideByZeroException) replayable - var replayableErrorMessagesCountAfterMakingReplayable = await thePersistence + await thePersistence .DeadLetters .MarkDeadLetterEnvelopesAsReplayableAsync(divideByZeroException.GetType().FullName!); @@ -117,7 +117,6 @@ public async Task move_replayable_error_messages_to_incoming() var counts = await thePersistence.Admin.FetchCountsAsync(); - replayableErrorMessagesCountAfterMakingReplayable.ShouldBe(1); counts.DeadLetter.ShouldBe(1); counts.Incoming.ShouldBe(1); counts.Scheduled.ShouldBe(0); diff --git a/src/Persistence/SqlServerTests/Persistence/SqlServerMessageStoreTests.cs b/src/Persistence/SqlServerTests/Persistence/SqlServerMessageStoreTests.cs index 733001692..3655b3b51 100644 --- a/src/Persistence/SqlServerTests/Persistence/SqlServerMessageStoreTests.cs +++ b/src/Persistence/SqlServerTests/Persistence/SqlServerMessageStoreTests.cs @@ -78,7 +78,7 @@ public async Task move_replayable_error_messages_to_incoming() await thePersistence.Inbox.MoveToDeadLetterStorageAsync(replayableEnvelope, applicationException); // make one of the messages(DivideByZeroException) replayable - var replayableErrorMessagesCountAfterMakingReplayable = await thePersistence + await thePersistence .DeadLetters .MarkDeadLetterEnvelopesAsReplayableAsync(divideByZeroException.GetType().FullName!); @@ -89,7 +89,6 @@ public async Task move_replayable_error_messages_to_incoming() var counts = await thePersistence.Admin.FetchCountsAsync(); - replayableErrorMessagesCountAfterMakingReplayable.ShouldBe(1); counts.DeadLetter.ShouldBe(1); counts.Incoming.ShouldBe(1); counts.Scheduled.ShouldBe(0); diff --git a/src/Persistence/SqlServerTests/Persistence/SqlServerMessageStore_with_IdAndDestination_Identity.cs b/src/Persistence/SqlServerTests/Persistence/SqlServerMessageStore_with_IdAndDestination_Identity.cs index d2f6a33f3..ede3672ca 100644 --- a/src/Persistence/SqlServerTests/Persistence/SqlServerMessageStore_with_IdAndDestination_Identity.cs +++ b/src/Persistence/SqlServerTests/Persistence/SqlServerMessageStore_with_IdAndDestination_Identity.cs @@ -105,7 +105,7 @@ public async Task move_replayable_error_messages_to_incoming() await thePersistence.Inbox.MoveToDeadLetterStorageAsync(replayableEnvelope, applicationException); // make one of the messages(DivideByZeroException) replayable - var replayableErrorMessagesCountAfterMakingReplayable = await thePersistence + await thePersistence .DeadLetters .MarkDeadLetterEnvelopesAsReplayableAsync(divideByZeroException.GetType().FullName!); @@ -115,8 +115,7 @@ public async Task move_replayable_error_messages_to_incoming() await theHost.InvokeAsync(batch); var counts = await thePersistence.Admin.FetchCountsAsync(); - - replayableErrorMessagesCountAfterMakingReplayable.ShouldBe(1); + counts.DeadLetter.ShouldBe(1); counts.Incoming.ShouldBe(1); counts.Scheduled.ShouldBe(0); diff --git a/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetters.cs b/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetters.cs index 89d3f4841..baceb47da 100644 --- a/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetters.cs +++ b/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetters.cs @@ -114,7 +114,7 @@ public async Task QueryDeadLetterEnvelopesAsync(DeadLe return deadLetterEnvelope; } - public async Task MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType) + public Task MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType) { var sql = $"update {SchemaName}.{DatabaseConstants.DeadLetterTable} set {DatabaseConstants.Replayable} = @replay"; @@ -124,7 +124,7 @@ public async Task MarkDeadLetterEnvelopesAsReplayableAsync(string exception sql = $"{sql} where {DatabaseConstants.ExceptionType} = @extype"; } - return await CreateCommand(sql).With("replay", true).With("extype", exceptionType) + return CreateCommand(sql).With("replay", true).With("extype", exceptionType) .ExecuteNonQueryAsync(_cancellation); } diff --git a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.DeadLetters.cs b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.DeadLetters.cs index 033337a55..cfdca0c86 100644 --- a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.DeadLetters.cs +++ b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.DeadLetters.cs @@ -75,7 +75,7 @@ public async Task QueryDeadLetterEnvelopesAsync(DeadLe return message.ToEnvelope(); } - public async Task MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType = "") + public async Task MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType = "") { using var session = _store.OpenAsyncSession(); var count = exceptionType.IsEmpty() @@ -111,8 +111,6 @@ from DeadLetterMessages as m QueryParameters = new(){{"exceptionType", exceptionType}} })); await op.WaitForCompletionAsync(); - - return count; } public async Task MarkDeadLetterEnvelopesAsReplayableAsync(Guid[] ids, string? tenantId = null) diff --git a/src/Wolverine/Persistence/Durability/IDeadLetters.cs b/src/Wolverine/Persistence/Durability/IDeadLetters.cs index 196dd6c34..cd8af3465 100644 --- a/src/Wolverine/Persistence/Durability/IDeadLetters.cs +++ b/src/Wolverine/Persistence/Durability/IDeadLetters.cs @@ -19,7 +19,7 @@ public interface IDeadLetters /// Exception Type that should be marked. Default is any. /// Number of envelopes marked. [Obsolete("Prefer ReplayAsync")] - Task MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType = ""); + Task MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType = ""); /// /// Marks the Envelope in DeadLetterTable diff --git a/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs b/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs index ea3a1b549..7b7736c19 100644 --- a/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs @@ -56,15 +56,13 @@ public MultiTenantedMessageStore(IMessageStore main, IWolverineRuntime runtime, public IMessageStore Main { get; } - async Task IDeadLetters.MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType) + async Task IDeadLetters.MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType) { - var size = 0; - foreach (var database in databases()) { try { - size += await database.DeadLetters.MarkDeadLetterEnvelopesAsReplayableAsync(exceptionType); + await database.DeadLetters.MarkDeadLetterEnvelopesAsReplayableAsync(exceptionType); } catch (Exception e) { @@ -72,8 +70,6 @@ async Task IDeadLetters.MarkDeadLetterEnvelopesAsReplayableAsync(string exc database.Name); } } - - return size; } public async Task MarkDeadLetterEnvelopesAsReplayableAsync(Guid[] ids, string? tenantId = null) diff --git a/src/Wolverine/Persistence/Durability/NullMessageStore.cs b/src/Wolverine/Persistence/Durability/NullMessageStore.cs index 7e7b9a795..f3d5f90ef 100644 --- a/src/Wolverine/Persistence/Durability/NullMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/NullMessageStore.cs @@ -195,7 +195,7 @@ public Task ClearAllAsync() return Task.CompletedTask; } - public Task MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType) + public Task MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType) { return Task.FromResult(0); } diff --git a/src/Wolverine/Persistence/StorageCommand.cs b/src/Wolverine/Persistence/StorageCommand.cs index 34539c2c8..8c98ab9f7 100644 --- a/src/Wolverine/Persistence/StorageCommand.cs +++ b/src/Wolverine/Persistence/StorageCommand.cs @@ -78,13 +78,12 @@ public override async Task Execute(StorageInput input) break; case StorageCommandAction.replay: - var markedCount = - await persistence.DeadLetters.MarkDeadLetterEnvelopesAsReplayableAsync(input.ExceptionTypeForReplayFlag); + await persistence.DeadLetters.MarkDeadLetterEnvelopesAsReplayableAsync(input.ExceptionTypeForReplayFlag); var exceptionType = string.IsNullOrEmpty(input.ExceptionTypeForReplayFlag) ? "any" : input.ExceptionTypeForReplayFlag; AnsiConsole.Write( - $"[green]Successfully replayed {markedCount} envelope(s) in dead letter with exception type '{exceptionType}'"); + $"[green]Successfully replayed envelope(s) in dead letter with exception type '{exceptionType}'"); break; } From 748af3f2dc4a6484736d22f9191108d21ca3f0e2 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Fri, 19 Sep 2025 13:51:46 -0500 Subject: [PATCH 20/25] Ton of work toward consolidating the DLQ admin service endpoints --- .../DeadLettersEndpointExtensions.cs | 85 ++++++++----- ...play_dead_letter_queue_of_event_wrapper.cs | 11 +- .../cross_database_message_storage.cs | 2 +- .../modular_monolith_usage.cs | 6 + .../PostgresqlMessageStoreTests.cs | 1 + ...ageStore_with_IdAndDestination_Identity.cs | 1 + .../Transport/external_message_tables.cs | 4 +- .../PostgresqlMessageStore.cs | 43 +------ .../MessageDatabase.DeadLetterAdminService.cs | 14 ++- .../MessageDatabase.DeadLetters.cs | 108 ----------------- .../RavenDbMessageStore.DeadLetters.cs | 112 ++++++++---------- .../Persistence/SqlServerMessageStore.cs | 37 +----- .../MessageStoreCompliance.cs | 98 ++++----------- .../Bugs/Bug_1594_ReplayDeadLetterQueue.cs | 17 ++- .../Bugs/Bug_DLQ_NotSavedToDatabase.cs | 10 +- ...ine_storage_dead_letter_queue_mechanics.cs | 18 +-- .../DeadLetterEnvelopeQuery.cs | 13 +- .../DeadLetterEnvelopeQueryParameters.cs | 12 -- .../DeadLetterEnvelopeResults.cs | 1 + .../Persistence/Durability/IDeadLetters.cs | 47 ++++---- .../Durability/MultiTenantedMessageStore.cs | 55 +-------- .../Durability/NullMessageStore.cs | 12 -- .../Persistence/MessageStoreCollection.cs | 50 ++++++++ 23 files changed, 270 insertions(+), 487 deletions(-) delete mode 100644 src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQueryParameters.cs diff --git a/src/Http/Wolverine.Http/DeadLettersEndpointExtensions.cs b/src/Http/Wolverine.Http/DeadLettersEndpointExtensions.cs index 98e360f0c..430b53d7f 100644 --- a/src/Http/Wolverine.Http/DeadLettersEndpointExtensions.cs +++ b/src/Http/Wolverine.Http/DeadLettersEndpointExtensions.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Options; using Wolverine.Persistence.Durability; using Wolverine.Persistence.Durability.DeadLetterManagement; +using Wolverine.Runtime; using Wolverine.Runtime.Handlers; namespace Wolverine.Http; @@ -15,10 +16,9 @@ public class DeadLetterEnvelopeGetRequest /// Number of records to return per page. /// public uint Limit { get; set; } = 100; - /// - /// Fetch records starting after the record with this ID. - /// - public Guid? StartId { get; set; } + + public int PageNumber { get; set; } + public string? MessageType { get; set; } public string? ExceptionType { get; set; } public string? ExceptionMessage { get; set; } @@ -63,38 +63,67 @@ public static RouteGroupBuilder MapDeadLettersEndpoints(this IEndpointRouteBuild deadlettersGroup.MapPost("/", async (DeadLetterEnvelopeGetRequest request, IMessageStore messageStore, HandlerGraph handlerGraph, IOptions opts) => { var deadLetters = messageStore.DeadLetters; - var queryParameters = new DeadLetterEnvelopeQueryParameters + var queryParameters = new DeadLetterEnvelopeQuery { - Limit = request.Limit, - StartId = request.StartId, + PageSize = (int)request.Limit, + PageNumber = request.PageNumber, MessageType = request.MessageType, ExceptionType = request.ExceptionType, ExceptionMessage = request.ExceptionMessage, - From = request.From, - Until = request.Until + Range = new TimeRange(request.From, request.Until) }; - var deadLetterEnvelopesFound = await deadLetters.QueryDeadLetterEnvelopesAsync(queryParameters, request.TenantId); - return new DeadLetterEnvelopesFoundResponse( - [.. deadLetterEnvelopesFound.DeadLetterEnvelopes.Select(x => new DeadLetterEnvelopeResponse( - x.Id, - x.ExecutionTime, - handlerGraph.TryFindMessageType(x.MessageType, out var messageType) ? opts.Value.DetermineSerializer(x.Envelope).ReadFromData(messageType, x.Envelope) : null, - x.MessageType, - x.ReceivedAt, - x.Source, - x.ExceptionType, - x.ExceptionMessage, - x.SentAt, - x.Replayable)) - ], - deadLetterEnvelopesFound.NextId); + + throw new NotImplementedException(); + // if (request.TenantId.IsNotEmpty()) + // { + // + // } + // else + // { + // var deadLetterEnvelopesFound = await deadLetters.QueryAsync(queryParameters, CancellationToken.None); + // } + // + // var deadLetterEnvelopesFound = await deadLetters.QueryAsync(queryParameters, request.TenantId); + // return new DeadLetterEnvelopesFoundResponse( + // [.. deadLetterEnvelopesFound.Select(x => new DeadLetterEnvelopeResponse( + // x.Id, + // x.ExecutionTime, + // handlerGraph.TryFindMessageType(x.MessageType, out var messageType) ? opts.Value.DetermineSerializer(x.Envelope).ReadFromData(messageType, x.Envelope) : null, + // x.MessageType, + // x.ReceivedAt, + // x.Source, + // x.ExceptionType, + // x.ExceptionMessage, + // x.SentAt, + // x.Replayable)) + // ], + // deadLetterEnvelopesFound.PageNumber); }); - deadlettersGroup.MapPost("/replay", (DeadLetterEnvelopeIdsRequest request, IMessageStore messageStore) => - messageStore.DeadLetters.MarkDeadLetterEnvelopesAsReplayableAsync(request.Ids, request.TenantId)); + deadlettersGroup.MapPost("/replay", (DeadLetterEnvelopeIdsRequest request, IWolverineRuntime runtime) => + { + if (request.TenantId.IsEmpty()) + { + return runtime.Stores.ReplayDeadLettersAsync(request.Ids); + } + else + { + return runtime.Stores.ReplayDeadLettersAsync(request.TenantId, request.Ids); + } + }); + - deadlettersGroup.MapDelete("/", ([FromBody]DeadLetterEnvelopeIdsRequest request, IMessageStore messageStore) => - messageStore.DeadLetters.DeleteDeadLetterEnvelopesAsync(request.Ids, request.TenantId)); + deadlettersGroup.MapDelete("/", ([FromBody] DeadLetterEnvelopeIdsRequest request, IWolverineRuntime runtime) => + { + if (request.TenantId.IsEmpty()) + { + return runtime.Stores.DiscardDeadLettersAsync(request.Ids); + } + else + { + return runtime.Stores.DiscardDeadLettersAsync(request.TenantId, request.Ids); + } + }); return deadlettersGroup; } diff --git a/src/Persistence/MartenTests/Bugs/Bug_971_replay_dead_letter_queue_of_event_wrapper.cs b/src/Persistence/MartenTests/Bugs/Bug_971_replay_dead_letter_queue_of_event_wrapper.cs index ae0986da1..d2dfa660d 100644 --- a/src/Persistence/MartenTests/Bugs/Bug_971_replay_dead_letter_queue_of_event_wrapper.cs +++ b/src/Persistence/MartenTests/Bugs/Bug_971_replay_dead_letter_queue_of_event_wrapper.cs @@ -10,6 +10,7 @@ using Wolverine; using Wolverine.ErrorHandling; using Wolverine.Marten; +using Wolverine.Persistence.Durability; using Wolverine.Persistence.Durability.DeadLetterManagement; using Wolverine.Runtime.Handlers; using Wolverine.Tracking; @@ -59,20 +60,20 @@ public async Task can_replay_dead_letter_event() { count++; var messages = - await runtime.Storage.DeadLetters.QueryDeadLetterEnvelopesAsync( - new DeadLetterEnvelopeQueryParameters()); + await runtime.Storage.DeadLetters.QueryAsync( + new DeadLetterEnvelopeQuery(), CancellationToken.None); - if (hasReplayed && !messages.DeadLetterEnvelopes.Any()) + if (hasReplayed && !messages.Envelopes.Any()) { break; // we're done! } - if (messages.DeadLetterEnvelopes.Any(x => !x.Replayable)) + if (messages.Envelopes.Any(x => !x.Replayable)) { ErrorCausingEventHandler.ShouldThrow = false; await runtime.Storage.DeadLetters.MarkDeadLetterEnvelopesAsReplayableAsync(messages - .DeadLetterEnvelopes.Select(x => x.Id).ToArray()); + .Envelopes.Select(x => x.Id).ToArray()); hasReplayed = true; } diff --git a/src/Persistence/MartenTests/MultiTenancy/cross_database_message_storage.cs b/src/Persistence/MartenTests/MultiTenancy/cross_database_message_storage.cs index 6c1ea0022..2e7661478 100644 --- a/src/Persistence/MartenTests/MultiTenancy/cross_database_message_storage.cs +++ b/src/Persistence/MartenTests/MultiTenancy/cross_database_message_storage.cs @@ -2,6 +2,7 @@ using Shouldly; using Wolverine.ComplianceTests; using Wolverine; +using Wolverine.Persistence.Durability; using Wolverine.Runtime.Agents; using Wolverine.Transports; @@ -809,7 +810,6 @@ public async Task MoveToDeadLetterStorageAsync_smoke_test() await Stores.Inbox.MoveToDeadLetterStorageAsync(envelopes[7], new NotImplementedException()); await Stores.Inbox.MoveToDeadLetterStorageAsync(envelopes[15], new NotImplementedException()); - await Stores.DeadLetters.MarkDeadLetterEnvelopesAsReplayableAsync(typeof(NotImplementedException).FullName); } [Fact] diff --git a/src/Persistence/PersistenceTests/modular_monolith_usage.cs b/src/Persistence/PersistenceTests/modular_monolith_usage.cs index e2bcc4a26..369e3bcdc 100644 --- a/src/Persistence/PersistenceTests/modular_monolith_usage.cs +++ b/src/Persistence/PersistenceTests/modular_monolith_usage.cs @@ -1,5 +1,6 @@ using IntegrationTests; using JasperFx.Core.Reflection; +using JasperFx.Resources; using Marten; using Microsoft.Extensions.Hosting; using Shouldly; @@ -89,6 +90,9 @@ public async Task do_not_override_when_the_schema_name_is_explicitly_set() { opts.Durability.MessageStorageSchemaName = "wolverine"; + // This declares to Wolverine what the "main" + //opts.PersistMessagesWithPostgresql(Servers.PostgresConnectionString, "different"); + opts.Services.AddMartenStore(m => { m.Connection(Servers.PostgresConnectionString); @@ -100,6 +104,8 @@ public async Task do_not_override_when_the_schema_name_is_explicitly_set() m.Connection(Servers.PostgresConnectionString); m.DatabaseSchemaName = "things"; }).IntegrateWithWolverine(schemaName:"different"); + + opts.Services.AddResourceSetupOnStartup(); }).StartAsync(); var runtime = host.GetRuntime(); diff --git a/src/Persistence/PostgresqlTests/PostgresqlMessageStoreTests.cs b/src/Persistence/PostgresqlTests/PostgresqlMessageStoreTests.cs index 8592a9b0e..ec078aff2 100644 --- a/src/Persistence/PostgresqlTests/PostgresqlMessageStoreTests.cs +++ b/src/Persistence/PostgresqlTests/PostgresqlMessageStoreTests.cs @@ -9,6 +9,7 @@ using Wolverine; using Wolverine.ComplianceTests; using Wolverine.Marten; +using Wolverine.Persistence.Durability; using Wolverine.RDBMS; using Wolverine.RDBMS.Durability; using Wolverine.RDBMS.Polling; diff --git a/src/Persistence/PostgresqlTests/PostgresqlMessageStore_with_IdAndDestination_Identity.cs b/src/Persistence/PostgresqlTests/PostgresqlMessageStore_with_IdAndDestination_Identity.cs index 60714a318..87714d038 100644 --- a/src/Persistence/PostgresqlTests/PostgresqlMessageStore_with_IdAndDestination_Identity.cs +++ b/src/Persistence/PostgresqlTests/PostgresqlMessageStore_with_IdAndDestination_Identity.cs @@ -8,6 +8,7 @@ using Wolverine; using Wolverine.ComplianceTests; using Wolverine.Marten; +using Wolverine.Persistence.Durability; using Wolverine.Postgresql.Schema; using Wolverine.RDBMS; using Wolverine.RDBMS.Durability; diff --git a/src/Persistence/PostgresqlTests/Transport/external_message_tables.cs b/src/Persistence/PostgresqlTests/Transport/external_message_tables.cs index 694c91adc..4aaeff253 100644 --- a/src/Persistence/PostgresqlTests/Transport/external_message_tables.cs +++ b/src/Persistence/PostgresqlTests/Transport/external_message_tables.cs @@ -213,8 +213,8 @@ public async Task pull_in_message_that_goes_to_dead_letter_queue_and_replay_it() Guid[] ids = new Guid[0]; while (!ids.Any()) { - var queued = await storage.DeadLetters.QueryDeadLetterEnvelopesAsync(new DeadLetterEnvelopeQueryParameters()); - ids = queued.DeadLetterEnvelopes.Select(x => x.Envelope.Id).ToArray(); + var queued = await storage.DeadLetters.QueryAsync(new DeadLetterEnvelopeQuery(TimeRange.AllTime()), CancellationToken.None); + ids = queued.Envelopes.Select(x => x.Envelope.Id).ToArray(); } // need to reset it diff --git a/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs b/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs index aeec51973..6631a7ac0 100644 --- a/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs +++ b/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs @@ -226,47 +226,6 @@ public override async Task FetchCountsAsync() return counts; } - - public override async Task MarkDeadLetterEnvelopesAsReplayableAsync(Guid[] ids, string? tenantId = null) - { - var builder = ToCommandBuilder(); - builder.Append($"update {SchemaName}.{DatabaseConstants.DeadLetterTable} set {DatabaseConstants.Replayable} = @replay where id = ANY(@ids)"); - - var cmd = builder.Compile(); - cmd.With("replay", true); - var param = new NpgsqlParameter("ids", NpgsqlDbType.Array | NpgsqlDbType.Uuid) { Value = ids }; - cmd.Parameters.Add(param); - await using var conn = await NpgsqlDataSource.OpenConnectionAsync(_cancellation); - cmd.Connection = conn; - try - { - await cmd.ExecuteNonQueryAsync(_cancellation); - } - finally - { - await conn.CloseAsync(); - } - } - - public override async Task DeleteDeadLetterEnvelopesAsync(Guid[] ids, string? tenantId = null) - { - var builder = ToCommandBuilder(); - builder.Append($"delete from {SchemaName}.{DatabaseConstants.DeadLetterTable} where id = ANY(@ids)"); - - var cmd = builder.Compile(); - var param = new NpgsqlParameter("ids", NpgsqlDbType.Array | NpgsqlDbType.Uuid) { Value = ids }; - cmd.Parameters.Add(param); - await using var conn = await NpgsqlDataSource.OpenConnectionAsync(_cancellation); - cmd.Connection = conn; - try - { - await cmd.ExecuteNonQueryAsync(_cancellation); - } - finally - { - await conn.CloseAsync(); - } - } public override async Task DiscardAndReassignOutgoingAsync(Envelope[] discards, Envelope[] reassigned, int nodeId) { @@ -475,7 +434,7 @@ public override IEnumerable AllObjects() yield return table; } - if (_settings.Role == MessageStoreRole.Main) + if (Role == MessageStoreRole.Main) { var nodeTable = new Table(new DbObjectName(SchemaName, DatabaseConstants.NodeTableName)); nodeTable.AddColumn("id").AsPrimaryKey(); diff --git a/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs b/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs index 48fc635a0..280e8e868 100644 --- a/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs +++ b/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetterAdminService.cs @@ -132,7 +132,13 @@ private void writeDeadLetterWhereClause(DeadLetterEnvelopeQuery query, DbCommand builder.Append($" and {DatabaseConstants.ExceptionType} = "); builder.AppendParameter(query.ExceptionType); } - + + if (query.ExceptionMessage.IsNotEmpty()) + { + builder.Append($" and {DatabaseConstants.ExceptionMessage} LIKE "); + builder.AppendParameter(query.ExceptionMessage); + } + if (query.MessageType.IsNotEmpty()) { builder.Append($" and {DatabaseConstants.MessageType} = "); @@ -176,11 +182,7 @@ public Task ReplayAsync(DeadLetterEnvelopeQuery query, CancellationToken token) builder.Append(" where 1 = 1"); writeDeadLetterWhereClause(query, builder); builder.Append(';'); - builder.Append( - $"delete from {SchemaName}.{DatabaseConstants.DeadLetterTable} where {DatabaseConstants.Replayable} = "); - builder.AppendParameter(true); - builder.Append(';'); - + return executeCommandBatch(builder, token); } diff --git a/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetters.cs b/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetters.cs index baceb47da..dc9e87a41 100644 --- a/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetters.cs +++ b/src/Persistence/Wolverine.RDBMS/MessageDatabase.DeadLetters.cs @@ -6,95 +6,6 @@ namespace Wolverine.RDBMS; public abstract partial class MessageDatabase { - public async Task QueryDeadLetterEnvelopesAsync(DeadLetterEnvelopeQueryParameters queryParameters, string? tenantId) - { - var query = $"select {DatabaseConstants.DeadLetterFields} from {SchemaName}.{DatabaseConstants.DeadLetterTable} where 1 = 1"; - - if (!string.IsNullOrEmpty(queryParameters.ExceptionType)) - { - query += $" and {DatabaseConstants.ExceptionType} = @exceptionType"; - } - - if (!string.IsNullOrEmpty(queryParameters.ExceptionMessage)) - { - query += $" and {DatabaseConstants.ExceptionMessage} LIKE @exceptionMessage"; - } - - if (!string.IsNullOrEmpty(queryParameters.MessageType)) - { - query += $" and {DatabaseConstants.MessageType} = @messageType"; - } - - if (queryParameters.From.HasValue) - { - query += $" and {DatabaseConstants.SentAt} >= @from"; - } - - if (queryParameters.Until.HasValue) - { - query += $" and {DatabaseConstants.SentAt} <= @until"; - } - - if (queryParameters.StartId.HasValue) - { - query += $" and {DatabaseConstants.Id} >= @startId"; - } - - if (queryParameters.Limit > 0) - { - query += " limit @limit"; - } - - var command = CreateCommand(query); - - if (!string.IsNullOrEmpty(queryParameters.ExceptionType)) - { - command = command.With("exceptionType", queryParameters.ExceptionType); - } - - if (!string.IsNullOrEmpty(queryParameters.ExceptionMessage)) - { - command = command.With("exceptionMessage", queryParameters.ExceptionMessage); - } - - if (!string.IsNullOrEmpty(queryParameters.MessageType)) - { - command = command.With("messageType", queryParameters.MessageType); - } - - if (queryParameters.From.HasValue) - { - command = command.With("from", queryParameters.From.Value.ToUniversalTime()); - } - - if (queryParameters.Until.HasValue) - { - command = command.With("until", queryParameters.Until.Value.ToUniversalTime()); - } - - if (queryParameters.StartId.HasValue) - { - command = command.With("startId", queryParameters.StartId.Value); - } - - if (queryParameters.Limit > 0) - { - command = command.With("limit", queryParameters.Limit + 1); - } - - var deadLetterEnvelopes = (List)await command.FetchListAsync(reader => - DatabasePersistence.ReadDeadLetterAsync(reader, _cancellation), cancellation: _cancellation); - - if (queryParameters.Limit > 0 && deadLetterEnvelopes.Count > queryParameters.Limit) - { - var nextId = deadLetterEnvelopes.LastOrDefault()?.Envelope.Id; - deadLetterEnvelopes.RemoveAt(deadLetterEnvelopes.Count - 1); - return new(deadLetterEnvelopes, nextId, tenantId); - } - - return new(deadLetterEnvelopes, null, tenantId); - } - public async Task DeadLetterEnvelopeByIdAsync(Guid id, string? tenantId = null) { await using var reader = await CreateCommand( @@ -114,24 +25,5 @@ public async Task QueryDeadLetterEnvelopesAsync(DeadLe return deadLetterEnvelope; } - public Task MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType) - { - var sql = - $"update {SchemaName}.{DatabaseConstants.DeadLetterTable} set {DatabaseConstants.Replayable} = @replay"; - - if (!string.IsNullOrEmpty(exceptionType)) - { - sql = $"{sql} where {DatabaseConstants.ExceptionType} = @extype"; - } - - return CreateCommand(sql).With("replay", true).With("extype", exceptionType) - .ExecuteNonQueryAsync(_cancellation); - } - - public abstract Task MarkDeadLetterEnvelopesAsReplayableAsync(Guid[] ids, string? tenantId = null); - - public abstract Task DeleteDeadLetterEnvelopesAsync(Guid[] ids, string? tenantId = null); - - } diff --git a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.DeadLetters.cs b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.DeadLetters.cs index cfdca0c86..31831b57e 100644 --- a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.DeadLetters.cs +++ b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.DeadLetters.cs @@ -15,56 +15,56 @@ private static string dlqId(Guid id) return $"dlq/{id}"; } - public async Task QueryDeadLetterEnvelopesAsync(DeadLetterEnvelopeQueryParameters queryParameters, string? tenantId = null) - { - using var session = _store.OpenAsyncSession(); - var queryable = session.Query().Customize(x => x.WaitForNonStaleResults()); - if (queryParameters.StartId.HasValue) - { - queryable = (IRavenQueryable)Queryable.Where(queryable, x => x.EnvelopeId >= queryParameters.StartId.Value); - } - - if (queryParameters.MessageType.IsNotEmpty()) - { - queryable = (IRavenQueryable)Queryable.Where(queryable, x => x.MessageType == queryParameters.MessageType); - } - - if (queryParameters.ExceptionType.IsNotEmpty()) - { - queryable = (IRavenQueryable)Queryable.Where(queryable, x => x.ExceptionType == queryParameters.ExceptionType); - } - - if (queryParameters.ExceptionMessage.IsNotEmpty()) - { - queryable = (IRavenQueryable)Queryable.Where(queryable, x => x.ExceptionMessage == queryParameters.ExceptionMessage); - } - - if (queryParameters.From.HasValue) - { - queryable = (IRavenQueryable)Queryable.Where(queryable, x => x.SentAt >= queryParameters.From.Value); - } - - if (queryParameters.Until.HasValue) - { - queryable = (IRavenQueryable)Queryable.Where(queryable, x => x.SentAt <= queryParameters.Until.Value); - } - - var messages = await queryable - .OrderBy(x => x.SentAt) - .Take((int)queryParameters.Limit + 1) - .ToListAsync(); - - var envelopes = messages.Select(x => x.ToEnvelope()).ToList(); - - var next = Guid.Empty; - if (envelopes.Count > queryParameters.Limit) - { - next = envelopes.Last().Envelope.Id; - envelopes.RemoveAt(envelopes.Count - 1); - } - - return new DeadLetterEnvelopesFound(envelopes, next, tenantId); - } + // public async Task QueryDeadLetterEnvelopesAsync(DeadLetterEnvelopeQueryParameters queryParameters, string? tenantId = null) + // { + // using var session = _store.OpenAsyncSession(); + // var queryable = session.Query().Customize(x => x.WaitForNonStaleResults()); + // if (queryParameters.StartId.HasValue) + // { + // queryable = (IRavenQueryable)Queryable.Where(queryable, x => x.EnvelopeId >= queryParameters.StartId.Value); + // } + // + // if (queryParameters.MessageType.IsNotEmpty()) + // { + // queryable = (IRavenQueryable)Queryable.Where(queryable, x => x.MessageType == queryParameters.MessageType); + // } + // + // if (queryParameters.ExceptionType.IsNotEmpty()) + // { + // queryable = (IRavenQueryable)Queryable.Where(queryable, x => x.ExceptionType == queryParameters.ExceptionType); + // } + // + // if (queryParameters.ExceptionMessage.IsNotEmpty()) + // { + // queryable = (IRavenQueryable)Queryable.Where(queryable, x => x.ExceptionMessage == queryParameters.ExceptionMessage); + // } + // + // if (queryParameters.From.HasValue) + // { + // queryable = (IRavenQueryable)Queryable.Where(queryable, x => x.SentAt >= queryParameters.From.Value); + // } + // + // if (queryParameters.Until.HasValue) + // { + // queryable = (IRavenQueryable)Queryable.Where(queryable, x => x.SentAt <= queryParameters.Until.Value); + // } + // + // var messages = await queryable + // .OrderBy(x => x.SentAt) + // .Take((int)queryParameters.Limit + 1) + // .ToListAsync(); + // + // var envelopes = messages.Select(x => x.ToEnvelope()).ToList(); + // + // var next = Guid.Empty; + // if (envelopes.Count > queryParameters.Limit) + // { + // next = envelopes.Last().Envelope.Id; + // envelopes.RemoveAt(envelopes.Count - 1); + // } + // + // return new DeadLetterEnvelopesFound(envelopes, next, tenantId); + // } public async Task DeadLetterEnvelopeByIdAsync(Guid id, string? tenantId = null) { @@ -75,6 +75,7 @@ public async Task QueryDeadLetterEnvelopesAsync(DeadLe return message.ToEnvelope(); } + // TODO -- use this in the new admin public async Task MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType = "") { using var session = _store.OpenAsyncSession(); @@ -124,17 +125,6 @@ public async Task MarkDeadLetterEnvelopesAsReplayableAsync(Guid[] ids, string? t await session.SaveChangesAsync(); } - public async Task DeleteDeadLetterEnvelopesAsync(Guid[] ids, string? tenantId = null) - { - using var session = _store.OpenAsyncSession(); - foreach (var id in ids) - { - session.Delete(dlqId(id)); - } - - await session.SaveChangesAsync(); - } - public Task> SummarizeAllAsync(string serviceName, TimeRange range, CancellationToken token) { throw new NotImplementedException(); diff --git a/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs b/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs index 9964e7333..e0c787959 100644 --- a/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs +++ b/src/Persistence/Wolverine.SqlServer/Persistence/SqlServerMessageStore.cs @@ -160,41 +160,6 @@ public override async Task FetchCountsAsync() /// public string DatabasePrincipal { get; set; } = "dbo"; - public override Task MarkDeadLetterEnvelopesAsReplayableAsync(Guid[] ids, string? tenantId = null) - { - var table = new DataTable(); - table.Columns.Add(new DataColumn("ID", typeof(Guid))); - foreach (var id in ids) - { - table.Rows.Add(id); - } - - var command = CreateCommand($"update {SchemaName}.{DatabaseConstants.DeadLetterTable} set {DatabaseConstants.Replayable} = @replay where id in (select ID from @IDLIST)"); - command.With("replay", true); - var list = command.AddNamedParameter("IDLIST", table).As(); - list.SqlDbType = SqlDbType.Structured; - list.TypeName = $"{SchemaName}.EnvelopeIdList"; - - return command.ExecuteNonQueryAsync(_cancellation); - } - - public override Task DeleteDeadLetterEnvelopesAsync(Guid[] ids, string? tenantId = null) - { - var table = new DataTable(); - table.Columns.Add(new DataColumn("ID", typeof(Guid))); - foreach (var id in ids) - { - table.Rows.Add(id); - } - - var command = CreateCommand($"delete from {SchemaName}.{DatabaseConstants.DeadLetterTable} where id in (select ID from @IDLIST)"); - var list = command.AddNamedParameter("IDLIST", table).As(); - list.SqlDbType = SqlDbType.Structured; - list.TypeName = $"{SchemaName}.EnvelopeIdList"; - - return command.ExecuteNonQueryAsync(_cancellation); - } - protected override string determineOutgoingEnvelopeSql(DurabilitySettings settings) { return @@ -423,7 +388,7 @@ public override IEnumerable AllObjects() yield return table; } - if (_settings.Role == MessageStoreRole.Main) + if (Role == MessageStoreRole.Main) { var nodeTable = new Table(new DbObjectName(SchemaName, DatabaseConstants.NodeTableName)); nodeTable.AddColumn("id").AsPrimaryKey(); diff --git a/src/Testing/Wolverine.ComplianceTests/MessageStoreCompliance.cs b/src/Testing/Wolverine.ComplianceTests/MessageStoreCompliance.cs index 0b5c4c9ce..8180aefc3 100644 --- a/src/Testing/Wolverine.ComplianceTests/MessageStoreCompliance.cs +++ b/src/Testing/Wolverine.ComplianceTests/MessageStoreCompliance.cs @@ -267,7 +267,7 @@ public async Task delete_dead_letter_message_by_id() await thePersistence .DeadLetters - .DeleteDeadLetterEnvelopesAsync([replayableEnvelope.Id]); + .DiscardAsync(new DeadLetterEnvelopeQuery([replayableEnvelope.Id]), CancellationToken.None); var counts = await thePersistence.Admin.FetchCountsAsync(); @@ -529,14 +529,12 @@ public async Task load_dead_letter_envelopes_with_limit() await thePersistence.Inbox.MoveToDeadLetterStorageAsync(report3.Envelope, ex); await thePersistence.Inbox.MoveToDeadLetterStorageAsync(report4.Envelope, ex); + var stored = await thePersistence.DeadLetters.QueryAsync(new DeadLetterEnvelopeQuery() + { + PageSize = 2 + }, CancellationToken.None); - var stored = await thePersistence.DeadLetters.QueryDeadLetterEnvelopesAsync( - new DeadLetterEnvelopeQueryParameters - { - Limit = 2 - }); - - stored.DeadLetterEnvelopes.Count.ShouldBe(2); + stored.Envelopes.Count.ShouldBe(2); } [Fact] @@ -565,60 +563,14 @@ public async Task query_dead_letter_envelopes_with_from_and_until() await thePersistence.Inbox.MoveToDeadLetterStorageAsync(report3.Envelope, ex); await thePersistence.Inbox.MoveToDeadLetterStorageAsync(report4.Envelope, ex); + var query = new DeadLetterEnvelopeQuery(new TimeRange(DateTimeOffset.Now.AddDays(-1), + DateTimeOffset.Now.AddDays(1))); + + var result = await thePersistence.DeadLetters.QueryAsync(query, CancellationToken.None); - var parameters = new DeadLetterEnvelopeQueryParameters - { - From = DateTimeOffset.Now.AddDays(-1), - Until = DateTimeOffset.Now.AddDays(1) - }; - - var result = await thePersistence.DeadLetters.QueryDeadLetterEnvelopesAsync(parameters); - - result.DeadLetterEnvelopes.Count.ShouldBe(3); + result.Envelopes.Count.ShouldBe(3); } - - [Fact] - public async Task query_dead_letter_envelopes_with_start_id() - { - var list = new List(); - - for (var i = 0; i < 10; i++) - { - var envelope = ObjectMother.Envelope(); - envelope.Id = Guid.Parse($"00000000-0000-0000-0000-00000000000{i}"); - envelope.Status = EnvelopeStatus.Incoming; - - - list.Add(envelope); - } - await thePersistence.Inbox.StoreIncomingAsync(list.ToArray()); - - - var ex = new DivideByZeroException("Kaboom!"); - - var report2 = new ErrorReport(list[2], ex); - var report3 = new ErrorReport(list[3], ex); - var report4 = new ErrorReport(list[4], ex); - - await thePersistence.Inbox.MoveToDeadLetterStorageAsync(report2.Envelope, ex); - await thePersistence.Inbox.MoveToDeadLetterStorageAsync(report3.Envelope, ex); - await thePersistence.Inbox.MoveToDeadLetterStorageAsync(report4.Envelope, ex); - - - var parameters = new DeadLetterEnvelopeQueryParameters - { - StartId = report3.Id - }; - - var result = await thePersistence.DeadLetters.QueryDeadLetterEnvelopesAsync(parameters); - - result.DeadLetterEnvelopes.Count.ShouldBe(2); - result.DeadLetterEnvelopes.ShouldNotContain(x => x.Envelope.Id == report2.Id); - result.DeadLetterEnvelopes.ShouldContain(x => x.Envelope.Id == report3.Id); - result.DeadLetterEnvelopes.ShouldContain(x => x.Envelope.Id == report4.Id); - } - [Fact] public async Task load_dead_letter_envelopes_by_exception_type() { @@ -646,16 +598,16 @@ public async Task load_dead_letter_envelopes_by_exception_type() await thePersistence.Inbox.MoveToDeadLetterStorageAsync(report4.Envelope, ex); - var stored = await thePersistence.DeadLetters.QueryDeadLetterEnvelopesAsync( - new DeadLetterEnvelopeQueryParameters + var stored = await thePersistence.DeadLetters.QueryAsync( + new() { ExceptionType = report2.ExceptionType - }); + }, CancellationToken.None); - stored.DeadLetterEnvelopes.Count.ShouldBe(3); - stored.DeadLetterEnvelopes.ShouldContain(x => x.Envelope.Id == report2.Id); - stored.DeadLetterEnvelopes.ShouldContain(x => x.Envelope.Id == report3.Id); - stored.DeadLetterEnvelopes.ShouldContain(x => x.Envelope.Id == report4.Id); + stored.Envelopes.Count.ShouldBe(3); + stored.Envelopes.ShouldContain(x => x.Envelope.Id == report2.Id); + stored.Envelopes.ShouldContain(x => x.Envelope.Id == report3.Id); + stored.Envelopes.ShouldContain(x => x.Envelope.Id == report4.Id); } [Fact] @@ -740,16 +692,16 @@ public async Task load_dead_letter_envelopes_by_message_type() await thePersistence.Inbox.MoveToDeadLetterStorageAsync(report4.Envelope, ex); - var stored = await thePersistence.DeadLetters.QueryDeadLetterEnvelopesAsync( - new DeadLetterEnvelopeQueryParameters + var stored = await thePersistence.DeadLetters.QueryAsync( + new() { MessageType = report2.Envelope.MessageType - }); + }, CancellationToken.None); - stored.DeadLetterEnvelopes.Count.ShouldBe(3); - stored.DeadLetterEnvelopes.ShouldContain(x => x.Envelope.Id == report2.Id); - stored.DeadLetterEnvelopes.ShouldContain(x => x.Envelope.Id == report3.Id); - stored.DeadLetterEnvelopes.ShouldContain(x => x.Envelope.Id == report4.Id); + stored.Envelopes.Count.ShouldBe(3); + stored.Envelopes.ShouldContain(x => x.Envelope.Id == report2.Id); + stored.Envelopes.ShouldContain(x => x.Envelope.Id == report3.Id); + stored.Envelopes.ShouldContain(x => x.Envelope.Id == report4.Id); } [Fact] diff --git a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Bugs/Bug_1594_ReplayDeadLetterQueue.cs b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Bugs/Bug_1594_ReplayDeadLetterQueue.cs index 1427c8372..c9b448363 100644 --- a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Bugs/Bug_1594_ReplayDeadLetterQueue.cs +++ b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Bugs/Bug_1594_ReplayDeadLetterQueue.cs @@ -61,15 +61,15 @@ public async Task can_replay_dead_letter_message(EndpointMode mode) await Task.Delay(1000); var messageStore = host.Services.GetRequiredService(); - var deadLetterQuery = new DeadLetterEnvelopeQueryParameters { Limit = 10 }; + var deadLetterQuery = new DeadLetterEnvelopeQuery { Limit = 10 }; var sw = Stopwatch.StartNew(); Guid? deadLetterId = null; while (sw.Elapsed < TimeSpan.FromSeconds(10)) { - var deadLetterResults = await messageStore.DeadLetters.QueryDeadLetterEnvelopesAsync(deadLetterQuery); - if (deadLetterResults.DeadLetterEnvelopes.Any()) + var deadLetterResults = await messageStore.DeadLetters.QueryAsync(deadLetterQuery, CancellationToken.None); + if (deadLetterResults.Envelopes.Any()) { - deadLetterId = deadLetterResults.DeadLetterEnvelopes.First().Id; + deadLetterId = deadLetterResults.Envelopes.First().Id; break; } await Task.Delay(100); @@ -78,9 +78,9 @@ public async Task can_replay_dead_letter_message(EndpointMode mode) deadLetterId.ShouldNotBeNull("Message should be in DLQ after failure"); // Log state before replay - var beforeReplay = await messageStore.DeadLetters.QueryDeadLetterEnvelopesAsync(deadLetterQuery); + var beforeReplay = await messageStore.DeadLetters.QueryAsync(deadLetterQuery, CancellationToken.None); var beforeIncoming = await messageStore.Admin.AllIncomingAsync(); - _output.WriteLine($"[BEFORE REPLAY] DLQ: {beforeReplay.DeadLetterEnvelopes.Count}, Incoming: {beforeIncoming.Count}"); + _output.WriteLine($"[BEFORE REPLAY] DLQ: {beforeReplay.Envelopes.Count}, Incoming: {beforeIncoming.Count}"); // Force handler to succeed on replay (mimic Marten test) ReplayTestHandler.FailFirst = false; @@ -93,9 +93,8 @@ public async Task can_replay_dead_letter_message(EndpointMode mode) .ExecuteAndWaitAsync((IMessageContext _) => messageStore.DeadLetters.MarkDeadLetterEnvelopesAsReplayableAsync(new[] { deadLetterId.Value })); // Log state after replay - var afterReplay = await messageStore.DeadLetters.QueryDeadLetterEnvelopesAsync(deadLetterQuery); + var afterReplay = await messageStore.DeadLetters.QueryAsync(deadLetterQuery, CancellationToken.None); var afterIncoming = await messageStore.Admin.AllIncomingAsync(); - _output.WriteLine($"[AFTER REPLAY] DLQ: {afterReplay.DeadLetterEnvelopes.Count}, Incoming: {afterIncoming.Count}"); foreach (var env in afterIncoming) { _output.WriteLine($"[INCOMING] Id: {env.Id}, Status: {env.Status}, OwnerId: {env.OwnerId}, ScheduledTime: {env.ScheduledTime}, Attempts: {env.Attempts}, ReceivedAt: {env.Destination}"); @@ -104,7 +103,7 @@ public async Task can_replay_dead_letter_message(EndpointMode mode) // Assert using the tracking result, mimicking the Marten test tracked.MessageSucceeded.SingleMessage() .ShouldNotBeNull("ReplayTestMessage should be successfully processed after replay"); - afterReplay.DeadLetterEnvelopes.Any(dl => dl.Id == deadLetterId).ShouldBeFalse("Message should be removed from DLQ after successful replay (this should work for both durable and non-durable queues)"); + afterReplay.Envelopes.Any(dl => dl.Id == deadLetterId).ShouldBeFalse("Message should be removed from DLQ after successful replay (this should work for both durable and non-durable queues)"); afterIncoming.Any(env => env.Status == EnvelopeStatus.Incoming && env.Id == deadLetterId).ShouldBeFalse("Message should not remain in Incoming after successful processing"); } } diff --git a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Bugs/Bug_DLQ_NotSavedToDatabase.cs b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Bugs/Bug_DLQ_NotSavedToDatabase.cs index 4fb337dca..106c2b21d 100644 --- a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Bugs/Bug_DLQ_NotSavedToDatabase.cs +++ b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/Bugs/Bug_DLQ_NotSavedToDatabase.cs @@ -339,21 +339,21 @@ private async Task SendMessageAndCheckDLQ(IHost host, string connectionString, s _output.WriteLine($"Final counts for {testDescription}: Incoming={finalCounts.Incoming}, Outgoing={finalCounts.Outgoing}, DeadLetter={finalCounts.DeadLetter}"); // Query dead letters using Wolverine's API - var deadLetterQuery = new DeadLetterEnvelopeQueryParameters + var deadLetterQuery = new DeadLetterEnvelopeQuery { Limit = 100 }; - var deadLetterResults = await messageStore.DeadLetters.QueryDeadLetterEnvelopesAsync(deadLetterQuery); + var deadLetterResults = await messageStore.DeadLetters.QueryAsync(deadLetterQuery, CancellationToken.None); - _output.WriteLine($"Dead letter results for {testDescription}: {deadLetterResults.DeadLetterEnvelopes.Count} entries"); + _output.WriteLine($"Dead letter results for {testDescription}: {deadLetterResults.Envelopes.Count} entries"); // Document the current behavior - the DLQ is empty - _output.WriteLine($"Current behavior: SQL Server DLQ contains {deadLetterResults.DeadLetterEnvelopes.Count} messages"); + _output.WriteLine($"Current behavior: SQL Server DLQ contains {deadLetterResults.Envelopes.Count} messages"); _output.WriteLine("Note: The message appears to have been sent to the RabbitMQ DLQ instead of the SQL Server DLQ"); // Assert that we should have DLQ entries in SQL Server when using WolverineStorage mode // This test will fail, demonstrating the current bug - deadLetterResults.DeadLetterEnvelopes.ShouldNotBeEmpty($"When using DeadLetterQueueMode.WolverineStorage, failed messages should be saved to the SQL Server DLQ, but found {deadLetterResults.DeadLetterEnvelopes.Count} entries"); + deadLetterResults.Envelopes.ShouldNotBeEmpty($"When using DeadLetterQueueMode.WolverineStorage, failed messages should be saved to the SQL Server DLQ, but found {deadLetterResults.Envelopes.Count} entries"); } } diff --git a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/wolverine_storage_dead_letter_queue_mechanics.cs b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/wolverine_storage_dead_letter_queue_mechanics.cs index 5f845b73d..af59bd33c 100644 --- a/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/wolverine_storage_dead_letter_queue_mechanics.cs +++ b/src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/wolverine_storage_dead_letter_queue_mechanics.cs @@ -114,17 +114,17 @@ public async Task should_save_failed_messages_to_sql_server_dlq() // Check that the message is in the SQL Server DLQ var messageStore = _host.Services.GetRequiredService(); - var deadLetterQuery = new DeadLetterEnvelopeQueryParameters + var deadLetterQuery = new DeadLetterEnvelopeQuery { - Limit = 100 + PageSize = 100 }; - var deadLetterResults = await messageStore.DeadLetters.QueryDeadLetterEnvelopesAsync(deadLetterQuery); + var deadLetterResults = await messageStore.DeadLetters.QueryAsync(deadLetterQuery, CancellationToken.None); // Should have at least one dead letter message - deadLetterResults.DeadLetterEnvelopes.ShouldNotBeEmpty("Failed messages should be saved to SQL Server DLQ when using WolverineStorage mode"); + deadLetterResults.Envelopes.ShouldNotBeEmpty("Failed messages should be saved to SQL Server DLQ when using WolverineStorage mode"); // Verify the message details - var deadLetter = deadLetterResults.DeadLetterEnvelopes.First(); + var deadLetter = deadLetterResults.Envelopes.First(); deadLetter.ExceptionType.ShouldBe(typeof(DivideByZeroException).FullName); deadLetter.ExceptionMessage.ShouldBe("Boom."); } @@ -160,14 +160,14 @@ public async Task should_work_with_durable_inbox() // Check that the message is in the SQL Server DLQ var messageStore = _host.Services.GetRequiredService(); - var deadLetterQuery = new DeadLetterEnvelopeQueryParameters + var deadLetterQuery = new DeadLetterEnvelopeQuery { - Limit = 100 + PageSize = 100 }; - var deadLetterResults = await messageStore.DeadLetters.QueryDeadLetterEnvelopesAsync(deadLetterQuery); + var deadLetterResults = await messageStore.DeadLetters.QueryAsync(deadLetterQuery, CancellationToken.None); // Should have at least one dead letter message - deadLetterResults.DeadLetterEnvelopes.ShouldNotBeEmpty("Failed messages should be saved to SQL Server DLQ even with durable inbox"); + deadLetterResults.Envelopes.ShouldNotBeEmpty("Failed messages should be saved to SQL Server DLQ even with durable inbox"); } } diff --git a/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQuery.cs b/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQuery.cs index 208ecd8cb..bd1d45f59 100644 --- a/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQuery.cs +++ b/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQuery.cs @@ -18,13 +18,20 @@ public DeadLetterEnvelopeQuery() public int PageNumber { get; set; } public int PageSize { get; set; } = 100; + + [Obsolete("Prefer PageSize")] + public int Limit + { + get => PageSize; + set => PageSize = value; + } + public string? MessageType { get; set; } public string? ExceptionType { get; set; } public string? ReceivedAt { get; set; } - [Obsolete("Remove this")] - public Uri? Database { get; set; } - + public string? ExceptionMessage { get; set; } + public TimeRange Range { get; set; } = TimeRange.AllTime(); /// diff --git a/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQueryParameters.cs b/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQueryParameters.cs deleted file mode 100644 index 289d7252b..000000000 --- a/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeQueryParameters.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace Wolverine.Persistence.Durability.DeadLetterManagement; - -public class DeadLetterEnvelopeQueryParameters -{ - public uint Limit { get; set; } = 100; - public Guid? StartId { get; set; } - public string? MessageType { get; set; } - public string? ExceptionType { get; set; } - public string? ExceptionMessage { get; set; } - public DateTimeOffset? From { get; set; } - public DateTimeOffset? Until { get; set; } -} \ No newline at end of file diff --git a/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeResults.cs b/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeResults.cs index eb8a52178..036372bda 100644 --- a/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeResults.cs +++ b/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeResults.cs @@ -4,5 +4,6 @@ public class DeadLetterEnvelopeResults { public int TotalCount { get; set; } public List Envelopes { get; set; } = new(); + public int PageNumber { get; set; } } \ No newline at end of file diff --git a/src/Wolverine/Persistence/Durability/IDeadLetters.cs b/src/Wolverine/Persistence/Durability/IDeadLetters.cs index cd8af3465..6f095ce20 100644 --- a/src/Wolverine/Persistence/Durability/IDeadLetters.cs +++ b/src/Wolverine/Persistence/Durability/IDeadLetters.cs @@ -2,42 +2,41 @@ namespace Wolverine.Persistence.Durability; -/// -/// This is the original V2/3 service for dead letter querying -/// -public interface IDeadLetters +public static class DeadLettersExtensions { - Task QueryDeadLetterEnvelopesAsync(DeadLetterEnvelopeQueryParameters queryParameters, string? tenantId = null); - - /// Leaving tenantId null will query all tenants - Task DeadLetterEnvelopeByIdAsync(Guid id, string? tenantId = null); - /// /// Marks the Envelopes in DeadLetterTable /// as replayable. DurabilityAgent will move the envelopes to IncomingTable. /// /// Exception Type that should be marked. Default is any. /// Number of envelopes marked. - [Obsolete("Prefer ReplayAsync")] - Task MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType = ""); - + public static Task MarkDeadLetterEnvelopesAsReplayableAsync(this IDeadLetters letters, string exceptionType = "") + { + return letters.ReplayAsync(new DeadLetterEnvelopeQuery + { Range = TimeRange.AllTime(), ExceptionType = exceptionType }, CancellationToken.None); + } + /// - /// Marks the Envelope in DeadLetterTable - /// as replayable. DurabilityAgent will move the envelopes to IncomingTable. + /// Polyfill for old API /// + /// /// - /// Leaving tenantId null will query all tenants - [Obsolete("Prefer ReplayAsync")] - Task MarkDeadLetterEnvelopesAsReplayableAsync(Guid[] ids, string? tenantId = null); + /// + public static Task MarkDeadLetterEnvelopesAsReplayableAsync(this IDeadLetters letters, Guid[] ids) + { + return letters.ReplayAsync(new DeadLetterEnvelopeQuery + { MessageIds = ids}, CancellationToken.None); + } +} - /// - /// Deletes the DeadLetterEnvelope from the DeadLetterTable - /// - /// +/// +/// This is the original V2/3 service for dead letter querying +/// +public interface IDeadLetters +{ /// Leaving tenantId null will query all tenants - [Obsolete("Prefer DiscardAsync")] - Task DeleteDeadLetterEnvelopesAsync(Guid[] ids, string? tenantId = null); - + Task DeadLetterEnvelopeByIdAsync(Guid id, string? tenantId = null); + /// /// Fetch a summary of the persisted dead letter queue envelopes /// diff --git a/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs b/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs index 7b7736c19..950415a48 100644 --- a/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs @@ -56,71 +56,24 @@ public MultiTenantedMessageStore(IMessageStore main, IWolverineRuntime runtime, public IMessageStore Main { get; } - async Task IDeadLetters.MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType) - { - foreach (var database in databases()) - { - try - { - await database.DeadLetters.MarkDeadLetterEnvelopesAsReplayableAsync(exceptionType); - } - catch (Exception e) - { - _logger.LogError(e, "Error trying to mark dead letter envelopes as replayable for database {Name}", - database.Name); - } - } - } - - public async Task MarkDeadLetterEnvelopesAsReplayableAsync(Guid[] ids, string? tenantId = null) - { - if (tenantId is not null) - { - var database = await GetDatabaseAsync(tenantId); - await database.DeadLetters.MarkDeadLetterEnvelopesAsReplayableAsync(ids); - return; - } - - foreach (var database in databases()) await database.DeadLetters.MarkDeadLetterEnvelopesAsReplayableAsync(ids); - } - - public async Task DeleteDeadLetterEnvelopesAsync(Guid[] ids, string? tenantId = null) - { - if (tenantId is not null) - { - var database = await GetDatabaseAsync(tenantId); - await database.DeadLetters.DeleteDeadLetterEnvelopesAsync(ids); - return; - } - - foreach (var database in databases()) await database.DeadLetters.DeleteDeadLetterEnvelopesAsync(ids); - } - public Task> SummarizeAllAsync(string serviceName, TimeRange range, CancellationToken token) { - throw new NotImplementedException(); + throw new NotSupportedException(); } public Task QueryAsync(DeadLetterEnvelopeQuery query, CancellationToken token) { - throw new NotImplementedException(); + throw new NotSupportedException(); } public Task DiscardAsync(DeadLetterEnvelopeQuery query, CancellationToken token) { - throw new NotImplementedException(); + throw new NotSupportedException(); } public Task ReplayAsync(DeadLetterEnvelopeQuery query, CancellationToken token) { - throw new NotImplementedException(); - } - - public async Task QueryDeadLetterEnvelopesAsync( - DeadLetterEnvelopeQueryParameters queryParameters, string? tenantId) - { - var database = await GetDatabaseAsync(tenantId); - return await database.DeadLetters.QueryDeadLetterEnvelopesAsync(queryParameters, tenantId); + throw new NotSupportedException(); } public async Task DeadLetterEnvelopeByIdAsync(Guid id, string? tenantId = null) diff --git a/src/Wolverine/Persistence/Durability/NullMessageStore.cs b/src/Wolverine/Persistence/Durability/NullMessageStore.cs index f3d5f90ef..fc459a462 100644 --- a/src/Wolverine/Persistence/Durability/NullMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/NullMessageStore.cs @@ -195,13 +195,6 @@ public Task ClearAllAsync() return Task.CompletedTask; } - public Task MarkDeadLetterEnvelopesAsReplayableAsync(string exceptionType) - { - return Task.FromResult(0); - } - - public Task MarkDeadLetterEnvelopesAsReplayableAsync(Guid[] ids, string? tenantId = null) => Task.CompletedTask; - public Task DeleteDeadLetterEnvelopesAsync(Guid[] ids, string? tenantId = null) => Task.CompletedTask; public Task> SummarizeAllAsync(string serviceName, TimeRange range, CancellationToken token) { return Task.FromResult>(new List()); @@ -252,11 +245,6 @@ public Task ReassignIncomingAsync(int ownerId, IReadOnlyList incoming) throw new NotSupportedException(); } - public Task QueryDeadLetterEnvelopesAsync(DeadLetterEnvelopeQueryParameters queryParameters, string? tenantId) - { - throw new NotImplementedException(); - } - public Task DeadLetterEnvelopeByIdAsync(Guid id, string? tenantId = null) { throw new NotImplementedException(); diff --git a/src/Wolverine/Persistence/MessageStoreCollection.cs b/src/Wolverine/Persistence/MessageStoreCollection.cs index 5e9507bb1..f318e41b8 100644 --- a/src/Wolverine/Persistence/MessageStoreCollection.cs +++ b/src/Wolverine/Persistence/MessageStoreCollection.cs @@ -329,6 +329,56 @@ public ValueTask DisposeAsync() var stores = _services.Enumerate().Select(x => x.Value).ToArray(); return stores.MaybeDisposeAllAsync(); } + + public async Task ReplayDeadLettersAsync(Guid[] ids) + { + foreach (var database in await FindAllAsync()) + { + await database.DeadLetters.ReplayAsync(new(ids), CancellationToken.None); + } + } + + public async Task ReplayDeadLettersAsync(string tenantId, Guid[] ids) + { + foreach (var tenantedMessageStore in _multiTenanted) + { + if (tenantedMessageStore.Source.Cardinality == DatabaseCardinality.DynamicMultiple) + { + await tenantedMessageStore.Source.RefreshAsync(); + } + + var tenanted = await tenantedMessageStore.Source.FindAsync(tenantId); + if (tenanted != null) + { + await tenanted.DeadLetters.ReplayAsync(new(ids), CancellationToken.None); + } + } + } + + public async Task DiscardDeadLettersAsync(Guid[] ids) + { + foreach (var database in await FindAllAsync()) + { + await database.DeadLetters.DiscardAsync(new(ids), CancellationToken.None); + } + } + + public async Task DiscardDeadLettersAsync(string tenantId, Guid[] ids) + { + foreach (var tenantedMessageStore in _multiTenanted) + { + if (tenantedMessageStore.Source.Cardinality == DatabaseCardinality.DynamicMultiple) + { + await tenantedMessageStore.Source.RefreshAsync(); + } + + var tenanted = await tenantedMessageStore.Source.FindAsync(tenantId); + if (tenanted != null) + { + await tenanted.DeadLetters.DiscardAsync(new(ids), CancellationToken.None); + } + } + } } public class InvalidWolverineStorageConfigurationException : Exception From 045a96400d254fccc5817ac6fcb92e9bff3f3f71 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Sun, 21 Sep 2025 13:13:16 -0500 Subject: [PATCH 21/25] All existing tests passing on the DLQ admin services --- .../dead_letter_endpoints.cs | 1 + .../DeadLettersEndpointExtensions.cs | 95 +++++-------------- .../DeadLetterAdminCompliance.cs | 14 ++- .../Durability/DeadLetterEnvelope.cs | 71 ++++++++++++++ .../DeadLetterEnvelopeResults.cs | 21 ++++ .../Persistence/Durability/IMessageStore.cs | 66 +++++-------- .../Persistence/MessageStoreCollection.cs | 75 +++++++++++++++ 7 files changed, 227 insertions(+), 116 deletions(-) create mode 100644 src/Wolverine/Persistence/Durability/DeadLetterEnvelope.cs diff --git a/src/Http/Wolverine.Http.Tests/dead_letter_endpoints.cs b/src/Http/Wolverine.Http.Tests/dead_letter_endpoints.cs index f8bf9e72f..ad5ead2dd 100644 --- a/src/Http/Wolverine.Http.Tests/dead_letter_endpoints.cs +++ b/src/Http/Wolverine.Http.Tests/dead_letter_endpoints.cs @@ -1,5 +1,6 @@ using JasperFx.Core.Reflection; using Shouldly; +using Wolverine.Persistence.Durability.DeadLetterManagement; using Wolverine.Tracking; using WolverineWebApi; diff --git a/src/Http/Wolverine.Http/DeadLettersEndpointExtensions.cs b/src/Http/Wolverine.Http/DeadLettersEndpointExtensions.cs index 430b53d7f..f0b9b486a 100644 --- a/src/Http/Wolverine.Http/DeadLettersEndpointExtensions.cs +++ b/src/Http/Wolverine.Http/DeadLettersEndpointExtensions.cs @@ -2,38 +2,19 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Routing; -using Microsoft.Extensions.Options; -using Wolverine.Persistence.Durability; +using Wolverine.Persistence; using Wolverine.Persistence.Durability.DeadLetterManagement; using Wolverine.Runtime; -using Wolverine.Runtime.Handlers; namespace Wolverine.Http; -public class DeadLetterEnvelopeGetRequest -{ - /// - /// Number of records to return per page. - /// - public uint Limit { get; set; } = 100; - - public int PageNumber { get; set; } - - public string? MessageType { get; set; } - public string? ExceptionType { get; set; } - public string? ExceptionMessage { get; set; } - public DateTimeOffset? From { get; set; } - public DateTimeOffset? Until { get; set; } - public string? TenantId { get; set; } -} - public class DeadLetterEnvelopeIdsRequest { public Guid[] Ids { get; set; } public string? TenantId { get; set; } } -public record DeadLetterEnvelopesFoundResponse(IReadOnlyList Messages, Guid? NextId); +public record DeadLetterEnvelopesFoundResponse(IReadOnlyList Messages, int Total); public record DeadLetterEnvelopeResponse( Guid Id, @@ -50,55 +31,29 @@ public record DeadLetterEnvelopeResponse( public static class DeadLettersEndpointExtensions { /// - /// Add endpoints to manage the Wolverine database-backed deal letter queue for this - /// application. + /// Add endpoints to manage the Wolverine database-backed deal letter queue for this + /// application. /// - /// Optionally override the group Url prefix for these endpoints. The default is "/dead-letters" - public static RouteGroupBuilder MapDeadLettersEndpoints(this IEndpointRouteBuilder endpoints, string? groupUrlPrefix = "/dead-letters") + /// + /// Optionally override the group Url prefix for these endpoints. The default is + /// "/dead-letters" + /// + public static RouteGroupBuilder MapDeadLettersEndpoints(this IEndpointRouteBuilder endpoints, + string? groupUrlPrefix = "/dead-letters") { if (groupUrlPrefix.IsEmpty()) + { throw new ArgumentNullException(nameof(groupUrlPrefix), "Cannot be empty or null"); + } + var deadlettersGroup = endpoints.MapGroup(groupUrlPrefix); - deadlettersGroup.MapPost("/", async (DeadLetterEnvelopeGetRequest request, IMessageStore messageStore, HandlerGraph handlerGraph, IOptions opts) => - { - var deadLetters = messageStore.DeadLetters; - var queryParameters = new DeadLetterEnvelopeQuery + deadlettersGroup.MapPost("/", + async (DeadLetterEnvelopeGetRequest request, MessageStoreCollection stores, + CancellationToken cancellation) => { - PageSize = (int)request.Limit, - PageNumber = request.PageNumber, - MessageType = request.MessageType, - ExceptionType = request.ExceptionType, - ExceptionMessage = request.ExceptionMessage, - Range = new TimeRange(request.From, request.Until) - }; - - throw new NotImplementedException(); - // if (request.TenantId.IsNotEmpty()) - // { - // - // } - // else - // { - // var deadLetterEnvelopesFound = await deadLetters.QueryAsync(queryParameters, CancellationToken.None); - // } - // - // var deadLetterEnvelopesFound = await deadLetters.QueryAsync(queryParameters, request.TenantId); - // return new DeadLetterEnvelopesFoundResponse( - // [.. deadLetterEnvelopesFound.Select(x => new DeadLetterEnvelopeResponse( - // x.Id, - // x.ExecutionTime, - // handlerGraph.TryFindMessageType(x.MessageType, out var messageType) ? opts.Value.DetermineSerializer(x.Envelope).ReadFromData(messageType, x.Envelope) : null, - // x.MessageType, - // x.ReceivedAt, - // x.Source, - // x.ExceptionType, - // x.ExceptionMessage, - // x.SentAt, - // x.Replayable)) - // ], - // deadLetterEnvelopesFound.PageNumber); - }); + return await stores.FetchDeadLetterEnvelopesAsync(request, cancellation); + }); deadlettersGroup.MapPost("/replay", (DeadLetterEnvelopeIdsRequest request, IWolverineRuntime runtime) => { @@ -106,12 +61,10 @@ public static RouteGroupBuilder MapDeadLettersEndpoints(this IEndpointRouteBuild { return runtime.Stores.ReplayDeadLettersAsync(request.Ids); } - else - { - return runtime.Stores.ReplayDeadLettersAsync(request.TenantId, request.Ids); - } + + return runtime.Stores.ReplayDeadLettersAsync(request.TenantId, request.Ids); }); - + deadlettersGroup.MapDelete("/", ([FromBody] DeadLetterEnvelopeIdsRequest request, IWolverineRuntime runtime) => { @@ -119,10 +72,8 @@ public static RouteGroupBuilder MapDeadLettersEndpoints(this IEndpointRouteBuild { return runtime.Stores.DiscardDeadLettersAsync(request.Ids); } - else - { - return runtime.Stores.DiscardDeadLettersAsync(request.TenantId, request.Ids); - } + + return runtime.Stores.DiscardDeadLettersAsync(request.TenantId, request.Ids); }); return deadlettersGroup; diff --git a/src/Testing/Wolverine.ComplianceTests/DeadLetterAdminCompliance.cs b/src/Testing/Wolverine.ComplianceTests/DeadLetterAdminCompliance.cs index c76b53809..66cb81101 100644 --- a/src/Testing/Wolverine.ComplianceTests/DeadLetterAdminCompliance.cs +++ b/src/Testing/Wolverine.ComplianceTests/DeadLetterAdminCompliance.cs @@ -518,8 +518,12 @@ await theDeadLetters.ReplayAsync( query, CancellationToken.None); var results = await theDeadLetters.QueryAsync(query, CancellationToken.None); - results.TotalCount.ShouldBe(0); - results.Envelopes.Count.ShouldBe(0); + + foreach (var envelopeResult in results.Envelopes) + { + envelopeResult.Replayable.ShouldBeTrue(); + } + } [Fact] @@ -615,7 +619,11 @@ public async Task replay_by_message_batch() // Reload await loadAllEnvelopes(); - allEnvelopes.Envelopes.Where(x => ids.Contains(x.Id)).Any().ShouldBeFalse(); + + foreach (var dle in allEnvelopes.Envelopes.Where(x => ids.Contains(x.Id))) + { + dle.Replayable.ShouldBeTrue(); + } } } diff --git a/src/Wolverine/Persistence/Durability/DeadLetterEnvelope.cs b/src/Wolverine/Persistence/Durability/DeadLetterEnvelope.cs new file mode 100644 index 000000000..ece0b2fbd --- /dev/null +++ b/src/Wolverine/Persistence/Durability/DeadLetterEnvelope.cs @@ -0,0 +1,71 @@ +using Microsoft.Extensions.Logging; +using Wolverine.Runtime; + +namespace Wolverine.Persistence.Durability; + +public class DeadLetterEnvelope +{ + public DeadLetterEnvelope( + Guid id, + DateTimeOffset? executionTime, + Envelope envelope, + string messageType, + string receivedAt, + string source, + string exceptionType, + string exceptionMessage, + DateTimeOffset sentAt, + bool replayable + ) + { + Id = id; + ExecutionTime = executionTime; + Envelope = envelope; + MessageType = messageType; + ReceivedAt = receivedAt; + Source = source; + ExceptionType = exceptionType; + ExceptionMessage = exceptionMessage; + SentAt = sentAt; + Replayable = replayable; + } + + public Guid Id { get; } + public DateTimeOffset? ExecutionTime { get; } + public Envelope Envelope { get; } + public string MessageType { get; } + public string ReceivedAt { get; } + public string Source { get; } + public string ExceptionType { get; } + public string ExceptionMessage { get; } + public DateTimeOffset SentAt { get; } + public bool Replayable { get; } + + /// + /// The actual message body + /// + public object? Data { get; set; } + + internal void TryReadData(IWolverineRuntime runtime) + { + if (runtime.Options.HandlerGraph.TryFindMessageType(MessageType, out var messageType)) + { + var endpoint = runtime.Endpoints.EndpointFor(Envelope.Destination); + var serializer = endpoint?.TryFindSerializer(Envelope.ContentType) ?? + runtime.Options.TryFindSerializer(Envelope.ContentType); + + if (serializer != null) + { + try + { + Data = serializer.ReadFromData(messageType, Envelope); + } + catch (Exception e) + { + runtime.Logger.LogError(e, + "Error trying to deserialize the data for an envelope {Id} being fetched from the dead letter queue storage", Id); + } + } + } + } +} \ No newline at end of file diff --git a/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeResults.cs b/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeResults.cs index 036372bda..585d78fbc 100644 --- a/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeResults.cs +++ b/src/Wolverine/Persistence/Durability/DeadLetterManagement/DeadLetterEnvelopeResults.cs @@ -6,4 +6,25 @@ public class DeadLetterEnvelopeResults public List Envelopes { get; set; } = new(); public int PageNumber { get; set; } + + public Uri? DatabaseUri { get; set; } +} + +public class DeadLetterEnvelopeGetRequest +{ + /// + /// Number of records to return per page. + /// + public uint Limit { get; set; } = 100; + + public int PageNumber { get; set; } + + public string? MessageType { get; set; } + public string? ExceptionType { get; set; } + public string? ExceptionMessage { get; set; } + public DateTimeOffset? From { get; set; } + public DateTimeOffset? Until { get; set; } + public string? TenantId { get; set; } + + public Uri? DatabaseUri { get; set; } } \ No newline at end of file diff --git a/src/Wolverine/Persistence/Durability/IMessageStore.cs b/src/Wolverine/Persistence/Durability/IMessageStore.cs index aceafc4d8..4b52672ca 100644 --- a/src/Wolverine/Persistence/Durability/IMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/IMessageStore.cs @@ -8,24 +8,24 @@ namespace Wolverine.Persistence.Durability; public enum MessageStoreRole { /// - /// Denotes that this message store is the main message store for the application and where - /// node information is stored + /// Denotes that this message store is the main message store for the application and where + /// node information is stored /// Main, - + /// - /// Denotes that this message store is an additional message store for the application, but - /// does not store the node information + /// Denotes that this message store is an additional message store for the application, but + /// does not store the node information /// Ancillary, - + /// - /// This message store is strictly for one or more tenants + /// This message store is strictly for one or more tenants /// Tenant, - + /// - /// This message store is a multi-tenanted composite of other message stores + /// This message store is a multi-tenanted composite of other message stores /// Composite } @@ -38,10 +38,10 @@ public interface IMessageInbox Task IncrementIncomingEnvelopeAttemptsAsync(Envelope envelope); Task StoreIncomingAsync(Envelope envelope); Task StoreIncomingAsync(IReadOnlyList envelopes); - + Task MarkIncomingEnvelopeAsHandledAsync(Envelope envelope); - + // Only called by DurableReceiver Task MarkIncomingEnvelopeAsHandledAsync(IReadOnlyList envelopes); @@ -60,20 +60,6 @@ public interface IMessageOutbox Task DiscardAndReassignOutgoingAsync(Envelope[] discards, Envelope[] reassigned, int nodeId); } -public record DeadLetterEnvelopesFound(IReadOnlyList DeadLetterEnvelopes, Guid? NextId, string? TenantId); -public record DeadLetterEnvelope( - Guid Id, - DateTimeOffset? ExecutionTime, - Envelope Envelope, - string MessageType, - string ReceivedAt, - string Source, - string ExceptionType, - string ExceptionMessage, - DateTimeOffset SentAt, - bool Replayable - ); - public interface IMessageStoreWithAgentSupport : IMessageStore { IAgent BuildAgent(IWolverineRuntime runtime); @@ -82,16 +68,16 @@ public interface IMessageStoreWithAgentSupport : IMessageStore public interface IMessageStore : IAsyncDisposable { /// - /// What is the role of this message store within the application? + /// What is the role of this message store within the application? /// MessageStoreRole Role { get; } - + /// - /// Unique identifier for a message store in case of systems that use multiple message - /// store databases. Must use the "messagedb" scheme, and reflect the database connection + /// Unique identifier for a message store in case of systems that use multiple message + /// store databases. Must use the "messagedb" scheme, and reflect the database connection /// Uri Uri { get; } - + // /// // /// Let's consuming services in Wolverine know that this message store // /// has been disposed and cannot be used in a "DrainAsync". This mostly @@ -109,6 +95,12 @@ public interface IMessageStore : IAsyncDisposable IDeadLetters DeadLetters { get; } + + /// + /// Descriptive name for cases of multiple message stores + /// + string Name { get; } + /// /// Called to initialize the Wolverine storage on application bootstrapping /// @@ -123,20 +115,14 @@ public interface IMessageStore : IAsyncDisposable Task> LoadPageOfGloballyOwnedIncomingAsync(Uri listenerAddress, int limit); Task ReassignIncomingAsync(int ownerId, IReadOnlyList incoming); - - - /// - /// Descriptive name for cases of multiple message stores - /// - string Name { get; } - + void PromoteToMain(IWolverineRuntime runtime); } public record IncomingCount(Uri Destination, int Count); /// -/// Marks a secondary message store for a Wolverine application +/// Marks a secondary message store for a Wolverine application /// public interface IAncillaryMessageStore : IMessageStore { @@ -149,6 +135,4 @@ public interface IAncillaryMessageStore : IAncillaryMessageStore public interface ITenantedMessageSource : ITenantedSource { - -} - +} \ No newline at end of file diff --git a/src/Wolverine/Persistence/MessageStoreCollection.cs b/src/Wolverine/Persistence/MessageStoreCollection.cs index f318e41b8..858a694b2 100644 --- a/src/Wolverine/Persistence/MessageStoreCollection.cs +++ b/src/Wolverine/Persistence/MessageStoreCollection.cs @@ -3,6 +3,7 @@ using JasperFx.Core.Reflection; using JasperFx.Descriptors; using Wolverine.Persistence.Durability; +using Wolverine.Persistence.Durability.DeadLetterManagement; using Wolverine.Runtime; using Wolverine.Runtime.Agents; @@ -379,6 +380,80 @@ public async Task DiscardDeadLettersAsync(string tenantId, Guid[] ids) } } } + + private async Task> findStoresAsync(DeadLetterEnvelopeGetRequest request) + { + var list = new List(); + if (request.DatabaseUri != null) + { + var store = await FindDatabaseAsync(request.DatabaseUri); + if (store != null) + { + list.Add(store); + } + } + else if (request.TenantId != null) + { + foreach (var tenantedMessageStore in _multiTenanted) + { + var store = await tenantedMessageStore.Source.FindAsync(request.TenantId); + if (store != null) + { + list.Add(store); + continue; + } + + if (tenantedMessageStore.Source.Cardinality == DatabaseCardinality.DynamicMultiple) + { + await tenantedMessageStore.Source.RefreshAsync(); + } + + store = await tenantedMessageStore.Source.FindAsync(request.TenantId); + if (store != null) + { + list.Add(store); + } + } + } + else + { + list.AddRange(await FindAllAsync()); + } + + return list; + } + + public async Task> FetchDeadLetterEnvelopesAsync( + DeadLetterEnvelopeGetRequest request, CancellationToken cancellation) + { + var query = new DeadLetterEnvelopeQuery + { + PageSize = (int)request.Limit, + PageNumber = request.PageNumber, + MessageType = request.MessageType, + ExceptionType = request.ExceptionType, + ExceptionMessage = request.ExceptionMessage, + Range = new TimeRange(request.From, request.Until) + }; + + var stores = await findStoresAsync(request); + var list = new List(); + foreach (var store in stores) + { + var result = await store.DeadLetters.QueryAsync(query, cancellation); + result.DatabaseUri = store.Uri; + foreach (var envelope in result.Envelopes) + { + envelope.TryReadData(_runtime); + } + + list.Add(result); + } + + return list; + } + + } public class InvalidWolverineStorageConfigurationException : Exception From 68ec644d74b5d34a25ea8000f8bdaa198d357f46 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Sun, 21 Sep 2025 13:33:02 -0500 Subject: [PATCH 22/25] Converted the tests on the HTTP endpoints for DLQ admin --- .../dead_letter_endpoints.cs | 35 +++++++++++-------- .../DeadLettersEndpointExtensions.cs | 14 -------- src/Wolverine/Envelope.cs | 4 ++- .../Durability/DeadLetterEnvelope.cs | 4 +-- 4 files changed, 25 insertions(+), 32 deletions(-) diff --git a/src/Http/Wolverine.Http.Tests/dead_letter_endpoints.cs b/src/Http/Wolverine.Http.Tests/dead_letter_endpoints.cs index ad5ead2dd..e33b3ff28 100644 --- a/src/Http/Wolverine.Http.Tests/dead_letter_endpoints.cs +++ b/src/Http/Wolverine.Http.Tests/dead_letter_endpoints.cs @@ -33,18 +33,20 @@ await Host.TrackActivity().DoNotAssertOnExceptionsDetected().ExecuteAndWaitAsync }); // Expect - var deadletters = result.ReadAsJson(); + var deadletters = (result.ReadAsJson>()).Single(); + + deadletters - .ShouldNotBeNull().Messages.Count.ShouldBe(1); - deadletters.Messages[0].ExceptionType.ShouldBe(typeof(AlwaysDeadLetterException).FullNameInCode()); - deadletters.Messages[0].ExceptionMessage.ShouldBe(exceptionMessage); - deadletters.Messages[0].Body.ShouldNotBeNull(); - deadletters.Messages[0].Id.ShouldNotBe(default); - deadletters.Messages[0].MessageType.ShouldNotBeNull(); - deadletters.Messages[0].ReceivedAt.ShouldNotBeNull(); - deadletters.Messages[0].Source.ShouldNotBeNull(); - deadletters.Messages[0].SentAt.ShouldNotBe(default); - deadletters.Messages[0].Replayable.ShouldBeFalse(); + .ShouldNotBeNull().Envelopes.Count.ShouldBe(1); + deadletters.Envelopes[0].ExceptionType.ShouldBe(typeof(AlwaysDeadLetterException).FullNameInCode()); + deadletters.Envelopes[0].ExceptionMessage.ShouldBe(exceptionMessage); + deadletters.Envelopes[0].Message.ShouldNotBeNull(); + deadletters.Envelopes[0].Id.ShouldNotBe(default); + deadletters.Envelopes[0].MessageType.ShouldNotBeNull(); + deadletters.Envelopes[0].ReceivedAt.ShouldNotBeNull(); + deadletters.Envelopes[0].Source.ShouldNotBeNull(); + deadletters.Envelopes[0].SentAt.ShouldNotBe(default); + deadletters.Envelopes[0].Replayable.ShouldBeFalse(); } [Fact] @@ -71,12 +73,14 @@ await Host.TrackActivity().DoNotAssertOnExceptionsDetected().ExecuteAndWaitAsync }); // When & Expect - var deadletters = result.ReadAsJson(); + var all = result.ReadAsJson>(); + var id = all[0].Envelopes.Single().Id; + await Scenario(x => { x.Post.Json(new DeadLetterEnvelopeIdsRequest { - Ids = [deadletters.Messages.Single().Id] + Ids = [id] }).ToUrl("/dead-letters/replay"); }); } @@ -105,13 +109,14 @@ await Host.TrackActivity().DoNotAssertOnExceptionsDetected().ExecuteAndWaitAsync }); // When & Expect - var deadletters = result.ReadAsJson(); + var deadletters = result.ReadAsJson>(); + var id = deadletters[0].Envelopes[0].Id; await Scenario(x => { x.Delete.Json(new DeadLetterEnvelopeIdsRequest { - Ids = [deadletters.Messages.Single().Id] + Ids = [id] }).ToUrl("/dead-letters"); }); } diff --git a/src/Http/Wolverine.Http/DeadLettersEndpointExtensions.cs b/src/Http/Wolverine.Http/DeadLettersEndpointExtensions.cs index f0b9b486a..b9ee91966 100644 --- a/src/Http/Wolverine.Http/DeadLettersEndpointExtensions.cs +++ b/src/Http/Wolverine.Http/DeadLettersEndpointExtensions.cs @@ -14,20 +14,6 @@ public class DeadLetterEnvelopeIdsRequest public string? TenantId { get; set; } } -public record DeadLetterEnvelopesFoundResponse(IReadOnlyList Messages, int Total); - -public record DeadLetterEnvelopeResponse( - Guid Id, - DateTimeOffset? ExecutionTime, - object? Body, - string MessageType, - string ReceivedAt, - string Source, - string ExceptionType, - string ExceptionMessage, - DateTimeOffset SentAt, - bool Replayable); - public static class DeadLettersEndpointExtensions { /// diff --git a/src/Wolverine/Envelope.cs b/src/Wolverine/Envelope.cs index 9a79c2600..a5f635440 100644 --- a/src/Wolverine/Envelope.cs +++ b/src/Wolverine/Envelope.cs @@ -1,4 +1,5 @@ -using JasperFx; +using System.Text.Json.Serialization; +using JasperFx; using JasperFx.Core; using JasperFx.Core.Reflection; using JasperFx.MultiTenancy; @@ -90,6 +91,7 @@ public DateTimeOffset? ScheduledTime /// is retained for testing purposes /// /// + [JsonIgnore] public TimeSpan? DeliverWithin { set diff --git a/src/Wolverine/Persistence/Durability/DeadLetterEnvelope.cs b/src/Wolverine/Persistence/Durability/DeadLetterEnvelope.cs index ece0b2fbd..06cf21e31 100644 --- a/src/Wolverine/Persistence/Durability/DeadLetterEnvelope.cs +++ b/src/Wolverine/Persistence/Durability/DeadLetterEnvelope.cs @@ -44,7 +44,7 @@ bool replayable /// /// The actual message body /// - public object? Data { get; set; } + public object? Message { get; set; } internal void TryReadData(IWolverineRuntime runtime) { @@ -58,7 +58,7 @@ internal void TryReadData(IWolverineRuntime runtime) { try { - Data = serializer.ReadFromData(messageType, Envelope); + Message = serializer.ReadFromData(messageType, Envelope); } catch (Exception e) { From 2dc074c213d4e6f48b23514afae92a1432d6b1a1 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Sun, 21 Sep 2025 15:24:07 -0500 Subject: [PATCH 23/25] Full support for ancillary stores with EF Core or Marten. Closes GH-1703. Closes GH-1702. Closes GH-1701 Fixes for tests End to end modular monolith tests w/ ancillary store w/ Marten End to end modular monolith tests w/ EF Core WIP: end to end modular monolith testing Using the delegating outbox when there are ancillary stores in play --- ..._ancillary_marten_stores_with_wolverine.cs | 11 +- .../end_to_end_modular_monolith.cs | 343 ++++++++++++++++++ .../modular_monolith_usage.cs | 13 +- .../registration_of_message_stores.cs | 60 ++- .../PersistenceTests/PersistenceTests.csproj | 1 + .../Items/StartNewItem.cs | 9 + .../SqlServerBackedListenerContext.cs | 4 + .../Codegen/EFCorePersistenceFrameProvider.cs | 9 + ...cillaryWolverineOptionsMartenExtensions.cs | 12 +- .../PostgresqlBackedPersistence.cs | 28 ++ .../PostgresqlMessageStore.cs | 14 - .../Wolverine.RDBMS/MessageDatabase.cs | 5 + .../Internals/RavenDbMessageStore.cs | 5 + .../SqlServerBackedPersistence.cs | 33 +- .../SqlServerConfigurationExtensions.cs | 2 +- .../CoreTests/Runtime/MockWolverineRuntime.cs | 2 +- .../Configuration/EndpointCollection.cs | 6 +- src/Wolverine/HostBuilderExtensions.cs | 2 + .../Persistence/ApplyAncillaryStoreFrame.cs | 13 + .../Durability/DurableSendingAgent.cs | 4 +- .../Persistence/Durability/IMessageStore.cs | 30 +- .../Durability/MultiTenantedMessageStore.cs | 15 +- .../Durability/NullMessageStore.cs | 5 + .../Persistence/MessageStoreCollection.cs | 28 +- src/Wolverine/Runtime/IWolverineRuntime.cs | 2 +- src/Wolverine/Runtime/MessageBus.cs | 2 +- src/Wolverine/Runtime/WolverineRuntime.cs | 2 +- .../Runtime/WorkerQueues/DurableReceiver.cs | 5 +- 28 files changed, 595 insertions(+), 70 deletions(-) create mode 100644 src/Persistence/PersistenceTests/ModularMonoliths/end_to_end_modular_monolith.cs rename src/Persistence/PersistenceTests/{ => ModularMonoliths}/modular_monolith_usage.cs (90%) rename src/Persistence/PersistenceTests/{ => ModularMonoliths}/registration_of_message_stores.cs (82%) create mode 100644 src/Wolverine/Persistence/ApplyAncillaryStoreFrame.cs diff --git a/src/Persistence/MartenTests/AncillaryStores/bootstrapping_ancillary_marten_stores_with_wolverine.cs b/src/Persistence/MartenTests/AncillaryStores/bootstrapping_ancillary_marten_stores_with_wolverine.cs index dd8d2fade..ec602f24c 100644 --- a/src/Persistence/MartenTests/AncillaryStores/bootstrapping_ancillary_marten_stores_with_wolverine.cs +++ b/src/Persistence/MartenTests/AncillaryStores/bootstrapping_ancillary_marten_stores_with_wolverine.cs @@ -138,17 +138,18 @@ private async Task dropSchemaOnDatabase(string connectionString, string schemaNa public void registers_the_single_tenant_ancillary_store() { theHost.DocumentStore().ShouldNotBeNull(); - var ancillaries = theHost.Services.GetServices(); - ancillaries.OfType>().Any().ShouldBeTrue(); + var ancillaries = theHost.Services.GetServices(); + ancillaries.Select(x => x.MarkerType == typeof(IPlayerStore)).Any().ShouldBeTrue(); } [Fact] public void registers_the_multiple_tenant_ancillary_store() { theHost.DocumentStore().ShouldNotBeNull(); - var ancillaries = theHost.Services.GetServices(); - ancillaries.OfType>().Any() - .ShouldBeTrue(); + var ancillaries = theHost.Services.GetServices(); + ancillaries.Single(x => x.MarkerType == typeof(IThingStore)) + .Inner + .ShouldBeOfType(); } [Fact] diff --git a/src/Persistence/PersistenceTests/ModularMonoliths/end_to_end_modular_monolith.cs b/src/Persistence/PersistenceTests/ModularMonoliths/end_to_end_modular_monolith.cs new file mode 100644 index 000000000..3cd0a2031 --- /dev/null +++ b/src/Persistence/PersistenceTests/ModularMonoliths/end_to_end_modular_monolith.cs @@ -0,0 +1,343 @@ +using IntegrationTests; +using JasperFx; +using JasperFx.Core; +using JasperFx.Resources; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Hosting; +using Npgsql; +using Shouldly; +using Weasel.Core; +using Weasel.Postgresql; +using Weasel.Postgresql.Migrations; +using Weasel.SqlServer.Tables; +using Wolverine; +using Wolverine.EntityFrameworkCore; +using Wolverine.Persistence.Durability; +using Wolverine.RabbitMQ; +using Wolverine.Runtime; +using Wolverine.SqlServer; +using Wolverine.Tracking; +using Wolverine.Transports; +using Xunit; +using Marten; +using Microsoft.Data.SqlClient; +using Weasel.SqlServer; +using Wolverine.Marten; +using Wolverine.Persistence; + +namespace PersistenceTests.ModularMonoliths; + +public class MonolithFixture : IAsyncLifetime +{ + public MonolithFixture() + { + ItemsTable = new Table(new DbObjectName("mt_items", "items")); + ItemsTable.AddColumn("Id").AsPrimaryKey(); + ItemsTable.AddColumn("Name"); + ItemsTable.AddColumn("Approved"); + } + + public IHost Host { get; private set; } + + public Table ItemsTable { get; } + + public async Task InitializeAsync() + { + await using var conn = new NpgsqlConnection(Servers.PostgresConnectionString); + await conn.OpenAsync(); + var thingsConnectionString = await CreateDatabaseIfNotExists(conn, "things"); + await conn.CloseAsync(); + + Host = await Microsoft.Extensions.Hosting.Host.CreateDefaultBuilder() + .UseWolverine(opts => + { + opts.Durability.MessageStorageSchemaName = "wolverine"; + opts.Policies.UseDurableLocalQueues(); + opts.Policies.UseDurableOutboxOnAllSendingEndpoints(); + opts.Policies.AutoApplyTransactions(); + + + opts.Discovery.DisableConventionalDiscovery() + .IncludeType(typeof(StartAndTriggerApprovalHarness)); + + opts.UseRabbitMq().AutoProvision().AutoPurgeOnStartup(); + opts.PublishMessage().ToRabbitQueue("items"); + opts.PublishMessage().ToRabbitQueue("things"); + + // One EF Core integration... + opts.Services.AddDbContextWithWolverineIntegration(x => + x.UseSqlServer(Servers.SqlServerConnectionString)); + + opts.UseEntityFrameworkCoreTransactions(); + + // Ancillary Sql Server store + opts.PersistMessagesWithSqlServer(Servers.SqlServerConnectionString, role: MessageStoreRole.Ancillary) + .Enroll(); + + // Primary storage here... + opts.Services.AddMarten(m => + { + m.Connection(Servers.PostgresConnectionString); + m.DatabaseSchemaName = "marten"; + m.DisableNpgsqlLogging = true; + }).IntegrateWithWolverine(); + + // Ancillary postgresql + opts.Services.AddMartenStore(m => + { + m.Connection(thingsConnectionString); + m.DisableNpgsqlLogging = true; + }).IntegrateWithWolverine(); + + opts.Services.AddResourceSetupOnStartup(); + }).StartAsync(); + + // Make it empty... + await Host.RebuildAllEnvelopeStorageAsync(); + + await withItemsTable(); + } + + private async Task withItemsTable() + { + await using (var conn = new SqlConnection(Servers.SqlServerConnectionString)) + { + await conn.OpenAsync(); + var migration = await SchemaMigration.DetermineAsync(conn, ItemsTable); + if (migration.Difference != SchemaPatchDifference.None) + { + var sqlServerMigrator = new SqlServerMigrator(); + + await sqlServerMigrator.ApplyAllAsync(conn, migration, AutoCreate.CreateOrUpdate); + } + + await conn.CloseAsync(); + } + } + + public Task DisposeAsync() + { + return Host.StopAsync(); + } + + private async Task CreateDatabaseIfNotExists(NpgsqlConnection conn, string databaseName) + { + var builder = new NpgsqlConnectionStringBuilder(Servers.PostgresConnectionString); + + var exists = await conn.DatabaseExists(databaseName); + if (!exists) + { + await new DatabaseSpecification().BuildDatabase(conn, databaseName); + } + + builder.Database = databaseName; + + return builder.ConnectionString; + } +} + +public class end_to_end_modular_monolith : IClassFixture, IAsyncLifetime +{ + private readonly IHost theHost; + + public end_to_end_modular_monolith(MonolithFixture fixture) + { + theHost = fixture.Host; + } + + public async Task InitializeAsync() + { + // Make it empty... + await theHost.RebuildAllEnvelopeStorageAsync(); + } + + public Task DisposeAsync() + { + return Task.CompletedTask; + } + + [Fact] + public void has_ancillary_stores() + { + var runtime = theHost.GetRuntime(); + + runtime.Stores.HasAnyAncillaryStores().ShouldBeTrue(); + runtime.Stores.HasAncillaryStoreFor(typeof(ItemsDbContext)).ShouldBeTrue(); + runtime.Stores.HasAncillaryStoreFor(typeof(IThingStore)).ShouldBeTrue(); + } + + [Fact] + public async Task has_all_the_expected_databases() + { + var runtime = theHost.GetRuntime(); + var databases = (await runtime.Stores.FindAllAsync()).Select(x => x.Uri).OrderBy(x => x.ToString()).ToArray(); + + databases.ShouldBe([ + new Uri("wolverinedb://postgresql/localhost/postgres/wolverine"), + new Uri("wolverinedb://postgresql/localhost/things/wolverine"), + new Uri("wolverinedb://sqlserver/localhost/master/wolverine"), + + ] + ); + } + + [Fact] + public async Task use_outbox_with_ancillary_store_with_ef_core() + { + var message = new StartAndTriggerApproval(Guid.NewGuid(), "Fix this."); + var session = await theHost.SendMessageAndWaitAsync(message); + + var envelope = session.Sent.SingleEnvelope(); + envelope.Store.Uri.ShouldBe(new Uri("wolverinedb://sqlserver/localhost/master/wolverine")); + var messageId = envelope.Id; + + // Message should have been deleted + var stored = await envelope.Store.Outbox.LoadOutgoingAsync(envelope.Destination); + stored.Any(x => x.Id == messageId).ShouldBeFalse(); + } + + [Fact] + public async Task use_inbox_with_ancillary_store_with_ef_core() + { + var message = new StartAndScheduleApproval(Guid.NewGuid(), "Rip in shirt"); + var session = await theHost.SendMessageAndWaitAsync(message); + var scheduledEnvelope = session.Scheduled.SingleEnvelope(); + + scheduledEnvelope.Store.Uri.ShouldBe(new Uri("wolverinedb://sqlserver/localhost/master/wolverine")); + + var stored = await scheduledEnvelope.Store.Admin.AllIncomingAsync(); + var persisted = stored.Where(x => x.MessageType == TransportConstants.ScheduledEnvelope).Single(); + persisted.Destination.ShouldBe(new Uri("local://durable")); + + var second = await session.PlayScheduledMessagesAsync(10.Seconds()); + var envelope = second.Sent.SingleEnvelope(); + envelope.Destination.ShouldBe(new Uri("rabbitmq://queue/items")); + + var all = await theHost.GetRuntime().Stores.FindAllAsync(); + foreach (var store in all) + { + var outgoing = await store.Admin.AllOutgoingAsync(); + outgoing.Any().ShouldBeFalse($"Database {store.Uri} has {outgoing.Count} envelopes in the outbox"); + } + + } + + [Fact] + public async Task use_outbox_with_ancillary_store_with_marten() + { + var message = new StartAndTriggerThing(Guid.NewGuid(), "Blue Shirt"); + var session = await theHost.SendMessageAndWaitAsync(message); + + var envelope = session.Sent.SingleEnvelope(); + envelope.Store.Uri.ShouldBe(new Uri("wolverinedb://postgresql/localhost/postgres/wolverine")); + var messageId = envelope.Id; + + // Message should have been deleted + var stored = await envelope.Store.Outbox.LoadOutgoingAsync(envelope.Destination); + stored.Any(x => x.Id == messageId).ShouldBeFalse(); + } + + [Fact] + public async Task use_inbox_with_ancillary_store_with_marten() + { + var message = new StartAndScheduleThing(Guid.NewGuid(), "Green Shirt"); + var session = await theHost.SendMessageAndWaitAsync(message); + var scheduledEnvelope = session.Scheduled.SingleEnvelope(); + + scheduledEnvelope.Store.Uri.ShouldBe(new Uri("wolverinedb://postgresql/localhost/postgres/wolverine")); + + var stored = await scheduledEnvelope.Store.Admin.AllIncomingAsync(); + var persisted = stored.Where(x => x.MessageType == TransportConstants.ScheduledEnvelope).Single(); + persisted.Destination.ShouldBe(new Uri("local://durable")); + + var second = await session.PlayScheduledMessagesAsync(10.Seconds()); + var envelope = second.Sent.SingleEnvelope(); + envelope.Destination.ShouldBe(new Uri("rabbitmq://queue/things")); + + var all = await theHost.GetRuntime().Stores.FindAllAsync(); + foreach (var store in all) + { + var outgoing = await store.Admin.AllOutgoingAsync(); + outgoing.Any().ShouldBeFalse($"Database {store.Uri} has {outgoing.Count} envelopes in the outbox"); + } + + } + +} + +public record ApproveItem1(Guid Id); + +public record StartAndTriggerApproval(Guid Id, string Name); +public record StartAndScheduleApproval(Guid Id, string Name); + + +public static class StartAndTriggerApprovalHarness +{ + public static ApproveItem1 Handle(StartAndTriggerApproval command, ItemsDbContext dbContext) + { + dbContext.Items.Add(new Item() { Id = command.Id, Name = command.Name }); + return new ApproveItem1(command.Id); + } + + public static object Handle(StartAndScheduleApproval command, ItemsDbContext dbContext) + { + dbContext.Items.Add(new Item() { Id = command.Id, Name = command.Name }); + return new ApproveItem1(command.Id).DelayedFor(1.Hours()); + } + + public static (IStorageAction, ApproveThing) Handle(StartAndTriggerThing command) + { + var storageAction = Storage.Insert(new Thing { Id = command.Id, Name = command.Name }); + return (storageAction, new ApproveThing(command.Id)); + } + + public static (IStorageAction, object) Handle(StartAndScheduleThing command) + { + var storageAction = Storage.Insert(new Thing { Id = command.Id, Name = command.Name }); + return (storageAction, new ApproveThing(command.Id).DelayedFor(1.Hours())); + } +} + +public class ItemsDbContext : DbContext +{ + public ItemsDbContext(DbContextOptions options) : base(options) + { + } + + public DbSet Items { get; set; } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + // Your normal EF Core mapping + modelBuilder.Entity(map => + { + map.ToTable("items", "mt_items"); + map.HasKey(x => x.Id); + map.Property(x => x.Id).HasColumnName("id"); + map.Property(x => x.Name).HasColumnName("name"); + map.Property(x => x.Approved).HasColumnName("approved"); + }); + } +} + +public class Item +{ + public Guid Id { get; set; } + public string Name { get; set; } + + public bool Approved { get; set; } +} + +public record StartAndTriggerThing(Guid Id, string Name); +public record StartAndScheduleThing(Guid Id, string Name); + +public record ApproveThing(Guid Id); + +public class Thing +{ + public Guid Id { get; set; } + public string Name { get; set; } +} + + + diff --git a/src/Persistence/PersistenceTests/modular_monolith_usage.cs b/src/Persistence/PersistenceTests/ModularMonoliths/modular_monolith_usage.cs similarity index 90% rename from src/Persistence/PersistenceTests/modular_monolith_usage.cs rename to src/Persistence/PersistenceTests/ModularMonoliths/modular_monolith_usage.cs index 369e3bcdc..26bf35de0 100644 --- a/src/Persistence/PersistenceTests/modular_monolith_usage.cs +++ b/src/Persistence/PersistenceTests/ModularMonoliths/modular_monolith_usage.cs @@ -1,17 +1,15 @@ using IntegrationTests; -using JasperFx.Core.Reflection; using JasperFx.Resources; using Marten; using Microsoft.Extensions.Hosting; using Shouldly; using Wolverine; using Wolverine.Marten; -using Wolverine.Persistence.Durability; using Wolverine.Postgresql; using Wolverine.Tracking; using Xunit; -namespace PersistenceTests; +namespace PersistenceTests.ModularMonoliths; public interface IPlayerStore : IDocumentStore; public interface IThingStore : IDocumentStore; @@ -143,12 +141,11 @@ public async Task using_the_marten_schema_name_with_no_other_settings() }).StartAsync(); var runtime = host.GetRuntime(); - var stores = (await runtime.Stores.FindAllAsync()).OfType().ToArray(); - stores.OfType>().Single().As() - .Settings.SchemaName.ShouldBe("players"); + runtime.Stores.HasAnyAncillaryStores().ShouldBeTrue(); - stores.OfType>().Single().As() - .Settings.SchemaName.ShouldBe("things"); + runtime.Stores.FindAncillaryStore(typeof(IPlayerStore)).ShouldBeOfType() .Settings.SchemaName.ShouldBe("players"); + runtime.Stores.FindAncillaryStore(typeof(IThingStore)).ShouldBeOfType() .Settings.SchemaName.ShouldBe("things"); } + } \ No newline at end of file diff --git a/src/Persistence/PersistenceTests/registration_of_message_stores.cs b/src/Persistence/PersistenceTests/ModularMonoliths/registration_of_message_stores.cs similarity index 82% rename from src/Persistence/PersistenceTests/registration_of_message_stores.cs rename to src/Persistence/PersistenceTests/ModularMonoliths/registration_of_message_stores.cs index 9687af7a2..f1d617061 100644 --- a/src/Persistence/PersistenceTests/registration_of_message_stores.cs +++ b/src/Persistence/PersistenceTests/ModularMonoliths/registration_of_message_stores.cs @@ -14,12 +14,13 @@ using Wolverine.Persistence; using Wolverine.Persistence.Durability; using Wolverine.Postgresql; +using Wolverine.Runtime; using Wolverine.SqlServer; using Wolverine.Tracking; using Xunit; using Xunit.Abstractions; -namespace PersistenceTests; +namespace PersistenceTests.ModularMonoliths; /* * TODO -- register multiple postgresql outside of Marten @@ -83,7 +84,7 @@ private async Task startHost(Action co _host = await Host.CreateDefaultBuilder() .UseWolverine(configure).ConfigureServices(services => services.AddResourceSetupOnStartup()).StartAsync(); - var collection = new MessageStoreCollection(_host.GetRuntime(), _host.Services.GetServices(), _host.Services.GetServices()); + var collection = new MessageStoreCollection(_host.GetRuntime(), _host.Services.GetServices(), _host.Services.GetServices()); await collection.InitializeAsync(); @@ -350,6 +351,61 @@ public async Task register_one_main_and_one_ancillary_sql_server() }); } + + [Fact] + public async Task basic_ancillary_store_registration_with_postgresql() + { + var collection = await startHost(opts => + { + opts.PersistMessagesWithPostgresql(Servers.PostgresConnectionString, "main"); + + opts.PersistMessagesWithPostgresql(connectionString1, role:MessageStoreRole.Ancillary).Enroll(typeof(SampleDbContext)); + }); + + collection.HasAnyAncillaryStores().ShouldBeTrue(); + var databases = await collection.FindAllAsync(); + databases.Select(x => x.Uri).OrderBy(x => x.ToString()).ShouldBe([new Uri("wolverinedb://postgresql/localhost/database1/wolverine"), new Uri("wolverinedb://postgresql/localhost/postgres/main")]); + + var store = _host.Services.GetRequiredService(); + store.MarkerType.ShouldBe(typeof(SampleDbContext)); + store.Inner.Uri.ShouldBe(new Uri("wolverinedb://postgresql/localhost/database1/wolverine")); + } + + [Fact] + public async Task basic_ancillary_store_registration_with_sql_server() + { + var collection = await startHost(opts => + { + opts.PersistMessagesWithPostgresql(Servers.PostgresConnectionString, "main"); + + opts.PersistMessagesWithSqlServer(Servers.SqlServerConnectionString, role:MessageStoreRole.Ancillary).Enroll(typeof(SampleDbContext)); + }); + + collection.HasAnyAncillaryStores().ShouldBeTrue(); + var databases = await collection.FindAllAsync(); + databases.Select(x => x.Uri).OrderBy(x => x.ToString()).ShouldBe([new Uri("wolverinedb://postgresql/localhost/postgres/main"), new Uri("wolverinedb://sqlserver/localhost/master/dbo")]); + + var store = _host.Services.GetRequiredService(); + store.MarkerType.ShouldBe(typeof(SampleDbContext)); + store.Inner.Uri.ShouldBe(new Uri("wolverinedb://sqlserver/localhost/master/dbo")); + } + + [Fact] + public async Task register_AncillaryMessageStoreApplication() + { + var collection = await startHost(opts => + { + opts.PersistMessagesWithPostgresql(Servers.PostgresConnectionString, "main"); + + opts.PersistMessagesWithSqlServer(Servers.SqlServerConnectionString, role:MessageStoreRole.Ancillary).Enroll(typeof(SampleDbContext)); + }); + + var application = _host.Services.GetRequiredService>(); + + var context = new MessageContext(_host.GetRuntime()); + application.Apply(context); + context.Storage.Uri.ShouldBe(new Uri("wolverinedb://sqlserver/localhost/master/dbo")); + } } diff --git a/src/Persistence/PersistenceTests/PersistenceTests.csproj b/src/Persistence/PersistenceTests/PersistenceTests.csproj index eeda39c15..cced4a65a 100644 --- a/src/Persistence/PersistenceTests/PersistenceTests.csproj +++ b/src/Persistence/PersistenceTests/PersistenceTests.csproj @@ -19,6 +19,7 @@ + diff --git a/src/Persistence/SharedPersistenceModels/Items/StartNewItem.cs b/src/Persistence/SharedPersistenceModels/Items/StartNewItem.cs index e3c388a26..5b7e6df71 100644 --- a/src/Persistence/SharedPersistenceModels/Items/StartNewItem.cs +++ b/src/Persistence/SharedPersistenceModels/Items/StartNewItem.cs @@ -1,3 +1,5 @@ +using Humanizer; +using Wolverine; using Wolverine.Http; using Wolverine.Persistence; @@ -19,6 +21,7 @@ public static IStorageAction Handle(StartNewItem command) } public record StartAndTriggerApproval(Guid Id, string Name); +public record StartAndScheduleApproval(Guid Id, string Name); public static class StartAndTriggerApprovalHandler { @@ -27,4 +30,10 @@ public static (IStorageAction, ApproveItem1) Handle(StartAndTriggerApprova var storageAction = Storage.Insert(new Item { Id = command.Id, Name = command.Name }); return (storageAction, new ApproveItem1(command.Id)); } + + public static (IStorageAction, object) Handle(StartAndScheduleApproval command) + { + var storageAction = Storage.Insert(new Item { Id = command.Id, Name = command.Name }); + return (storageAction, new ApproveItem1(command.Id).DelayedFor(1.Hours())); + } } \ No newline at end of file diff --git a/src/Persistence/SqlServerTests/Persistence/SqlServerBackedListenerContext.cs b/src/Persistence/SqlServerTests/Persistence/SqlServerBackedListenerContext.cs index 77591d577..3bec2aca8 100644 --- a/src/Persistence/SqlServerTests/Persistence/SqlServerBackedListenerContext.cs +++ b/src/Persistence/SqlServerTests/Persistence/SqlServerBackedListenerContext.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; using Wolverine; +using Wolverine.Persistence; using Wolverine.RDBMS; using Wolverine.RDBMS.Sagas; using Wolverine.Runtime; @@ -34,6 +35,9 @@ public SqlServerBackedListenerContext() var runtime = Substitute.For(); runtime.Storage.Returns(thePersistence); runtime.Pipeline.Returns(thePipeline); + runtime.Options.Returns(new WolverineOptions()); + + //runtime.Stores.Returns(new MessageStoreCollection(runtime, [], [])); runtime.DurabilitySettings.Returns(theSettings); diff --git a/src/Persistence/Wolverine.EntityFrameworkCore/Codegen/EFCorePersistenceFrameProvider.cs b/src/Persistence/Wolverine.EntityFrameworkCore/Codegen/EFCorePersistenceFrameProvider.cs index 6d4873d3e..10775ebdf 100644 --- a/src/Persistence/Wolverine.EntityFrameworkCore/Codegen/EFCorePersistenceFrameProvider.cs +++ b/src/Persistence/Wolverine.EntityFrameworkCore/Codegen/EFCorePersistenceFrameProvider.cs @@ -114,6 +114,13 @@ public void ApplyTransactionSupport(IChain chain, IServiceContainer container) chain.Tags.Add(UsingEfCoreTransaction, true); var dbContextType = DetermineDbContextType(chain, container); + var runtime = container.Services.GetRequiredService(); + if (runtime.Stores.HasAncillaryStoreFor(dbContextType)) + { + var frame = typeof(ApplyAncillaryStoreFrame<>).CloseAndBuildAs(dbContextType); + chain.Middleware.Insert(0, frame); + } + if (isMultiTenanted(container, dbContextType)) { var createContext = typeof(CreateTenantedDbContext<>).CloseAndBuildAs(dbContextType); @@ -123,6 +130,8 @@ public void ApplyTransactionSupport(IChain chain, IServiceContainer container) { chain.Middleware.Insert(0, new EnrollDbContextInTransaction(dbContextType)); } + + var saveChangesAsync = dbContextType.GetMethod(nameof(DbContext.SaveChangesAsync), [typeof(CancellationToken)]); diff --git a/src/Persistence/Wolverine.Marten/AncillaryWolverineOptionsMartenExtensions.cs b/src/Persistence/Wolverine.Marten/AncillaryWolverineOptionsMartenExtensions.cs index f6d1a02a2..f6060e64d 100644 --- a/src/Persistence/Wolverine.Marten/AncillaryWolverineOptionsMartenExtensions.cs +++ b/src/Persistence/Wolverine.Marten/AncillaryWolverineOptionsMartenExtensions.cs @@ -62,7 +62,7 @@ public static MartenServiceCollectionExtensions.MartenStoreExpression Integra expression.Services.AddSingleton, MartenOverrides>(); - expression.Services.AddSingleton(s => + expression.Services.AddSingleton(s => { var store = s.GetRequiredService().As(); @@ -103,7 +103,7 @@ internal static NpgsqlDataSource findMasterDataSource(DocumentStore store, "There is no configured connectivity for the required master PostgreSQL message database"); } - internal static IAncillaryMessageStore BuildMultiTenantedMessageDatabase(string schemaName, + internal static AncillaryMessageStore BuildMultiTenantedMessageDatabase(string schemaName, AutoCreate? autoCreate, string? masterDatabaseConnectionString, NpgsqlDataSource? masterDataSource, @@ -121,7 +121,7 @@ internal static IAncillaryMessageStore BuildMultiTenantedMessageDatabase(s }; var dataSource = findMasterDataSource(store, mainSettings); - var master = new PostgresqlMessageStore(mainSettings, runtime.Options.Durability, dataSource, + var master = new PostgresqlMessageStore(mainSettings, runtime.Options.Durability, dataSource, runtime.LoggerFactory.CreateLogger()) { Name = "Master", @@ -131,10 +131,10 @@ internal static IAncillaryMessageStore BuildMultiTenantedMessageDatabase(s master.Initialize(runtime); - return new MultiTenantedMessageStore(master, runtime, source); + return new(typeof(T), new MultiTenantedMessageStore(master, runtime, source)); } - internal static IAncillaryMessageStore BuildSinglePostgresqlMessageStore( + internal static AncillaryMessageStore BuildSinglePostgresqlMessageStore( string schemaName, AutoCreate? autoCreate, DocumentStore store, @@ -151,7 +151,7 @@ internal static IAncillaryMessageStore BuildSinglePostgresqlMessageStore( var dataSource = store.Storage.Database.As().DataSource; - return new PostgresqlMessageStore(settings, runtime.Options.Durability, dataSource, logger); + return new(typeof(T), new PostgresqlMessageStore(settings, runtime.Options.Durability, dataSource, logger)) ; } /// diff --git a/src/Persistence/Wolverine.Postgresql/PostgresqlBackedPersistence.cs b/src/Persistence/Wolverine.Postgresql/PostgresqlBackedPersistence.cs index 9317f86e3..dfc3a45b9 100644 --- a/src/Persistence/Wolverine.Postgresql/PostgresqlBackedPersistence.cs +++ b/src/Persistence/Wolverine.Postgresql/PostgresqlBackedPersistence.cs @@ -26,6 +26,22 @@ public interface IPostgresqlBackedPersistence /// IPostgresqlBackedPersistence EnableMessageTransport(Action? configure = null); + /// + /// Tell Wolverine that the persistence service (Marten? EF Core DbContext? Something else?) of the given + /// type should be enrolled in envelope storage with this PostgreSQL database + /// + /// + /// + IPostgresqlBackedPersistence Enroll(Type type); + + /// + /// Tell Wolverine that the persistence service (Marten? EF Core DbContext? Something else?) of the given + /// type should be enrolled in envelope storage with this PostgreSQL database + /// + /// + /// + IPostgresqlBackedPersistence Enroll(); + /// /// By default, Wolverine takes the AutoCreate settings from JasperFxOptions, but /// you can override the application default for just the PostgreSQL backed queues @@ -256,6 +272,18 @@ public IPostgresqlBackedPersistence EnableMessageTransport(Action(s => new (type,BuildMessageStore(s.GetRequiredService()))); + + return this; + } + + public IPostgresqlBackedPersistence Enroll() + { + return Enroll(typeof(T)); + } + IPostgresqlBackedPersistence IPostgresqlBackedPersistence.OverrideAutoCreateResources(AutoCreate autoCreate) { AutoCreate = autoCreate; diff --git a/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs b/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs index 6631a7ac0..6431a142f 100644 --- a/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs +++ b/src/Persistence/Wolverine.Postgresql/PostgresqlMessageStore.cs @@ -25,20 +25,6 @@ namespace Wolverine.Postgresql; -/// -/// Built to work with separate Marten stores -/// -/// -internal class PostgresqlMessageStore : PostgresqlMessageStore, IAncillaryMessageStore -{ - public PostgresqlMessageStore(DatabaseSettings databaseSettings, DurabilitySettings settings, NpgsqlDataSource dataSource, ILogger logger) : base(databaseSettings, settings, dataSource, logger) - { - - } - - public Type MarkerType => typeof(T); -} - internal class PostgresqlMessageStore : MessageDatabase { private readonly string _deleteOutgoingEnvelopesSql; diff --git a/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs b/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs index ecabc53ff..d2ecf4f6d 100644 --- a/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs +++ b/src/Persistence/Wolverine.RDBMS/MessageDatabase.cs @@ -115,6 +115,11 @@ public void PromoteToMain(IWolverineRuntime runtime) Initialize(runtime); } + public void DemoteToAncillary() + { + Role = MessageStoreRole.Ancillary; + } + public DatabaseSettings Settings => _settings; public INodeAgentPersistence Nodes { get; } diff --git a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.cs b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.cs index 08e2ff946..40c7ec457 100644 --- a/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.cs +++ b/src/Persistence/Wolverine.RavenDb/Internals/RavenDbMessageStore.cs @@ -35,6 +35,11 @@ public void PromoteToMain(IWolverineRuntime runtime) Role = MessageStoreRole.Main; } + public void DemoteToAncillary() + { + Role = MessageStoreRole.Ancillary; + } + public string Name => _store.Identifier; public Uri Uri => new("ravendb://durability"); diff --git a/src/Persistence/Wolverine.SqlServer/SqlServerBackedPersistence.cs b/src/Persistence/Wolverine.SqlServer/SqlServerBackedPersistence.cs index 09e5028d0..837f5f065 100644 --- a/src/Persistence/Wolverine.SqlServer/SqlServerBackedPersistence.cs +++ b/src/Persistence/Wolverine.SqlServer/SqlServerBackedPersistence.cs @@ -91,6 +91,19 @@ public interface ISqlServerBackedPersistence /// ISqlServerBackedPersistence UseMasterTableTenancy(Action configure); + /// + /// Tell Wolverine that the persistence service (Marten? EF Core DbContext? Something else?) of the given + /// type should be enrolled in envelope storage with this PostgreSQL database + /// + /// + ISqlServerBackedPersistence Enroll(Type serviceType); + + /// + /// Tell Wolverine that the persistence service (Marten? EF Core DbContext? Something else?) of the given + /// type should be enrolled in envelope storage with this PostgreSQL database + /// + /// + ISqlServerBackedPersistence Enroll(); } /// @@ -98,6 +111,13 @@ public interface ISqlServerBackedPersistence /// internal class SqlServerBackedPersistence : IWolverineExtension, ISqlServerBackedPersistence { + private readonly WolverineOptions _options; + + public SqlServerBackedPersistence(WolverineOptions options) + { + _options = options; + } + public string? ConnectionString { get; set; } public string EnvelopeStorageSchemaName { get; set; } = "wolverine"; @@ -266,7 +286,18 @@ ISqlServerBackedPersistence ISqlServerBackedPersistence.UseMasterTableTenancy(Ac TenantConnections = source; return this; } - + + public ISqlServerBackedPersistence Enroll(Type serviceType) + { + _options.Services.AddSingleton(s => new (serviceType,BuildMessageStore(s.GetRequiredService()))); + return this; + } + + public ISqlServerBackedPersistence Enroll() + { + return Enroll(typeof(T)); + } + /// /// This is any default connection strings by tenant that should be loaded at start up time /// diff --git a/src/Persistence/Wolverine.SqlServer/SqlServerConfigurationExtensions.cs b/src/Persistence/Wolverine.SqlServer/SqlServerConfigurationExtensions.cs index 446f58976..f7e6c3b89 100644 --- a/src/Persistence/Wolverine.SqlServer/SqlServerConfigurationExtensions.cs +++ b/src/Persistence/Wolverine.SqlServer/SqlServerConfigurationExtensions.cs @@ -20,7 +20,7 @@ public static class SqlServerConfigurationExtensions public static ISqlServerBackedPersistence PersistMessagesWithSqlServer(this WolverineOptions options, string connectionString, string? schema = null, MessageStoreRole role = MessageStoreRole.Main) { - var extension = new SqlServerBackedPersistence + var extension = new SqlServerBackedPersistence(options) { ConnectionString = connectionString, Role = role diff --git a/src/Testing/CoreTests/Runtime/MockWolverineRuntime.cs b/src/Testing/CoreTests/Runtime/MockWolverineRuntime.cs index 59b4620a0..2b82fe8ac 100644 --- a/src/Testing/CoreTests/Runtime/MockWolverineRuntime.cs +++ b/src/Testing/CoreTests/Runtime/MockWolverineRuntime.cs @@ -106,7 +106,7 @@ public IMessageRouter RoutingFor(Type messageType) throw new NotImplementedException(); } - public IAncillaryMessageStore FindAncillaryStoreForMarkerType(Type markerType) + public IMessageStore FindAncillaryStoreForMarkerType(Type markerType) { throw new NotImplementedException(); } diff --git a/src/Wolverine/Configuration/EndpointCollection.cs b/src/Wolverine/Configuration/EndpointCollection.cs index db07ebd43..ebaaa9a1c 100644 --- a/src/Wolverine/Configuration/EndpointCollection.cs +++ b/src/Wolverine/Configuration/EndpointCollection.cs @@ -313,9 +313,13 @@ private ISendingAgent buildSendingAgent(ISender sender, Endpoint endpoint) switch (endpoint.Mode) { case EndpointMode.Durable: + var outbox = _runtime.Stores.HasAnyAncillaryStores() + ? new DelegatingMessageOutbox(_runtime.Storage.Outbox, _runtime.Stores) + : _runtime.Storage.Outbox; + return new DurableSendingAgent(sender, _options.Durability, _runtime.LoggerFactory.CreateLogger(), _runtime.MessageTracking, - _runtime.Storage, endpoint); + outbox, endpoint); case EndpointMode.BufferedInMemory: return new BufferedSendingAgent(_runtime.LoggerFactory.CreateLogger(), diff --git a/src/Wolverine/HostBuilderExtensions.cs b/src/Wolverine/HostBuilderExtensions.cs index 1ccae6629..344b76e52 100644 --- a/src/Wolverine/HostBuilderExtensions.cs +++ b/src/Wolverine/HostBuilderExtensions.cs @@ -98,6 +98,8 @@ internal static IServiceCollection AddWolverine(this IServiceCollection services services.AddJasperFx(); services.AddSingleton(); services.AddSingleton(); + + services.AddSingleton(typeof(AncillaryMessageStoreApplication<>)); services.AddSingleton(services); diff --git a/src/Wolverine/Persistence/ApplyAncillaryStoreFrame.cs b/src/Wolverine/Persistence/ApplyAncillaryStoreFrame.cs new file mode 100644 index 000000000..bee5217a5 --- /dev/null +++ b/src/Wolverine/Persistence/ApplyAncillaryStoreFrame.cs @@ -0,0 +1,13 @@ +using System.Reflection; +using JasperFx.CodeGeneration.Frames; +using Wolverine.Persistence.Durability; + +namespace Wolverine.Persistence; + +public class ApplyAncillaryStoreFrame : MethodCall +{ + public ApplyAncillaryStoreFrame() : base(typeof(AncillaryMessageStoreApplication ), "Apply") + { + } + +} \ No newline at end of file diff --git a/src/Wolverine/Persistence/Durability/DurableSendingAgent.cs b/src/Wolverine/Persistence/Durability/DurableSendingAgent.cs index 8d20880ee..8882d50d2 100644 --- a/src/Wolverine/Persistence/Durability/DurableSendingAgent.cs +++ b/src/Wolverine/Persistence/Durability/DurableSendingAgent.cs @@ -22,11 +22,11 @@ internal class DurableSendingAgent : SendingAgent public DurableSendingAgent(ISender sender, DurabilitySettings settings, ILogger logger, IMessageTracker messageLogger, - IMessageStore persistence, Endpoint endpoint) : base(logger, messageLogger, sender, settings, endpoint) + IMessageOutbox outbox, Endpoint endpoint) : base(logger, messageLogger, sender, settings, endpoint) { _logger = logger; - _outbox = persistence.Outbox; + _outbox = outbox; _deleteOutgoingOne = new RetryBlock((e, _) => _outbox.DeleteOutgoingAsync(e), logger, settings.Cancellation); diff --git a/src/Wolverine/Persistence/Durability/IMessageStore.cs b/src/Wolverine/Persistence/Durability/IMessageStore.cs index 4b52672ca..1ccc7998d 100644 --- a/src/Wolverine/Persistence/Durability/IMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/IMessageStore.cs @@ -117,20 +117,38 @@ public interface IMessageStore : IAsyncDisposable Task ReassignIncomingAsync(int ownerId, IReadOnlyList incoming); void PromoteToMain(IWolverineRuntime runtime); + void DemoteToAncillary(); } public record IncomingCount(Uri Destination, int Count); -/// -/// Marks a secondary message store for a Wolverine application -/// -public interface IAncillaryMessageStore : IMessageStore +public class AncillaryMessageStoreApplication { - Type MarkerType { get; } + private readonly IMessageStore? _store; + + public AncillaryMessageStoreApplication(IWolverineRuntime runtime) + { + _store = runtime.Stores.FindAncillaryStore(typeof(T)); + } + + public void Apply(MessageContext context) + { + context.Storage = _store; + } } -public interface IAncillaryMessageStore : IAncillaryMessageStore +public class AncillaryMessageStore { + public Type MarkerType { get; } + public IMessageStore Inner { get; } + + public AncillaryMessageStore(Type markerType, IMessageStore inner) + { + MarkerType = markerType; + Inner = inner; + + inner.DemoteToAncillary(); + } } public interface ITenantedMessageSource : ITenantedSource diff --git a/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs b/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs index 950415a48..90bb41d94 100644 --- a/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/MultiTenantedMessageStore.cs @@ -14,16 +14,6 @@ namespace Wolverine.Persistence.Durability; -public class MultiTenantedMessageStore : MultiTenantedMessageStore, IAncillaryMessageStore -{ - public MultiTenantedMessageStore(IMessageStore main, IWolverineRuntime runtime, ITenantedMessageSource source) : - base(main, runtime, source) - { - } - - public Type MarkerType => typeof(T); -} - public partial class MultiTenantedMessageStore : IMessageStore, IMessageInbox, IMessageOutbox, IMessageStoreAdmin, IDeadLetters, ISagaSupport, INodeAgentPersistence { @@ -48,6 +38,11 @@ public MultiTenantedMessageStore(IMessageStore main, IWolverineRuntime runtime, Main = main; } + public void DemoteToAncillary() + { + Main.DemoteToAncillary(); + } + public MessageStoreRole Role => MessageStoreRole.Composite; public ITenantedMessageSource Source { get; } diff --git a/src/Wolverine/Persistence/Durability/NullMessageStore.cs b/src/Wolverine/Persistence/Durability/NullMessageStore.cs index fc459a462..66c3af8c7 100644 --- a/src/Wolverine/Persistence/Durability/NullMessageStore.cs +++ b/src/Wolverine/Persistence/Durability/NullMessageStore.cs @@ -29,6 +29,11 @@ public void PromoteToMain(IWolverineRuntime runtime) } + public void DemoteToAncillary() + { + + } + public Task MarkIncomingEnvelopeAsHandledAsync(IReadOnlyList envelopes) { return Task.CompletedTask; diff --git a/src/Wolverine/Persistence/MessageStoreCollection.cs b/src/Wolverine/Persistence/MessageStoreCollection.cs index 858a694b2..362cf270c 100644 --- a/src/Wolverine/Persistence/MessageStoreCollection.cs +++ b/src/Wolverine/Persistence/MessageStoreCollection.cs @@ -14,14 +14,14 @@ public class MessageStoreCollection : IAgentFamily, IAsyncDisposable private readonly IWolverineRuntime _runtime; private readonly List _multiTenanted = new(); private ImHashMap _services = ImHashMap.Empty; - private ImHashMap _ancillaryStores = ImHashMap.Empty; + private ImHashMap _ancillaryStores = ImHashMap.Empty; private bool _onlyOneDatabase; - public MessageStoreCollection(IWolverineRuntime runtime, IEnumerable stores, IEnumerable ancillaryMessageStores) + public MessageStoreCollection(IWolverineRuntime runtime, IEnumerable stores, IEnumerable ancillaryMessageStores) { _runtime = runtime; - foreach (var store in stores.Concat(ancillaryMessageStores)) + foreach (var store in stores.Concat(ancillaryMessageStores.Select(x => x.Inner))) { if (store is MultiTenantedMessageStore multiTenanted) { @@ -36,7 +36,12 @@ public MessageStoreCollection(IWolverineRuntime runtime, IEnumerable MultiTenanted => _multiTenanted; private void categorizeStore(IMessageStore store) @@ -249,7 +259,7 @@ public async ValueTask> FindDatabasesAsync(Uri[] ur return list; } - public IAncillaryMessageStore FindAncillaryStore(Type markerType) + public IMessageStore FindAncillaryStore(Type markerType) { if (_ancillaryStores.TryFind(markerType, out var store)) return store; @@ -452,8 +462,12 @@ public async Task> FetchDeadLetterEnvel return list; } - - + + + public bool HasAncillaryStoreFor(Type applicationType) + { + return _ancillaryStores.Contains(applicationType); + } } public class InvalidWolverineStorageConfigurationException : Exception diff --git a/src/Wolverine/Runtime/IWolverineRuntime.cs b/src/Wolverine/Runtime/IWolverineRuntime.cs index 89c105a86..eff6ff6a2 100644 --- a/src/Wolverine/Runtime/IWolverineRuntime.cs +++ b/src/Wolverine/Runtime/IWolverineRuntime.cs @@ -41,7 +41,7 @@ public interface IWolverineRuntime /// Task TryFindMainMessageStore() where T : class; - IAncillaryMessageStore FindAncillaryStoreForMarkerType(Type markerType); + IMessageStore FindAncillaryStoreForMarkerType(Type markerType); /// /// Schedule an envelope for later execution in memory diff --git a/src/Wolverine/Runtime/MessageBus.cs b/src/Wolverine/Runtime/MessageBus.cs index c5c41dfe9..fb66ffde0 100644 --- a/src/Wolverine/Runtime/MessageBus.cs +++ b/src/Wolverine/Runtime/MessageBus.cs @@ -54,7 +54,7 @@ public virtual Task ReScheduleCurrentAsync(DateTimeOffset rescheduledAt) } public IWolverineRuntime Runtime { get; } - public IMessageStore Storage { get; protected set; } + public IMessageStore Storage { get; internal set; } public IEnumerable Outstanding => _outstanding; diff --git a/src/Wolverine/Runtime/WolverineRuntime.cs b/src/Wolverine/Runtime/WolverineRuntime.cs index 3156f47ab..c25c542c5 100644 --- a/src/Wolverine/Runtime/WolverineRuntime.cs +++ b/src/Wolverine/Runtime/WolverineRuntime.cs @@ -189,7 +189,7 @@ public void ScheduleLocalExecutionInMemory(DateTimeOffset executionTime, Envelop ScheduledJobs.Enqueue(executionTime, envelope); } - public IAncillaryMessageStore FindAncillaryStoreForMarkerType(Type markerType) + public IMessageStore FindAncillaryStoreForMarkerType(Type markerType) { return _stores.Value.FindAncillaryStore(markerType); } diff --git a/src/Wolverine/Runtime/WorkerQueues/DurableReceiver.cs b/src/Wolverine/Runtime/WorkerQueues/DurableReceiver.cs index d2bda4ca5..8b03a6001 100644 --- a/src/Wolverine/Runtime/WorkerQueues/DurableReceiver.cs +++ b/src/Wolverine/Runtime/WorkerQueues/DurableReceiver.cs @@ -38,9 +38,8 @@ public DurableReceiver(Endpoint endpoint, IWolverineRuntime runtime, IHandlerPip _endpoint = endpoint; _settings = runtime.DurabilitySettings; - // HERE, HERE, HERE - - _inbox = runtime.Storage.Inbox; + // the check for Stores being null is honestly just because of some tests that use a little too much mocking + _inbox = runtime .Stores != null && runtime.Stores.HasAnyAncillaryStores() ? new DelegatingMessageInbox(runtime.Storage.Inbox, runtime.Stores) : runtime.Storage.Inbox; _logger = runtime.LoggerFactory.CreateLogger(); Uri = endpoint.Uri; From 928253951d666e07587e1d6c515da53c534c5d76 Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Mon, 22 Sep 2025 11:50:45 -0500 Subject: [PATCH 24/25] Detached from JasperFx code, imported PascalToKebabCase() again --- .../Wolverine.Marten/Wolverine.Marten.csproj | 4 +- .../Wolverine.Postgresql.csproj | 5 +- .../Wolverine.RDBMS/Wolverine.RDBMS.csproj | 5 +- .../Wolverine.SqlServer.csproj | 2 +- src/Wolverine/Util/WolverineMessageNaming.cs | 7 +- src/Wolverine/Wolverine.csproj | 7 +- wolverine.sln | 107 ------------------ 7 files changed, 20 insertions(+), 117 deletions(-) diff --git a/src/Persistence/Wolverine.Marten/Wolverine.Marten.csproj b/src/Persistence/Wolverine.Marten/Wolverine.Marten.csproj index 4a2dc7865..fde96659f 100644 --- a/src/Persistence/Wolverine.Marten/Wolverine.Marten.csproj +++ b/src/Persistence/Wolverine.Marten/Wolverine.Marten.csproj @@ -9,8 +9,10 @@ false - + + + diff --git a/src/Persistence/Wolverine.Postgresql/Wolverine.Postgresql.csproj b/src/Persistence/Wolverine.Postgresql/Wolverine.Postgresql.csproj index 3f2db6415..297045e70 100644 --- a/src/Persistence/Wolverine.Postgresql/Wolverine.Postgresql.csproj +++ b/src/Persistence/Wolverine.Postgresql/Wolverine.Postgresql.csproj @@ -12,7 +12,10 @@ - + + + + diff --git a/src/Persistence/Wolverine.RDBMS/Wolverine.RDBMS.csproj b/src/Persistence/Wolverine.RDBMS/Wolverine.RDBMS.csproj index 51225786b..573403b78 100644 --- a/src/Persistence/Wolverine.RDBMS/Wolverine.RDBMS.csproj +++ b/src/Persistence/Wolverine.RDBMS/Wolverine.RDBMS.csproj @@ -11,7 +11,10 @@ - + + + + diff --git a/src/Persistence/Wolverine.SqlServer/Wolverine.SqlServer.csproj b/src/Persistence/Wolverine.SqlServer/Wolverine.SqlServer.csproj index 692ccf8fc..554a7aed7 100644 --- a/src/Persistence/Wolverine.SqlServer/Wolverine.SqlServer.csproj +++ b/src/Persistence/Wolverine.SqlServer/Wolverine.SqlServer.csproj @@ -13,7 +13,7 @@ - + diff --git a/src/Wolverine/Util/WolverineMessageNaming.cs b/src/Wolverine/Util/WolverineMessageNaming.cs index 603c0a589..2b26d1532 100644 --- a/src/Wolverine/Util/WolverineMessageNaming.cs +++ b/src/Wolverine/Util/WolverineMessageNaming.cs @@ -18,13 +18,18 @@ public bool TryDetermineName(Type messageType, out string messageTypeName) { if (messageType.CanBeCastTo()) { - messageTypeName = messageType.NameInCode().PascalToKebabCase(); + messageTypeName = PascalToKebabCase(messageType.NameInCode()); return true; } messageTypeName = default!; return false; } + + public static string PascalToKebabCase(string value) + { + return value.SplitPascalCase().Replace(' ', '_').ToLowerInvariant(); + } } internal class InteropAttributeForwardingNaming : IMessageTypeNaming diff --git a/src/Wolverine/Wolverine.csproj b/src/Wolverine/Wolverine.csproj index 3359d2429..062f29b2b 100644 --- a/src/Wolverine/Wolverine.csproj +++ b/src/Wolverine/Wolverine.csproj @@ -4,6 +4,8 @@ WolverineFx + + @@ -34,10 +36,5 @@ - - - - - diff --git a/wolverine.sln b/wolverine.sln index 3c7f3e6bc..c10cbb1dd 100644 --- a/wolverine.sln +++ b/wolverine.sln @@ -289,22 +289,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wolverine.SignalR", "src\Tr EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wolverine.SignalR.Tests", "src\Transports\SignalR\Wolverine.SignalR.Tests\Wolverine.SignalR.Tests.csproj", "{3F62DB30-9A29-487A-9EE2-22A097E3EE3F}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Attached", "Attached", "{3B05C001-84E3-1311-E8D3-16CD1127E3D5}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JasperFx", "..\jasperfx\src\JasperFx\JasperFx.csproj", "{AFD6655F-131F-4DDD-B148-3D4276F028D4}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JasperFx.Events", "..\jasperfx\src\JasperFx.Events\JasperFx.Events.csproj", "{891F4009-E643-4083-B33E-8AB582446A42}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EventTests", "..\jasperfx\src\EventTests\EventTests.csproj", "{ED2CCB28-1469-4216-8D16-FAB64A896A28}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JasperFx.RuntimeCompiler", "..\jasperfx\src\JasperFx.RuntimeCompiler\JasperFx.RuntimeCompiler.csproj", "{EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Weasel.Core", "..\weasel\src\Weasel.Core\Weasel.Core.csproj", "{357619FF-D2E6-4877-83FF-924C88523A53}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Weasel.Postgresql", "..\weasel\src\Weasel.Postgresql\Weasel.Postgresql.csproj", "{0445097D-4DC5-47F3-8D1F-DF80CD65A62B}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Weasel.SqlServer", "..\weasel\src\Weasel.SqlServer\Weasel.SqlServer.csproj", "{CE6643E5-E869-41DF-8058-17FA45116652}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1635,90 +1619,6 @@ Global {3F62DB30-9A29-487A-9EE2-22A097E3EE3F}.Release|x64.Build.0 = Release|Any CPU {3F62DB30-9A29-487A-9EE2-22A097E3EE3F}.Release|x86.ActiveCfg = Release|Any CPU {3F62DB30-9A29-487A-9EE2-22A097E3EE3F}.Release|x86.Build.0 = Release|Any CPU - {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Debug|x64.ActiveCfg = Debug|Any CPU - {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Debug|x64.Build.0 = Debug|Any CPU - {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Debug|x86.ActiveCfg = Debug|Any CPU - {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Debug|x86.Build.0 = Debug|Any CPU - {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Release|Any CPU.Build.0 = Release|Any CPU - {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Release|x64.ActiveCfg = Release|Any CPU - {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Release|x64.Build.0 = Release|Any CPU - {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Release|x86.ActiveCfg = Release|Any CPU - {AFD6655F-131F-4DDD-B148-3D4276F028D4}.Release|x86.Build.0 = Release|Any CPU - {891F4009-E643-4083-B33E-8AB582446A42}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {891F4009-E643-4083-B33E-8AB582446A42}.Debug|Any CPU.Build.0 = Debug|Any CPU - {891F4009-E643-4083-B33E-8AB582446A42}.Debug|x64.ActiveCfg = Debug|Any CPU - {891F4009-E643-4083-B33E-8AB582446A42}.Debug|x64.Build.0 = Debug|Any CPU - {891F4009-E643-4083-B33E-8AB582446A42}.Debug|x86.ActiveCfg = Debug|Any CPU - {891F4009-E643-4083-B33E-8AB582446A42}.Debug|x86.Build.0 = Debug|Any CPU - {891F4009-E643-4083-B33E-8AB582446A42}.Release|Any CPU.ActiveCfg = Release|Any CPU - {891F4009-E643-4083-B33E-8AB582446A42}.Release|Any CPU.Build.0 = Release|Any CPU - {891F4009-E643-4083-B33E-8AB582446A42}.Release|x64.ActiveCfg = Release|Any CPU - {891F4009-E643-4083-B33E-8AB582446A42}.Release|x64.Build.0 = Release|Any CPU - {891F4009-E643-4083-B33E-8AB582446A42}.Release|x86.ActiveCfg = Release|Any CPU - {891F4009-E643-4083-B33E-8AB582446A42}.Release|x86.Build.0 = Release|Any CPU - {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Debug|Any CPU.Build.0 = Debug|Any CPU - {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Debug|x64.ActiveCfg = Debug|Any CPU - {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Debug|x64.Build.0 = Debug|Any CPU - {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Debug|x86.ActiveCfg = Debug|Any CPU - {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Debug|x86.Build.0 = Debug|Any CPU - {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Release|Any CPU.ActiveCfg = Release|Any CPU - {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Release|Any CPU.Build.0 = Release|Any CPU - {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Release|x64.ActiveCfg = Release|Any CPU - {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Release|x64.Build.0 = Release|Any CPU - {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Release|x86.ActiveCfg = Release|Any CPU - {ED2CCB28-1469-4216-8D16-FAB64A896A28}.Release|x86.Build.0 = Release|Any CPU - {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Debug|Any CPU.Build.0 = Debug|Any CPU - {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Debug|x64.ActiveCfg = Debug|Any CPU - {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Debug|x64.Build.0 = Debug|Any CPU - {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Debug|x86.ActiveCfg = Debug|Any CPU - {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Debug|x86.Build.0 = Debug|Any CPU - {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Release|Any CPU.ActiveCfg = Release|Any CPU - {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Release|Any CPU.Build.0 = Release|Any CPU - {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Release|x64.ActiveCfg = Release|Any CPU - {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Release|x64.Build.0 = Release|Any CPU - {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Release|x86.ActiveCfg = Release|Any CPU - {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8}.Release|x86.Build.0 = Release|Any CPU - {357619FF-D2E6-4877-83FF-924C88523A53}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {357619FF-D2E6-4877-83FF-924C88523A53}.Debug|Any CPU.Build.0 = Debug|Any CPU - {357619FF-D2E6-4877-83FF-924C88523A53}.Debug|x64.ActiveCfg = Debug|Any CPU - {357619FF-D2E6-4877-83FF-924C88523A53}.Debug|x64.Build.0 = Debug|Any CPU - {357619FF-D2E6-4877-83FF-924C88523A53}.Debug|x86.ActiveCfg = Debug|Any CPU - {357619FF-D2E6-4877-83FF-924C88523A53}.Debug|x86.Build.0 = Debug|Any CPU - {357619FF-D2E6-4877-83FF-924C88523A53}.Release|Any CPU.ActiveCfg = Release|Any CPU - {357619FF-D2E6-4877-83FF-924C88523A53}.Release|Any CPU.Build.0 = Release|Any CPU - {357619FF-D2E6-4877-83FF-924C88523A53}.Release|x64.ActiveCfg = Release|Any CPU - {357619FF-D2E6-4877-83FF-924C88523A53}.Release|x64.Build.0 = Release|Any CPU - {357619FF-D2E6-4877-83FF-924C88523A53}.Release|x86.ActiveCfg = Release|Any CPU - {357619FF-D2E6-4877-83FF-924C88523A53}.Release|x86.Build.0 = Release|Any CPU - {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Debug|x64.ActiveCfg = Debug|Any CPU - {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Debug|x64.Build.0 = Debug|Any CPU - {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Debug|x86.ActiveCfg = Debug|Any CPU - {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Debug|x86.Build.0 = Debug|Any CPU - {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Release|Any CPU.ActiveCfg = Release|Any CPU - {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Release|Any CPU.Build.0 = Release|Any CPU - {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Release|x64.ActiveCfg = Release|Any CPU - {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Release|x64.Build.0 = Release|Any CPU - {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Release|x86.ActiveCfg = Release|Any CPU - {0445097D-4DC5-47F3-8D1F-DF80CD65A62B}.Release|x86.Build.0 = Release|Any CPU - {CE6643E5-E869-41DF-8058-17FA45116652}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {CE6643E5-E869-41DF-8058-17FA45116652}.Debug|Any CPU.Build.0 = Debug|Any CPU - {CE6643E5-E869-41DF-8058-17FA45116652}.Debug|x64.ActiveCfg = Debug|Any CPU - {CE6643E5-E869-41DF-8058-17FA45116652}.Debug|x64.Build.0 = Debug|Any CPU - {CE6643E5-E869-41DF-8058-17FA45116652}.Debug|x86.ActiveCfg = Debug|Any CPU - {CE6643E5-E869-41DF-8058-17FA45116652}.Debug|x86.Build.0 = Debug|Any CPU - {CE6643E5-E869-41DF-8058-17FA45116652}.Release|Any CPU.ActiveCfg = Release|Any CPU - {CE6643E5-E869-41DF-8058-17FA45116652}.Release|Any CPU.Build.0 = Release|Any CPU - {CE6643E5-E869-41DF-8058-17FA45116652}.Release|x64.ActiveCfg = Release|Any CPU - {CE6643E5-E869-41DF-8058-17FA45116652}.Release|x64.Build.0 = Release|Any CPU - {CE6643E5-E869-41DF-8058-17FA45116652}.Release|x86.ActiveCfg = Release|Any CPU - {CE6643E5-E869-41DF-8058-17FA45116652}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1852,12 +1752,5 @@ Global {1EF7D49F-DDB8-469A-88A0-4A8D6237561C} = {84D32C8B-9CCE-4925-9AEC-8F445C7A2E3D} {36645C4B-BE1F-4184-A14C-D5BFA3F86A86} = {1EF7D49F-DDB8-469A-88A0-4A8D6237561C} {3F62DB30-9A29-487A-9EE2-22A097E3EE3F} = {1EF7D49F-DDB8-469A-88A0-4A8D6237561C} - {AFD6655F-131F-4DDD-B148-3D4276F028D4} = {3B05C001-84E3-1311-E8D3-16CD1127E3D5} - {891F4009-E643-4083-B33E-8AB582446A42} = {3B05C001-84E3-1311-E8D3-16CD1127E3D5} - {ED2CCB28-1469-4216-8D16-FAB64A896A28} = {3B05C001-84E3-1311-E8D3-16CD1127E3D5} - {EFF06F8A-31F0-496A-83D2-B5F8C5ECF0D8} = {3B05C001-84E3-1311-E8D3-16CD1127E3D5} - {357619FF-D2E6-4877-83FF-924C88523A53} = {3B05C001-84E3-1311-E8D3-16CD1127E3D5} - {0445097D-4DC5-47F3-8D1F-DF80CD65A62B} = {3B05C001-84E3-1311-E8D3-16CD1127E3D5} - {CE6643E5-E869-41DF-8058-17FA45116652} = {3B05C001-84E3-1311-E8D3-16CD1127E3D5} EndGlobalSection EndGlobal From 24f41adef45bb48c8994079f1e666d7879e73bfe Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Mon, 22 Sep 2025 11:59:07 -0500 Subject: [PATCH 25/25] Tweak for unit tests that use too much mocking:( --- src/Testing/CoreTests/Runtime/MockWolverineRuntime.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Testing/CoreTests/Runtime/MockWolverineRuntime.cs b/src/Testing/CoreTests/Runtime/MockWolverineRuntime.cs index 2b82fe8ac..c1457532c 100644 --- a/src/Testing/CoreTests/Runtime/MockWolverineRuntime.cs +++ b/src/Testing/CoreTests/Runtime/MockWolverineRuntime.cs @@ -99,7 +99,7 @@ public IMessageRouter RoutingFor(Type messageType) public ILogger Logger { get; } = Substitute.For(); - public MessageStoreCollection Stores => throw new NotSupportedException(); + public MessageStoreCollection Stores => new MessageStoreCollection(this, [], []); public Task TryFindMainMessageStore() where T : class { @@ -113,7 +113,7 @@ public IMessageStore FindAncillaryStoreForMarkerType(Type markerType) public void ScheduleLocalExecutionInMemory(DateTimeOffset executionTime, Envelope envelope) { - throw new NotSupportedException(); + } public void RegisterMessageType(Type messageType)