Skip to content

Commit 3a97129

Browse files
committed
Discord connector + test app
1 parent dcd2a0e commit 3a97129

13 files changed

Lines changed: 665 additions & 0 deletions

Directory.Packages.props

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,14 @@
33
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
44
</PropertyGroup>
55
<ItemGroup>
6+
<PackageVersion Include="Aspire.Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.0-preview.7.24251.11" />
67
<PackageVersion Include="Azure.AI.FormRecognizer" Version="4.1.0" />
78
<PackageVersion Include="Azure.Identity" Version="1.11.2" />
89
<PackageVersion Include="Azure.Search.Documents" Version="11.5.1" />
910
<PackageVersion Include="Azure.Storage.Blobs" Version="12.19.1" />
1011
<PackageVersion Include="Azure.Storage.Queues" Version="12.17.1" />
1112
<PackageVersion Include="Elastic.Clients.Elasticsearch" Version="8.11.0" />
13+
<PackageVersion Include="Discord.Net" Version="3.14.1" />
1214
<PackageVersion Include="HtmlAgilityPack" Version="1.11.61" />
1315
<PackageVersion Include="LLamaSharp" Version="0.11.2" />
1416
<PackageVersion Include="LLamaSharp.Backend.Cpu" Version="0.11.2" />
@@ -21,6 +23,7 @@
2123
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
2224
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1" />
2325
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
26+
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
2427
<PackageVersion Include="Microsoft.Extensions.Http" Version="8.0.0" />
2528
<PackageVersion Include="Microsoft.Extensions.Logging" Version="8.0.0" />
2629
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />

KernelMemory.sln

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elasticsearch", "extensions
274274
EndProject
275275
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elasticsearch.UnitTests", "extensions\Elasticsearch\Elasticsearch.FunctionalTests\Elasticsearch.FunctionalTests.csproj", "{C5E6B28C-F54D-423D-954D-A9EAEFB89732}"
276276
EndProject
277+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Discord", "extensions\Discord\Discord\Discord.csproj", "{43877864-6AE8-4B03-BEDA-6B6FA8BB1D8B}"
278+
EndProject
279+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "301-discord-test-application", "examples\301-discord-test-application\301-discord-test-application.csproj", "{FAE4C6B8-38B2-43E7-8881-99693C9CEDC6}"
280+
EndProject
277281
Global
278282
GlobalSection(SolutionConfigurationPlatforms) = preSolution
279283
Debug|Any CPU = Debug|Any CPU
@@ -500,6 +504,13 @@ Global
500504
{C5E6B28C-F54D-423D-954D-A9EAEFB89732}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
501505
{C5E6B28C-F54D-423D-954D-A9EAEFB89732}.Debug|Any CPU.Build.0 = Debug|Any CPU
502506
{C5E6B28C-F54D-423D-954D-A9EAEFB89732}.Release|Any CPU.ActiveCfg = Release|Any CPU
507+
{43877864-6AE8-4B03-BEDA-6B6FA8BB1D8B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
508+
{43877864-6AE8-4B03-BEDA-6B6FA8BB1D8B}.Debug|Any CPU.Build.0 = Debug|Any CPU
509+
{43877864-6AE8-4B03-BEDA-6B6FA8BB1D8B}.Release|Any CPU.ActiveCfg = Release|Any CPU
510+
{43877864-6AE8-4B03-BEDA-6B6FA8BB1D8B}.Release|Any CPU.Build.0 = Release|Any CPU
511+
{FAE4C6B8-38B2-43E7-8881-99693C9CEDC6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
512+
{FAE4C6B8-38B2-43E7-8881-99693C9CEDC6}.Debug|Any CPU.Build.0 = Debug|Any CPU
513+
{FAE4C6B8-38B2-43E7-8881-99693C9CEDC6}.Release|Any CPU.ActiveCfg = Release|Any CPU
503514
EndGlobalSection
504515
GlobalSection(SolutionProperties) = preSolution
505516
HideSolutionNode = FALSE
@@ -580,6 +591,8 @@ Global
580591
{B9BE1099-F78F-4A5F-A897-BF2C75E19C57} = {155DA079-E267-49AF-973A-D1D44681970F}
581592
{2E10420F-BF96-411C-8FE0-F6268F2EEB67} = {155DA079-E267-49AF-973A-D1D44681970F}
582593
{C5E6B28C-F54D-423D-954D-A9EAEFB89732} = {3C17F42B-CFC8-4900-8CFB-88936311E919}
594+
{43877864-6AE8-4B03-BEDA-6B6FA8BB1D8B} = {155DA079-E267-49AF-973A-D1D44681970F}
595+
{FAE4C6B8-38B2-43E7-8881-99693C9CEDC6} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
583596
EndGlobalSection
584597
GlobalSection(ExtensibilityGlobals) = postSolution
585598
SolutionGuid = {CC136C62-115C-41D1-B414-F9473EFF6EA8}
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<Project Sdk="Microsoft.NET.Sdk.Web">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net8.0</TargetFramework>
6+
<NoWarn>$(NoWarn);CA1303;CA1031;</NoWarn>
7+
</PropertyGroup>
8+
9+
<ItemGroup>
10+
<ProjectReference Include="..\..\extensions\Discord\Discord\Discord.csproj" />
11+
<ProjectReference Include="..\..\service\Service.AspNetCore\Service.AspNetCore.csproj" />
12+
</ItemGroup>
13+
14+
<ItemGroup>
15+
<PackageReference Include="Aspire.Npgsql.EntityFrameworkCore.PostgreSQL" />
16+
</ItemGroup>
17+
18+
</Project>
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using Microsoft.EntityFrameworkCore;
4+
5+
namespace Microsoft.Discord.TestApplication;
6+
7+
public class DiscordDbContext : DbContext
8+
{
9+
public DbContextOptions<DiscordDbContext> Options { get; }
10+
11+
public DbSet<DiscordDbMessage> Messages { get; set; }
12+
13+
public DiscordDbContext(DbContextOptions<DiscordDbContext> options) : base(options)
14+
{
15+
this.Options = options;
16+
}
17+
}
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System.ComponentModel.DataAnnotations;
4+
using Microsoft.KernelMemory.Sources.DiscordBot;
5+
6+
namespace Microsoft.Discord.TestApplication;
7+
8+
public class DiscordDbMessage : DiscordMessage
9+
{
10+
[Key]
11+
public string Id
12+
{
13+
get
14+
{
15+
return this.MessageId;
16+
}
17+
set
18+
{
19+
this.MessageId = value;
20+
}
21+
}
22+
}
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System.Text.Json;
4+
using Microsoft.KernelMemory;
5+
using Microsoft.KernelMemory.Diagnostics;
6+
using Microsoft.KernelMemory.Pipeline;
7+
using Microsoft.KernelMemory.Sources.DiscordBot;
8+
9+
namespace Microsoft.Discord.TestApplication;
10+
11+
/// <summary>
12+
/// KM pipeline handler fetching discord data files from content storage
13+
/// and storing messages in Postgres.
14+
/// </summary>
15+
public sealed class DiscordMessageHandler : IPipelineStepHandler, IDisposable, IAsyncDisposable
16+
{
17+
// Name of the file where to store Discord data
18+
private readonly string _filename;
19+
20+
// KM pipelines orchestrator
21+
private readonly IPipelineOrchestrator _orchestrator;
22+
23+
// .NET service provider, used to get thread safe instances of EF DbContext
24+
private readonly IServiceProvider _serviceProvider;
25+
26+
// EF DbContext used to create the database
27+
private DiscordDbContext? _firstInvokeDb;
28+
29+
// .NET logger
30+
private readonly ILogger<DiscordMessageHandler> _log;
31+
32+
public string StepName { get; } = string.Empty;
33+
34+
public DiscordMessageHandler(
35+
string stepName,
36+
IPipelineOrchestrator orchestrator,
37+
DiscordConnectorConfig config,
38+
IServiceProvider serviceProvider,
39+
ILoggerFactory? loggerFactory = null)
40+
{
41+
this.StepName = stepName;
42+
this._log = loggerFactory?.CreateLogger<DiscordMessageHandler>() ?? DefaultLogger<DiscordMessageHandler>.Instance;
43+
44+
this._orchestrator = orchestrator;
45+
this._serviceProvider = serviceProvider;
46+
this._filename = config.FileName;
47+
48+
// This DbContext instance is used only to create the database
49+
this._firstInvokeDb = serviceProvider.GetService<DiscordDbContext>() ?? throw new ConfigurationException("Discord DB Content is not defined");
50+
}
51+
52+
public async Task<(bool success, DataPipeline updatedPipeline)> InvokeAsync(DataPipeline pipeline, CancellationToken cancellationToken = default)
53+
{
54+
this.OnFirstInvoke();
55+
56+
// Note: use a new DbContext instance each time, because DbContext is not thread safe and would throw the following
57+
// exception: System.InvalidOperationException: a second operation was started on this context instance before a previous
58+
// operation completed. This is usually caused by different threads concurrently using the same instance of DbContext.
59+
// For more information on how to avoid threading issues with DbContext, see https://go.microsoft.com/fwlink/?linkid=2097913.
60+
await using (var db = (this._serviceProvider.GetService<DiscordDbContext>())!)
61+
{
62+
foreach (DataPipeline.FileDetails uploadedFile in pipeline.Files)
63+
{
64+
// Process only the file containing the discord data
65+
if (uploadedFile.Name != this._filename) { continue; }
66+
67+
string fileContent = await this._orchestrator.ReadTextFileAsync(pipeline, uploadedFile.Name, cancellationToken).ConfigureAwait(false);
68+
69+
DiscordDbMessage? data;
70+
try
71+
{
72+
data = JsonSerializer.Deserialize<DiscordDbMessage>(fileContent);
73+
if (data == null)
74+
{
75+
this._log.LogError("Failed to deserialize Discord data file, result is NULL");
76+
return (true, pipeline);
77+
}
78+
}
79+
catch (Exception e)
80+
{
81+
this._log.LogError(e, "Failed to deserialize Discord data file");
82+
return (true, pipeline);
83+
}
84+
85+
await db.Messages.AddAsync(data, cancellationToken).ConfigureAwait(false);
86+
}
87+
88+
await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false);
89+
}
90+
91+
return (true, pipeline);
92+
}
93+
94+
public void Dispose()
95+
{
96+
this._firstInvokeDb?.Dispose();
97+
this._firstInvokeDb = null;
98+
}
99+
100+
public async ValueTask DisposeAsync()
101+
{
102+
if (this._firstInvokeDb != null) { await this._firstInvokeDb.DisposeAsync(); }
103+
104+
this._firstInvokeDb = null;
105+
}
106+
107+
private void OnFirstInvoke()
108+
{
109+
if (this._firstInvokeDb == null) { return; }
110+
111+
lock (this._firstInvokeDb)
112+
{
113+
// Create DB / Tables if needed
114+
this._firstInvokeDb.Database.EnsureCreated();
115+
this._firstInvokeDb.Dispose();
116+
this._firstInvokeDb = null;
117+
}
118+
}
119+
}
Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using Microsoft.KernelMemory;
4+
using Microsoft.KernelMemory.ContentStorage.DevTools;
5+
using Microsoft.KernelMemory.Sources.DiscordBot;
6+
7+
namespace Microsoft.Discord.TestApplication;
8+
9+
/* Example: Listen for new messages in Discord, and save them in a table in Postgres.
10+
*
11+
* Use ASP.NET hosted services to host a Discord Bot. The discord bot logic is based
12+
* on DiscordConnector class.
13+
*
14+
* While the Discord bot is running, every time there is a new message, DiscordConnector
15+
* invokes KM.ImportDocument API, uploading a JSON file that contains details about the
16+
* Discord message, including server ID, channel ID, author ID, message content, etc.
17+
*
18+
* The call to KM.ImportDocument API asks to process the JSON file uploaded using
19+
* DiscordMessageHandler, included in this project. No other handlers are used.
20+
*
21+
* DiscordMessageHandler, loads the uploaded file, deserializes its content, and
22+
* save each Discord message into a table in Postgres, using Entity Framework.
23+
*/
24+
25+
internal static class Program
26+
{
27+
public static void Main(string[] args)
28+
{
29+
WebApplicationBuilder appBuilder = WebApplication.CreateBuilder();
30+
31+
appBuilder.Configuration
32+
.AddJsonFile("appsettings.json")
33+
.AddJsonFile("appsettings.Development.json", optional: true)
34+
.AddEnvironmentVariables()
35+
.AddCommandLine(args);
36+
37+
// Discord setup
38+
// Use DiscordConnector to connect to Discord and listen for messages.
39+
// The Discord connection can listen from multiple servers and channels.
40+
// For each message, DiscordConnector will send a file to Kernel Memory to process.
41+
// Files sent to Kernel Memory are processed by DiscordMessageHandler (in this project)
42+
var discordCfg = appBuilder.Configuration.GetSection("Discord").Get<DiscordConnectorConfig>();
43+
ArgumentNullExceptionEx.ThrowIfNull(discordCfg, nameof(discordCfg), "Discord config is NULL");
44+
appBuilder.Services.AddSingleton<DiscordConnectorConfig>(discordCfg);
45+
appBuilder.Services.AddHostedService<DiscordConnector>();
46+
47+
// Postgres with Entity Framework
48+
// DiscordMessageHandler reads files received by Kernel Memory and store each message in a table in Postgres.
49+
// See DiscordDbMessage for the table schema.
50+
appBuilder.AddNpgsqlDbContext<DiscordDbContext>("postgresDb");
51+
52+
// Run Kernel Memory and DiscordMessageHandler
53+
// var kmApp = BuildAsynchronousKernelMemoryApp(appBuilder, discordConfig);
54+
var kmApp = BuildSynchronousKernelMemoryApp(appBuilder, discordCfg);
55+
56+
Console.WriteLine("Starting KM application...\n");
57+
kmApp.Run();
58+
Console.WriteLine("\n... KM application stopped.");
59+
}
60+
61+
private static WebApplication BuildSynchronousKernelMemoryApp(WebApplicationBuilder appBuilder, DiscordConnectorConfig discordConfig)
62+
{
63+
appBuilder.AddKernelMemory(kmb =>
64+
{
65+
// Note: there's no queue system, so the memory instance will be synchronous (ie MemoryServerless)
66+
67+
// Store files on disk
68+
kmb.WithSimpleFileStorage(SimpleFileStorageConfig.Persistent);
69+
70+
// Disable AI, not needed for this example
71+
kmb.WithoutEmbeddingGenerator();
72+
kmb.WithoutTextGenerator();
73+
});
74+
75+
WebApplication app = appBuilder.Build();
76+
77+
// In synchronous apps, handlers are added to the serverless memory orchestrator
78+
(app.Services.GetService<IKernelMemory>() as MemoryServerless)!
79+
.Orchestrator
80+
.AddHandler<DiscordMessageHandler>(discordConfig.Steps[0]);
81+
82+
return app;
83+
}
84+
85+
private static WebApplication BuildAsynchronousKernelMemoryApp(WebApplicationBuilder appBuilder, DiscordConnectorConfig discordConfig)
86+
{
87+
appBuilder.Services.AddHandlerAsHostedService<DiscordMessageHandler>(discordConfig.Steps[0]);
88+
appBuilder.AddKernelMemory(kmb =>
89+
{
90+
// Note: because of this the memory instance will be asynchronous (ie MemoryService)
91+
kmb.WithSimpleQueuesPipeline();
92+
93+
// Store files on disk
94+
kmb.WithSimpleFileStorage(SimpleFileStorageConfig.Persistent);
95+
96+
// Disable AI, not needed for this example
97+
kmb.WithoutEmbeddingGenerator();
98+
kmb.WithoutTextGenerator();
99+
});
100+
101+
return appBuilder.Build();
102+
}
103+
}
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
{
2+
"Discord": {
3+
// Discord bot authentication token
4+
// See https://discord.com/developers
5+
"DiscordToken": "",
6+
// Index where to store files, e.g. disk folder, Azure blobs folder, etc.
7+
"Index": "discord",
8+
// File name used when uploading a message to content storage.
9+
"FileName": "discord-msg.json",
10+
// Handlers processing the incoming Discord events
11+
"Steps": [
12+
"store_discord_message"
13+
]
14+
},
15+
"ConnectionStrings": {
16+
// Db where Discord messages are stored, e.g.
17+
// "Host=contoso.postgres.database.azure.com;Port=5432;Username=adminuser;Password=mypassword;Database=discorddata;SSL Mode=VerifyFull"
18+
"postgresDb": "Host=localhost;Port=5432;Username=;Password="
19+
},
20+
"Logging": {
21+
"LogLevel": {
22+
"Default": "Warning"
23+
},
24+
"Console": {
25+
"LogToStandardErrorThreshold": "Critical",
26+
"FormatterName": "simple",
27+
"FormatterOptions": {
28+
"TimestampFormat": "[HH:mm:ss.fff] ",
29+
"SingleLine": true,
30+
"UseUtcTimestamp": false,
31+
"IncludeScopes": false,
32+
"JsonWriterOptions": {
33+
"Indented": true
34+
}
35+
}
36+
}
37+
}
38+
}

0 commit comments

Comments
 (0)