Add Elasticsearch connector

Directory.Packages.props

@@ -8,6 +8,7 @@
     <PackageVersion Include="Azure.Search.Documents" Version="11.5.1" />
     <PackageVersion Include="Azure.Storage.Blobs" Version="12.19.1" />
     <PackageVersion Include="Azure.Storage.Queues" Version="12.17.1" />
+    <PackageVersion Include="Elastic.Clients.Elasticsearch" Version="8.11.0" />
     <PackageVersion Include="HtmlAgilityPack" Version="1.11.61" />
     <PackageVersion Include="LLamaSharp" Version="0.11.2" />
     <PackageVersion Include="LLamaSharp.Backend.Cpu" Version="0.11.2" />

extensions/Elasticsearch/Elasticsearch.FunctionalTests/DefaultTests.cs

@@ -1,31 +1,29 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-#if FALSE
-using FreeMindLabs.KernelMemory.Elasticsearch;
-using Microsoft.Core.FunctionalTests.DefaultTestCases;
 using Microsoft.KernelMemory;
-using Microsoft.TestHelpers;
+using Microsoft.KM.Core.FunctionalTests.DefaultTestCases;
+using Microsoft.KM.TestHelpers;
 
 namespace Microsoft.Elasticsearch.FunctionalTests;
 
 public class DefaultTests : BaseFunctionalTestCase
 {
     private readonly MemoryServerless _memory;
-    private readonly ElasticsearchConfig _elasticsearchConfig;
+    private readonly ElasticsearchConfig _esConfig;
 
     public DefaultTests(IConfiguration cfg, ITestOutputHelper output) : base(cfg, output)
     {
         Assert.False(string.IsNullOrEmpty(this.OpenAiConfig.APIKey));
 
-        this._elasticsearchConfig = cfg.GetSection("KernelMemory:Services:Elasticsearch").Get<ElasticsearchConfig>()!;
+        this._esConfig = cfg.GetSection("KernelMemory:Services:Elasticsearch").Get<ElasticsearchConfig>()!;
 
         this._memory = new KernelMemoryBuilder()
             .With(new KernelMemoryConfig { DefaultIndexName = "default4tests" })
             .WithSearchClientConfig(new SearchClientConfig { EmptyAnswer = NotFound })
             .WithOpenAI(this.OpenAiConfig)
             // .WithAzureOpenAITextGeneration(this.AzureOpenAITextConfiguration)
             // .WithAzureOpenAITextEmbeddingGeneration(this.AzureOpenAIEmbeddingConfiguration)
-            .WithElasticsearch(this._elasticsearchConfig)
+            .WithElasticsearch(this._esConfig)
             .Build<MemoryServerless>();
     }
 
@@ -106,4 +104,3 @@ public async Task ItSupportsTags()
         await DocumentUploadTest.ItSupportsTags(this._memory, this.Log);
     }
 }
-#endif

extensions/Elasticsearch/Elasticsearch.FunctionalTests/Elasticsearch.FunctionalTests.csproj

@@ -9,7 +9,7 @@
         <Nullable>enable</Nullable>
         <IsPackable>false</IsPackable>
         <GenerateDocumentationFile>false</GenerateDocumentationFile>
-        <NoWarn>$(NoWarn);CA1859;</NoWarn>
+        <NoWarn>$(NoWarn);</NoWarn>
     </PropertyGroup>
 
     <ItemGroup>

extensions/Elasticsearch/Elasticsearch/CREDITS.txt

@@ -0,0 +1,6 @@
+Special credits to Free Mind Labs, Inc. and Alessandro Federici
+for their work on the Elasticsearch Memory connector.
+
+The Elasticsearch was originally developed at
+https://github.com/freemindlabsinc/FreeMindLabs.KernelMemory.Elasticsearch
+and kindly submitted to Kernel Memory repository.

extensions/Elasticsearch/Elasticsearch/ConfigurationException.cs

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace Microsoft.KernelMemory.MemoryDb.Elasticsearch;
+
+/// <summary>
+/// Exception thrown when the Elasticsearch configuration is invalid in appSettings, secrets, etc.
+/// </summary>
+public class ConfigurationException : ElasticsearchException
+{
+    /// <inheritdoc />
+    public ConfigurationException() { }
+
+    /// <inheritdoc />
+    public ConfigurationException(string message) : base(message) { }
+
+    /// <inheritdoc />
+    public ConfigurationException(string message, Exception? innerException) : base(message, innerException) { }
+}

extensions/Elasticsearch/Elasticsearch/DependencyInjection.cs

@@ -0,0 +1,94 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Elastic.Clients.Elasticsearch;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.KernelMemory.MemoryDb.Elasticsearch;
+using Microsoft.KernelMemory.MemoryDb.Elasticsearch.Internals;
+using Microsoft.KernelMemory.MemoryStorage;
+
+#pragma warning disable IDE0130 // reduce number of "using" statements
+// ReSharper disable once CheckNamespace - reduce number of "using" statements
+namespace Microsoft.KernelMemory;
+
+/// <summary>
+/// DI pipelines for Elasticsearch Memory.
+/// </summary>
+public static partial class KernelMemoryBuilderExtensions
+{
+    /// <summary>
+    /// Kernel Memory Builder extension method to add the Elasticsearch memory connector.
+    /// </summary>
+    /// <param name="builder">KM builder instance</param>
+    /// <param name="configuration">Elasticsearch configuration</param>"
+    public static IKernelMemoryBuilder WithElasticsearch(
+        this IKernelMemoryBuilder builder,
+        ElasticsearchConfig configuration)
+    {
+        builder.Services.AddElasticsearchAsMemoryDb(configuration);
+
+        return builder;
+    }
+
+    /// <summary>
+    /// Extension method to add the Elasticsearch memory connector.
+    /// </summary>
+    /// <param name="builder">KM builder instance</param>
+    /// <param name="configure">Action to configure Elasticsearch</param>
+    public static IKernelMemoryBuilder WithElasticsearch(
+        this IKernelMemoryBuilder builder,
+        Action<ElasticsearchConfigBuilder> configure)
+    {
+        ArgumentNullExceptionEx.ThrowIfNull(configure, nameof(configure), "The configure action is NULL");
+
+        var cfg = new ElasticsearchConfigBuilder();
+        configure(cfg);
+
+        builder.Services.AddElasticsearchAsMemoryDb(cfg.Build());
+        return builder;
+    }
+}
+
+/// <summary>
+/// Setup Elasticsearch memory within the DI service collection.
+/// </summary>
+public static partial class DependencyInjection
+{
+    /// <summary>
+    /// Inject Elasticsearch as the default implementation of IMemoryDb
+    /// </summary>
+    public static IServiceCollection AddElasticsearchAsMemoryDb(
+        this IServiceCollection services,
+        ElasticsearchConfig config)
+    {
+        ArgumentNullExceptionEx.ThrowIfNull(config, nameof(config), "The configuration is NULL");
+
+        // The ElasticsearchClient type is thread-safe and can be shared and
+        // reused across multiple threads in consuming applications.
+        // See https://www.elastic.co/guide/en/elasticsearch/client/net-api/current/recommendations.html
+        services.AddSingleton(sp =>
+        {
+            var esConfig = sp.GetRequiredService<ElasticsearchConfig>();
+            return new ElasticsearchClient(esConfig.ToElasticsearchClientSettings());
+        });
+
+        return services
+            .AddSingleton<IIndexNameHelper, IndexNameHelper>()
+            .AddSingleton(config)
+            .AddSingleton<IMemoryDb, ElasticsearchMemory>();
+    }
+
+    /// <summary>
+    /// Inject Elasticsearch as the default implementation of IMemoryDb
+    /// </summary>
+    public static IServiceCollection AddElasticsearchAsMemoryDb(
+        this IServiceCollection services,
+        Action<ElasticsearchConfigBuilder> configure)
+    {
+        ArgumentNullExceptionEx.ThrowIfNull(configure, nameof(configure), "The configuration action is NULL");
+
+        var cfg = new ElasticsearchConfigBuilder();
+        configure(cfg);
+
+        return services.AddElasticsearchAsMemoryDb(cfg.Build());
+    }
+}

extensions/Elasticsearch/Elasticsearch/Elasticsearch.csproj

@@ -0,0 +1,37 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <RollForward>LatestMajor</RollForward>
+        <AssemblyName>Microsoft.KernelMemory.MemoryDb.Elasticsearch</AssemblyName>
+        <RootNamespace>Microsoft.KernelMemory.MemoryDb.Elasticsearch</RootNamespace>
+        <NoWarn>$(NoWarn);KMEXP00;KMEXP03;CS1591;CA1308;CA1724;CA1812;</NoWarn>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>        
+    </PropertyGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="..\..\..\service\Abstractions\Abstractions.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Elastic.Clients.Elasticsearch" />
+    </ItemGroup>
+
+    <Import Project="../../../infra/dotnet/code-analysis.props" />
+
+    <Import Project="../../../infra/dotnet/nuget-package.props" />
+
+    <PropertyGroup>
+        <IsPackable>true</IsPackable>
+        <PackageId>Microsoft.KernelMemory.MemoryDb.Elasticsearch</PackageId>
+        <Product>Elasticsearch connector for Kernel Memory</Product>
+        <Description>Elasticsearch connector for Microsoft Kernel Memory, to store and search memory using Elasticsearch vector search and other Elasticsearch features.</Description>
+        <PackageTags>Elasticsearch Memory, RAG, Kernel Memory, Elasticsearch, HNSW, AI, Artificial Intelligence, Embeddings, Vector DB, Vector Search, ETL</PackageTags>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <None Include="..\README.md" Link="README.md" Pack="true" PackagePath="." Visible="false" />
+    </ItemGroup>
+
+</Project>

extensions/Elasticsearch/Elasticsearch/ElasticsearchConfig.cs

@@ -0,0 +1,60 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Elastic.Clients.Elasticsearch.Mapping;
+using Microsoft.KernelMemory.MemoryDb.Elasticsearch;
+
+#pragma warning disable IDE0130 // reduce number of "using" statements
+// ReSharper disable once CheckNamespace - reduce number of "using" statements
+namespace Microsoft.KernelMemory;
+
+/// <summary>
+/// The configuration for the Elasticsearch connector.
+/// Use <see cref="ElasticsearchConfigBuilder"/> to instantiate and configure this class.
+/// </summary>
+public class ElasticsearchConfig
+{
+    public ElasticsearchConfig()
+    {
+    }
+
+    /// <summary>
+    /// The certificate fingerprint for the Elasticsearch instance.
+    /// See <see href="https://www.elastic.co/guide/en/elasticsearch/reference/current/configuring-stack-security.html#_use_the_ca_fingerprint_5"/>.
+    /// </summary>
+    public string CertificateFingerPrint { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The Elasticsearch endpoint.
+    /// </summary>
+    public string Endpoint { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The username used to connect to Elasticsearch.
+    /// </summary>
+    public string UserName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The password used to connect to Elasticsearch.
+    /// </summary>
+    public string Password { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The prefix to be prepend to the index names in Elasticsearch.
+    /// </summary>
+    public string IndexPrefix { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The number of shards to use for the Elasticsearch index.
+    /// </summary>
+    public int? ShardCount { get; set; } = 1;
+
+    /// <summary>
+    /// The number of replicas to use for the Elasticsearch index.
+    /// </summary>
+    public int? ReplicaCount { get; set; } = 0;
+
+    /// <summary>
+    /// A delegate to configure the Elasticsearch index properties.
+    /// </summary>
+    public Action<PropertiesDescriptor<ElasticsearchMemoryRecord>>? ConfigureProperties { get; internal set; }
+}