Add GPT3 tokenizer (workaround for broken SK tokenizer)

Author: dluc

.editorconfig

@@ -115,26 +115,23 @@ dotnet_diagnostic.RCS1214.severity = warning # Unnecessary interpolated string.
 dotnet_diagnostic.RCS1225.severity = warning # Make class sealed.
 dotnet_diagnostic.RCS1232.severity = warning # Order elements in documentation comment.
 
-# Commented out because `dotnet format` change can be disruptive.
-# dotnet_diagnostic.RCS1085.severity = warning # Use auto-implemented property.
-
-# Commented out because `dotnet format` removes the xmldoc element, while we should add the missing documentation instead.
-# dotnet_diagnostic.RCS1228.severity = warning # Unused element in documentation comment.
-
 # Diagnostics elevated as warnings
 dotnet_diagnostic.CA1000.severity = warning # Do not declare static members on generic types
 dotnet_diagnostic.CA1031.severity = warning # Do not catch general exception types
 dotnet_diagnostic.CA1050.severity = warning # Declare types in namespaces
 dotnet_diagnostic.CA1063.severity = warning # Implement IDisposable correctly
 dotnet_diagnostic.CA1064.severity = warning # Exceptions should be public
+dotnet_diagnostic.CA1303.severity = warning # Do not pass literals as localized parameters
 dotnet_diagnostic.CA1416.severity = warning # Validate platform compatibility
 dotnet_diagnostic.CA1508.severity = warning # Avoid dead conditional code
+dotnet_diagnostic.CS1591.severity = warning # Missing XML comment for publicly visible type or member
 dotnet_diagnostic.CA1852.severity = warning # Sealed classes
 dotnet_diagnostic.CA1859.severity = warning # Use concrete types when possible for improved performance
 dotnet_diagnostic.CA1860.severity = warning # Prefer comparing 'Count' to 0 rather than using 'Any()', both for clarity and for performance
 dotnet_diagnostic.CA2000.severity = warning # Call System.IDisposable.Dispose on object before all references to it are out of scope
 dotnet_diagnostic.CA2007.severity = error # Do not directly await a Task
 dotnet_diagnostic.CA2201.severity = warning # Exception type System.Exception is not sufficiently specific
+dotnet_diagnostic.CA2225.severity = warning # Operator overloads have named alternates
 
 dotnet_diagnostic.IDE0001.severity = warning # Simplify name
 dotnet_diagnostic.IDE0005.severity = warning # Remove unnecessary using directives
@@ -158,37 +155,40 @@ dotnet_diagnostic.IDE0090.severity = warning # Simplify new expression
 dotnet_diagnostic.IDE0130.severity = warning # Namespace does not match folder structure
 dotnet_diagnostic.IDE0161.severity = warning # Use file-scoped namespace
 
+dotnet_diagnostic.RCS1032.severity = warning # Remove redundant parentheses.
+dotnet_diagnostic.RCS1118.severity = warning # Mark local variable as const.
+dotnet_diagnostic.RCS1141.severity = warning # Add 'param' element to documentation comment.
+dotnet_diagnostic.RCS1197.severity = warning # Optimize StringBuilder.AppendLine call.
+dotnet_diagnostic.RCS1205.severity = warning # Order named arguments according to the order of parameters.
+dotnet_diagnostic.RCS1229.severity = warning # Use async/await when necessary.
+
 dotnet_diagnostic.VSTHRD111.severity = error # Use .ConfigureAwait(bool)
 
 # Suppressed diagnostics
+
+# Commented out because `dotnet format` change can be disruptive.
+# dotnet_diagnostic.RCS1085.severity = warning # Use auto-implemented property.
+
+# Commented out because `dotnet format` removes the xmldoc element, while we should add the missing documentation instead.
+# dotnet_diagnostic.RCS1228.severity = warning # Unused element in documentation comment.
+
 dotnet_diagnostic.CA1002.severity = none # Change 'List<string>' in '...' to use 'Collection<T>' ...
 dotnet_diagnostic.CA1032.severity = none # We're using RCS1194 which seems to cover more ctors
 dotnet_diagnostic.CA1034.severity = none # Do not nest type. Alternatively, change its accessibility so that it is not externally visible
 dotnet_diagnostic.CA1062.severity = none # Disable null check, C# already does it for us
-dotnet_diagnostic.CA1303.severity = warning # Do not pass literals as localized parameters
-dotnet_diagnostic.CS1591.severity = warning # Missing XML comment for publicly visible type or member
 dotnet_diagnostic.CA1805.severity = none # Member is explicitly initialized to its default value
 dotnet_diagnostic.CA1822.severity = none # Member does not access instance data and can be marked as static
 dotnet_diagnostic.CA1848.severity = none # For improved performance, use the LoggerMessage delegates
-dotnet_diagnostic.CA2007.severity = warning # Do not directly await a Task
-dotnet_diagnostic.CA2225.severity = warning # Operator overloads have named alternates
 dotnet_diagnostic.CA2227.severity = none # Change to be read-only by removing the property setter
 dotnet_diagnostic.CA2253.severity = none # Named placeholders in the logging message template should not be comprised of only numeric characters
-
-dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave
-dotnet_diagnostic.xUnit1004.severity = none # Test methods should not be skipped. Remove the Skip property to start running the test again.
-
 dotnet_diagnostic.RCS1021.severity = none # Use expression-bodied lambda.
-dotnet_diagnostic.RCS1032.severity = warning # Remove redundant parentheses.
 dotnet_diagnostic.RCS1061.severity = none # Merge 'if' with nested 'if'.
 dotnet_diagnostic.RCS1069.severity = none # Remove unnecessary case label.
 dotnet_diagnostic.RCS1074.severity = none # Remove redundant constructor.
 dotnet_diagnostic.RCS1077.severity = none # Optimize LINQ method call.
-dotnet_diagnostic.RCS1118.severity = warning # Mark local variable as const.
 dotnet_diagnostic.RCS1124.severity = none # Inline local variable.
 dotnet_diagnostic.RCS1129.severity = none # Remove redundant field initialization.
 dotnet_diagnostic.RCS1140.severity = none # Add exception to documentation comment.
-dotnet_diagnostic.RCS1141.severity = warning # Add 'param' element to documentation comment.
 dotnet_diagnostic.RCS1142.severity = none # Add 'typeparam' element to documentation comment.
 dotnet_diagnostic.RCS1146.severity = none # Use conditional access.
 dotnet_diagnostic.RCS1151.severity = none # Remove redundant cast.
@@ -201,18 +201,14 @@ dotnet_diagnostic.RCS1181.severity = none # Convert comment to documentation com
 dotnet_diagnostic.RCS1186.severity = none # Use Regex instance instead of static method.
 dotnet_diagnostic.RCS1188.severity = none # Remove redundant auto-property initialization.
 dotnet_diagnostic.RCS1189.severity = none # Add region name to #endregion.
-dotnet_diagnostic.RCS1197.severity = warning # Optimize StringBuilder.AppendLine call.
 dotnet_diagnostic.RCS1201.severity = none # Use method chaining.
-dotnet_diagnostic.RCS1205.severity = warning # Order named arguments according to the order of parameters.
 dotnet_diagnostic.RCS1212.severity = none # Remove redundant assignment.
 dotnet_diagnostic.RCS1217.severity = none # Convert interpolated string to concatenation.
 dotnet_diagnostic.RCS1222.severity = none # Merge preprocessor directives.
 dotnet_diagnostic.RCS1226.severity = none # Add paragraph to documentation comment.
-dotnet_diagnostic.RCS1229.severity = warning # Use async/await when necessary.
 dotnet_diagnostic.RCS1234.severity = none # Enum duplicate value
 dotnet_diagnostic.RCS1238.severity = none # Avoid nested ?: operators.
 dotnet_diagnostic.RCS1241.severity = none # Implement IComparable when implementing IComparable<T><T>.
-
 dotnet_diagnostic.IDE0001.severity = none # Simplify name
 dotnet_diagnostic.IDE0002.severity = none # Simplify member access
 dotnet_diagnostic.IDE0004.severity = none # Remove unnecessary cast
@@ -227,6 +223,8 @@ dotnet_diagnostic.IDE0100.severity = none # Remove unnecessary equality operator
 dotnet_diagnostic.IDE0110.severity = none # Remove unnecessary discards
 dotnet_diagnostic.IDE0032.severity = none # Use auto property
 dotnet_diagnostic.IDE0160.severity = none # Use block-scoped namespace
+dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave
+dotnet_diagnostic.xUnit1004.severity = none # Test methods should not be skipped. Remove the Skip property to start running the test again.
 
 ###############################
 # Naming Conventions          #

README.md

@@ -161,7 +161,7 @@ to **start the Semantic Memory Service**:
 > ### On Windows:
 >
 > ```shell
-> cd dotnet/Service
+> cd dotnet\Service
 > setup.cmd
 > run.cmd
 > ```

SemanticMemory.sln

@@ -18,6 +18,7 @@ EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "samples", "samples", "{0A43C65C-6007-4BB4-B3FE-8D439FC91841}"
 	ProjectSection(SolutionItems) = preProject
 		samples\README.md = samples\README.md
+		samples\.editorconfig = samples\.editorconfig
 	EndProjectSection
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "001-dotnet-Serverless", "samples\001-dotnet-Serverless\001-dotnet-Serverless.csproj", "{CF55CAED-81D1-4748-8172-AE678047B760}"
@@ -69,6 +70,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "nuget", "nuget", "{02B67859
 		dotnet\nuget\NUGET.md = dotnet\nuget\NUGET.md
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "010-using-core-nuget", "samples\010-using-core-nuget\010-using-core-nuget.csproj", "{17F42A38-46CB-4471-AF93-3AE6981B7278}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -85,6 +88,7 @@ Global
 		{8A90555B-C28B-401B-960A-619DE9A87C4C} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
 		{87DEAE8D-138C-4FDD-B4C9-11C3A7817E8F} = {6EF76FD8-4C35-4370-8539-5DDF45357A50}
 		{02B67859-5B16-46E3-9CB4-916887275649} = {87DEAE8D-138C-4FDD-B4C9-11C3A7817E8F}
+		{17F42A38-46CB-4471-AF93-3AE6981B7278} = {0A43C65C-6007-4BB4-B3FE-8D439FC91841}
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
 		{8A9FA587-7EBA-4D43-BE47-38D798B1C74C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
@@ -131,5 +135,9 @@ Global
 		{8A90555B-C28B-401B-960A-619DE9A87C4C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{8A90555B-C28B-401B-960A-619DE9A87C4C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{8A90555B-C28B-401B-960A-619DE9A87C4C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{17F42A38-46CB-4471-AF93-3AE6981B7278}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{17F42A38-46CB-4471-AF93-3AE6981B7278}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{17F42A38-46CB-4471-AF93-3AE6981B7278}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{17F42A38-46CB-4471-AF93-3AE6981B7278}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

dotnet/CoreLib/AI/Tokenizers/GPT3/EmbeddedResource.cs

@@ -0,0 +1,54 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.IO;
+using System.Reflection;
+using Microsoft.SemanticMemory.Core.Configuration;
+
+namespace Microsoft.SemanticMemory.Core.AI.Tokenizers.GPT3;
+
+/// <summary>
+/// Resource helper to load resources embedded in the assembly. By default we embed only
+/// text files, so the helper is limited to returning text.
+///
+/// You can find information about embedded resources here:
+/// * https://learn.microsoft.com/dotnet/core/extensions/create-resource-files
+/// * https://learn.microsoft.com/dotnet/api/system.reflection.assembly.getmanifestresourcestream?view=net-7.0
+///
+/// To know which resources are embedded, check the csproj file.
+/// </summary>
+internal static class EmbeddedResource
+{
+    private static readonly string? s_namespace = typeof(EmbeddedResource).Namespace;
+
+    /// <summary>
+    /// Return content of BPE file.
+    /// </summary>
+    internal static string ReadBytePairEncodingTable()
+    {
+        return Read("vocab.bpe");
+    }
+
+    /// <summary>
+    /// Return content of encoding table file.
+    /// </summary>
+    internal static string ReadEncodingTable()
+    {
+        return Read("encoder.json");
+    }
+
+    private static string Read(string fileName)
+    {
+        // Get the current assembly. Note: this class is in the same assembly where the embedded resources are stored.
+        Assembly? assembly = typeof(EmbeddedResource).GetTypeInfo().Assembly;
+        if (assembly == null) { throw new ConfigurationException($"[{s_namespace}] {fileName} assembly not found"); }
+
+        // Resources are mapped like types, using the namespace and appending "." (dot) and the file name
+        var resourceName = $"{s_namespace}." + fileName;
+        using Stream? resource = assembly.GetManifestResourceStream(resourceName);
+        if (resource == null) { throw new ConfigurationException($"{resourceName} resource not found"); }
+
+        // Return the resource content, in text format.
+        using var reader = new StreamReader(resource);
+        return reader.ReadToEnd();
+    }
+}

dotnet/CoreLib/AI/Tokenizers/GPT3/GPT3Settings.cs

@@ -0,0 +1,63 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using Microsoft.SemanticKernel.AI;
+
+namespace Microsoft.SemanticMemory.Core.AI.Tokenizers.GPT3;
+
+internal static class GPT3Settings
+{
+    // Lazy load and cache encoding table (encoder.json)
+    internal static Dictionary<string, int> Encoder => s_encoder.Value;
+
+    // Lazy load and cache byte pair encoding table (vocab.bpe)
+    internal static Dictionary<Tuple<string, string>, int> BpeRanks => s_bpeRanks.Value;
+
+    // Lazy load and cache encoding table (encoder.json)
+    private static readonly Lazy<Dictionary<string, int>> s_encoder = new(BuildEncoder);
+
+    // Lazy load and cache byte pair encoding table (vocab.bpe)
+    private static readonly Lazy<Dictionary<Tuple<string, string>, int>> s_bpeRanks = new(BuildBpeRanks);
+
+    private static Dictionary<Tuple<string, string>, int> BuildBpeRanks()
+    {
+        string[] lines = EmbeddedResource.ReadBytePairEncodingTable().Split('\n');
+        List<Tuple<string, string>> bpeMerges = new ArraySegment<string>(lines, 1, lines.Length - 1)
+            .Where(x => x.Trim().Length > 0)
+            .Select(x =>
+            {
+                string[] y = x.Split(' ');
+                return new Tuple<string, string>(y[0], y[1]);
+            }).ToList();
+        return DictZip(bpeMerges, Range(0, bpeMerges.Count));
+    }
+
+    private static Dictionary<string, int> BuildEncoder()
+    {
+        string json = EmbeddedResource.ReadEncodingTable();
+        var encoder = JsonSerializer.Deserialize<Dictionary<string, int>>(json, new JsonSerializerOptions());
+
+        return encoder
+               ?? throw new AIException(AIException.ErrorCodes.InvalidConfiguration,
+                   "Encoding table deserialization returned NULL");
+    }
+
+    private static Dictionary<Tuple<string, string>, int> DictZip(List<Tuple<string, string>> x, List<int> y)
+    {
+        var result = new Dictionary<Tuple<string, string>, int>();
+        for (int i = 0; i < x.Count; i++)
+        {
+            result.Add(x[i], y[i]);
+        }
+
+        return result;
+    }
+
+    private static List<int> Range(int x, int y)
+    {
+        return Enumerable.Range(x, y - x).ToList();
+    }
+}