Skip to content

Commit 13613e3

Browse files
dlucdluc
committed
String normalization extension
1 parent 353b662 commit 13613e3

10 files changed

Lines changed: 177 additions & 20 deletions

File tree

‎extensions/Chunkers/Chunkers/MarkDownChunker.cs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
using Microsoft.KernelMemory.AI;
1313
using Microsoft.KernelMemory.Chunkers.internals;
1414
using Microsoft.KernelMemory.DataFormats;
15+
using Microsoft.KernelMemory.Text;
1516

1617
namespace Microsoft.KernelMemory.Chunkers;
1718

@@ -152,10 +153,7 @@ public List<string> Split(string text, MarkDownChunkerOptions options)
152153
ArgumentNullException.ThrowIfNull(options);
153154

154155
// Clean up text. Note: LLMs don't use \r char
155-
text = text
156-
.Replace("\r\n", "\n", StringComparison.OrdinalIgnoreCase)
157-
.Replace("\r", "\n", StringComparison.OrdinalIgnoreCase)
158-
.Trim();
156+
text = text.NormalizeNewlines(true);
159157

160158
// Calculate chunk size leaving room for the optional chunk header
161159
int maxChunk1Size = options.MaxTokensPerChunk - this.TokenCount(options.ChunkHeader);

‎extensions/Chunkers/Chunkers/PlainTextChunker.cs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
using Microsoft.KernelMemory.AI;
1313
using Microsoft.KernelMemory.Chunkers.internals;
1414
using Microsoft.KernelMemory.DataFormats;
15+
using Microsoft.KernelMemory.Text;
1516

1617
namespace Microsoft.KernelMemory.Chunkers;
1718

@@ -127,10 +128,7 @@ public List<string> Split(string text, PlainTextChunkerOptions options)
127128
ArgumentNullException.ThrowIfNull(options);
128129

129130
// Clean up text. Note: LLMs don't use \r char
130-
text = text
131-
.Replace("\r\n", "\n", StringComparison.OrdinalIgnoreCase)
132-
.Replace("\r", "\n", StringComparison.OrdinalIgnoreCase)
133-
.Trim();
131+
text = text.NormalizeNewlines(true);
134132

135133
// Calculate chunk size leaving room for the optional chunk header
136134
int maxChunk1Size = options.MaxTokensPerChunk - this.TokenCount(options.ChunkHeader);

‎service/Abstractions/Text/StringExtensions.cs‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
namespace Microsoft.KernelMemory.Text;
4+
5+
public static class StringExtensions
6+
{
7+
public static string NormalizeNewlines(this string text, bool trim = false)
8+
{
9+
if (string.IsNullOrEmpty(text))
10+
{
11+
return text;
12+
}
13+
14+
// We won't need more than the original length
15+
char[] buffer = new char[text.Length];
16+
int bufferPos = 0;
17+
18+
// Skip leading whitespace if trimming
19+
int i = 0;
20+
if (trim)
21+
{
22+
while (i < text.Length && char.IsWhiteSpace(text[i]))
23+
{
24+
i++;
25+
}
26+
}
27+
28+
// Tracks the last non-whitespace position written into buffer
29+
int lastNonWhitespacePos = -1;
30+
31+
// 2) Single pass: replace \r\n or \r with \n, record last non-whitespace
32+
for (; i < text.Length; i++)
33+
{
34+
char c = text[i];
35+
36+
if (c == '\r')
37+
{
38+
// If \r\n then skip the \n
39+
if (i + 1 < text.Length && text[i + 1] == '\n')
40+
{
41+
i++;
42+
}
43+
44+
// Write a single \n
45+
buffer[bufferPos] = '\n';
46+
}
47+
else
48+
{
49+
buffer[bufferPos] = c;
50+
}
51+
52+
// If trimming, update lastNonWhitespacePos only when char isn't whitespace
53+
// If not trimming, always update because we keep everything
54+
if (!trim || !char.IsWhiteSpace(buffer[bufferPos]))
55+
{
56+
lastNonWhitespacePos = bufferPos;
57+
}
58+
59+
bufferPos++;
60+
}
61+
62+
// Cut off trailing whitespace if trimming
63+
// If every char was whitespace, lastNonWhitespacePos stays -1 and the result is an empty string
64+
int finalLength = (trim && lastNonWhitespacePos >= 0)
65+
? lastNonWhitespacePos + 1
66+
: bufferPos;
67+
68+
// Safety check if everything was trimmed away
69+
if (finalLength < 0)
70+
{
71+
finalLength = 0;
72+
}
73+
74+
return new string(buffer, 0, finalLength);
75+
}
76+
}

‎service/Core/DataFormats/Image/ImageDecoder.cs‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using Microsoft.Extensions.Logging;
99
using Microsoft.KernelMemory.Diagnostics;
1010
using Microsoft.KernelMemory.Pipeline;
11+
using Microsoft.KernelMemory.Text;
1112

1213
namespace Microsoft.KernelMemory.DataFormats.Image;
1314

@@ -64,7 +65,7 @@ public async Task<FileContent> DecodeAsync(Stream data, CancellationToken cancel
6465

6566
var result = new FileContent(MimeTypes.PlainText);
6667
var content = await this.ImageToTextAsync(data, cancellationToken).ConfigureAwait(false);
67-
result.Sections.Add(new(content.Trim(), 1, Chunk.Meta(sentencesAreComplete: true)));
68+
result.Sections.Add(new(content, 1, Chunk.Meta(sentencesAreComplete: true)));
6869

6970
return result;
7071
}
@@ -87,10 +88,14 @@ private async Task<string> ImageToTextAsync(BinaryData data, CancellationToken c
8788
}
8889
}
8990

90-
private Task<string> ImageToTextAsync(Stream data, CancellationToken cancellationToken = default)
91+
private async Task<string> ImageToTextAsync(Stream data, CancellationToken cancellationToken = default)
9192
{
92-
return this._ocrEngine is null
93-
? throw new NotSupportedException($"Image extraction not configured")
94-
: this._ocrEngine.ExtractTextFromImageAsync(data, cancellationToken);
93+
if (this._ocrEngine is null)
94+
{
95+
throw new NotSupportedException($"Image extraction not configured");
96+
}
97+
98+
string text = await this._ocrEngine.ExtractTextFromImageAsync(data, cancellationToken).ConfigureAwait(false);
99+
return text.NormalizeNewlines(true);
95100
}
96101
}

‎service/Core/DataFormats/Office/MsExcelDecoder.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ public Task<FileContent> DecodeAsync(Stream data, CancellationToken cancellation
151151
sb.AppendLineNix(this._config.EndOfWorksheetMarkerTemplate.Replace("{number}", $"{worksheetNumber}", StringComparison.OrdinalIgnoreCase));
152152
}
153153

154-
string worksheetContent = sb.ToString().Trim();
154+
string worksheetContent = sb.ToString().NormalizeNewlines(true);
155155
sb.Clear();
156156
result.Sections.Add(new Chunk(worksheetContent, worksheetNumber, Chunk.Meta(sentencesAreComplete: true)));
157157
}

‎service/Core/DataFormats/Office/MsPowerPointDecoder.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ public Task<FileContent> DecodeAsync(Stream data, CancellationToken cancellation
113113
}
114114
}
115115

116-
string slideContent = sb.ToString().Trim();
116+
string slideContent = sb.ToString().NormalizeNewlines(true);
117117
sb.Clear();
118118
result.Sections.Add(new Chunk(slideContent, slideNumber, Chunk.Meta(sentencesAreComplete: true)));
119119
}

‎service/Core/DataFormats/Office/MsWordDecoder.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,8 @@ public Task<FileContent> DecodeAsync(Stream data, CancellationToken cancellation
8080
var lastRenderedPageBreak = p.GetFirstChild<Run>()?.GetFirstChild<LastRenderedPageBreak>();
8181
if (lastRenderedPageBreak != null)
8282
{
83-
string pageContent = sb.ToString().Trim();
83+
// Note: no trimming, use original spacing when working with pages
84+
string pageContent = sb.ToString().NormalizeNewlines(false);
8485
sb.Clear();
8586
result.Sections.Add(new Chunk(pageContent, pageNumber, Chunk.Meta(sentencesAreComplete: true)));
8687
pageNumber++;
@@ -90,7 +91,8 @@ public Task<FileContent> DecodeAsync(Stream data, CancellationToken cancellation
9091
}
9192
}
9293

93-
var lastPageContent = sb.ToString().Trim();
94+
// Note: no trimming, use original spacing when working with pages
95+
string lastPageContent = sb.ToString().NormalizeNewlines(false);
9496
result.Sections.Add(new Chunk(lastPageContent, pageNumber, Chunk.Meta(sentencesAreComplete: true)));
9597

9698
return Task.FromResult(result);

‎service/Core/DataFormats/Pdf/PdfDecoder.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using Microsoft.Extensions.Logging;
1010
using Microsoft.KernelMemory.Diagnostics;
1111
using Microsoft.KernelMemory.Pipeline;
12+
using Microsoft.KernelMemory.Text;
1213
using UglyToad.PdfPig;
1314
using UglyToad.PdfPig.Content;
1415
using UglyToad.PdfPig.DocumentLayoutAnalysis.TextExtractor;
@@ -56,8 +57,9 @@ public Task<FileContent> DecodeAsync(Stream data, CancellationToken cancellation
5657

5758
foreach (Page? page in pdfDocument.GetPages().Where(x => x != null))
5859
{
59-
// Note: no trimming, use original spacing
60-
string pageContent = ContentOrderTextExtractor.GetText(page) ?? string.Empty;
60+
// Note: no trimming, use original spacing when working with pages
61+
string pageContent = ContentOrderTextExtractor.GetText(page).NormalizeNewlines(false) ?? string.Empty;
62+
6163
result.Sections.Add(new Chunk(pageContent, page.Number, Chunk.Meta(sentencesAreComplete: false)));
6264
}
6365

‎service/Core/DataFormats/WebPages/HtmlDecoder.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using Microsoft.Extensions.Logging;
1010
using Microsoft.KernelMemory.Diagnostics;
1111
using Microsoft.KernelMemory.Pipeline;
12+
using Microsoft.KernelMemory.Text;
1213

1314
namespace Microsoft.KernelMemory.DataFormats.WebPages;
1415

@@ -51,7 +52,7 @@ public Task<FileContent> DecodeAsync(Stream data, CancellationToken cancellation
5152
var doc = new HtmlDocument();
5253
doc.Load(data);
5354

54-
result.Sections.Add(new Chunk(doc.DocumentNode.InnerText.Trim(), 1, Chunk.Meta(sentencesAreComplete: true)));
55+
result.Sections.Add(new Chunk(doc.DocumentNode.InnerText.NormalizeNewlines(true), 1, Chunk.Meta(sentencesAreComplete: true)));
5556

5657
return Task.FromResult(result);
5758
}

‎service/tests/Abstractions.UnitTests/Text/StringExtensionsTest.cs‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using Microsoft.KernelMemory.Text;
4+
5+
namespace Microsoft.KM.Abstractions.UnitTests.Text;
6+
7+
public class StringExtensionsTest
8+
{
9+
[Theory]
10+
[Trait("Category", "UnitTest")]
11+
[InlineData(null, null)]
12+
[InlineData("", "")]
13+
[InlineData(" ", " ")]
14+
[InlineData("\n", "\n")]
15+
[InlineData("\r", "\n")] // Old Mac
16+
[InlineData("\r\n", "\n")] // Windows
17+
[InlineData("\n\r", "\n\n")] // Not standard, that's 2 line endings
18+
[InlineData("\n\n\n", "\n\n\n")]
19+
[InlineData("\r\r\r", "\n\n\n")]
20+
[InlineData("\r\r\n\r", "\n\n\n")]
21+
[InlineData("\n\r\n\r", "\n\n\n")]
22+
[InlineData("ciao", "ciao")]
23+
[InlineData("ciao ", "ciao ")]
24+
[InlineData(" ciao ", " ciao ")]
25+
[InlineData("\r ciao ", "\n ciao ")]
26+
[InlineData(" \rciao ", " \nciao ")]
27+
[InlineData(" \r\nciao ", " \nciao ")]
28+
[InlineData(" \r\nciao\n ", " \nciao\n ")]
29+
[InlineData(" \r\nciao \n", " \nciao \n")]
30+
[InlineData(" \r\nciao \r", " \nciao \n")]
31+
[InlineData(" \r\nciao \rn", " \nciao \nn")]
32+
public void ItNormalizesLineEndings(string? input, string? expected)
33+
{
34+
// Act
35+
string actual = input.NormalizeNewlines();
36+
37+
// Assert
38+
Assert.Equal(expected, actual);
39+
}
40+
41+
[Theory]
42+
[Trait("Category", "UnitTest")]
43+
[InlineData(null, null)]
44+
[InlineData("", "")]
45+
[InlineData(" ", "")]
46+
[InlineData("\n", "")]
47+
[InlineData("\r", "")]
48+
[InlineData("\r\n", "")]
49+
[InlineData("\n\r", "")]
50+
[InlineData("\n\n\n", "")]
51+
[InlineData("\r\r\r", "")]
52+
[InlineData("\r\r\n\r", "")]
53+
[InlineData("\n\r\n\r", "")]
54+
[InlineData("ciao", "ciao")]
55+
[InlineData("ciao ", "ciao")]
56+
[InlineData(" ciao ", "ciao")]
57+
[InlineData("\r ciao ", "ciao")]
58+
[InlineData(" \rciao ", "ciao")]
59+
[InlineData(" \r\nciao ", "ciao")]
60+
[InlineData(" \r\nciao\n ", "ciao")]
61+
[InlineData(" \r\nciao \n", "ciao")]
62+
[InlineData(" \r\nciao \r", "ciao")]
63+
[InlineData(" \r\nciao \rn", "ciao \nn")]
64+
[InlineData(" \r\nc\ri\ra\no \r", "c\ni\na\no")]
65+
[InlineData(" \r\nc\r\ni\n\na\r\ro \r", "c\ni\n\na\n\no")]
66+
[InlineData(" \r\nccc\r\ni\n\naaa\r\ro \r", "ccc\ni\n\naaa\n\no")]
67+
public void ItCanTrimWhileNormalizingLineEndings(string? input, string? expected)
68+
{
69+
// Act
70+
string actual = input.NormalizeNewlines(true);
71+
72+
// Assert
73+
Assert.Equal(expected, actual);
74+
}
75+
}

0 commit comments

Comments
 (0)