HtmlHelper.toPlainText: correctly decode character references

eclipse-scout-core/src/encoder/PlainTextEncoder.ts

@@ -17,7 +17,8 @@ export interface PlainTextEncoderOptions {
   compact?: boolean;
 
   /**
-   * Calls string.trim(). White space at the beginning and the end of the text gets removed. Default is false.
+   * Calls string.trim(). Empty lines at the beginning and the end of the text get removed. Default is false.
+   * Spaces are always removed.
    */
   trim?: boolean;
 

eclipse-scout-core/test/encoder/PlainTextEncoderSpec.ts

@@ -15,6 +15,7 @@ describe('PlainTextEncoder', () => {
   let encoder = new PlainTextEncoder();
 
   it('converts HTML to plain text', () => {
+    expect(encoder.encode(null)).toBe(null);
     expect(encoder.encode('')).toBe('');
 
     let htmlText = '<b>hello</b>';
@@ -113,6 +114,15 @@ describe('PlainTextEncoder', () => {
 
   it('removes leading and trailing newlines if configured', () => {
     let htmlText = '\n\nHello!\n\n';
+    expect(encoder.encode(htmlText, {trim: false})).toBe('\n\nHello!\n\n');
+    expect(encoder.encode(htmlText, {trim: true})).toBe('Hello!');
+
+    htmlText = '\n\n Hello! \n\n';
+    expect(encoder.encode(htmlText, {trim: false})).toBe('\n\nHello!\n\n');
+    expect(encoder.encode(htmlText, {trim: true})).toBe('Hello!');
+
+    htmlText = '  \n  \n  Hello!  \n  \n  ';
+    expect(encoder.encode(htmlText, {trim: false})).toBe('\n\nHello!\n\n');
     expect(encoder.encode(htmlText, {trim: true})).toBe('Hello!');
   });
 
@@ -259,4 +269,23 @@ describe('PlainTextEncoder', () => {
     htmlText = '<\tabc attr=\'someText\'>';
     expect(encoder.removeAttributeValues(htmlText)).toBe('<\tabc attr=\'someText\'>');
   });
+
+  it('trims lines, but preserves other white-space', () => {
+    expect(encoder.encode('hello')).toBe('hello');
+    expect(encoder.encode('one\ntwo')).toBe('one\ntwo');
+    expect(encoder.encode('one\r\ntwo')).toBe('one\ntwo');
+    expect(encoder.encode('one\rtwo')).toBe('one\ntwo');
+    expect(encoder.encode('one   two')).toBe('one   two');
+    expect(encoder.encode('one&nbsp;&nbsp; two')).toBe('one\u00a0\u00a0 two');
+    expect(encoder.encode('one &#9; two')).toBe('one \t two');
+    expect(encoder.encode(' one \n two  \n   three ')).toBe('one\ntwo\nthree');
+    expect(encoder.encode('a\n&nbsp;&nbsp;b\n&nbsp;&nbsp;&nbsp;&nbsp;c')).toBe('a\n\u00A0\u00A0b\n\u00A0\u00A0\u00A0\u00A0c');
+  });
+
+  it('decodes special characters', () => {
+    expect(encoder.encode('&amp;amp;')).toBe('&amp;');
+    expect(encoder.encode('&amp;&amp;amp;amp;')).toBe('&&amp;amp;');
+    expect(encoder.encode('Hell&ouml;!')).toBe('Hellö!');
+    expect(encoder.encode('a&lt;br&gt;b')).toBe('a<br>b');
+  });
 });

org.eclipse.scout.rt.platform.test/src/test/java/org/eclipse/scout/rt/platform/html/HtmlEntitiesTest.java

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2010, 2025 BSI Business Systems Integration AG
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipse.scout.rt.platform.html;
+
+import static org.junit.Assert.*;
+
+import org.eclipse.scout.rt.platform.BEANS;
+import org.eclipse.scout.rt.testing.platform.runner.PlatformTestRunner;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+@SuppressWarnings({"ConcatenationWithEmptyString", "SpellCheckingInspection", "TextBlockMigration"})
+@RunWith(PlatformTestRunner.class)
+public class HtmlEntitiesTest {
+
+  @Test
+  public void testUnescapeAll_Empty() {
+    final HtmlEntities entities = BEANS.get(HtmlEntities.class);
+    assertNull(entities.unescapeAll(null));
+    assertEquals("", entities.unescapeAll(""));
+    assertEquals("  ", entities.unescapeAll("  "));
+  }
+
+  @Test
+  public void testUnescapeAll_Named() {
+    final HtmlEntities entities = BEANS.get(HtmlEntities.class);
+    assertEquals("ß", entities.unescapeAll("ß"));
+    assertEquals("Ü", entities.unescapeAll("Ü"));
+    assertEquals("&", entities.unescapeAll("&"));
+    assertEquals("A&&Z", entities.unescapeAll("A&&Z"));
+    assertEquals("ä", entities.unescapeAll("ä"));
+    assertEquals("auml;", entities.unescapeAll("auml;"));
+    assertEquals("drag&drop", entities.unescapeAll("drag&drop"));
+  }
+
+  @Test
+  public void testUnescapeAll_NumericDecimal() {
+    final HtmlEntities entities = BEANS.get(HtmlEntities.class);
+    assertEquals("'", entities.unescapeAll("'"));
+    assertEquals("€", entities.unescapeAll("€"));
+    assertEquals("A€€Z", entities.unescapeAll("A€€Z"));
+  }
+
+  @Test
+  public void testUnescapeAll_NumericHex() {
+    final HtmlEntities entities = BEANS.get(HtmlEntities.class);
+    assertEquals("B", entities.unescapeAll("B"));
+    assertEquals("B", entities.unescapeAll("B"));
+    assertEquals("?", entities.unescapeAll("?"));
+    assertEquals("?", entities.unescapeAll("?"));
+    assertEquals("🦕", entities.unescapeAll("🦕"));
+    assertEquals("A🦕🦕Z", entities.unescapeAll("A🦕🦕Z"));
+  }
+
+  @Test
+  public void testUnescapeAll_ComplexStrings() {
+    final HtmlEntities entities = BEANS.get(HtmlEntities.class);
+    assertEquals("text'text", entities.unescapeAll("text'text"));
+    assertEquals("'€", entities.unescapeAll("'€"));
+    assertEquals("/ˈʊmlaʊt/", entities.unescapeAll("/ˈʊmlaʊt/"));
+    assertEquals("Kühlflüssigkeitsüberlaufbehälter", entities.unescapeAll("Kühlflüssigkeitsüberlaufbehälter"));
+    assertEquals(" [Viele Kühe machen Mühe!] ", entities.unescapeAll(" [Viele Kühe machen Mühe!] "));
+    assertEquals("🤸🏾‍♀️", entities.unescapeAll("🤸🏾‍♀️"));
+    assertEquals("🏴‍☠️", entities.unescapeAll("🏴‍☠️"));
+    assertEquals(""
+            + "Face with Tears of Joy Emoji: \uD83D\uDE02\n"
+            + "Party Popper Emoji: \uD83C\uDF89\n"
+            + "Man Technologist: Medium-light Skin Tone: \uD83D\uDC68\uD83C\uDFFC\u200D\uD83D\uDCBB",
+        entities.unescapeAll(""
+            + "Face with Tears of Joy Emoji: 😂\n"
+            + "Party Popper Emoji: 🎉\n"
+            + "Man Technologist: Medium-light Skin Tone: 👨🏼‍💻"));
+    assertEquals("<body><div data-value=\"¡Hola!\">Zürich</div><hr></body>", entities.unescapeAll("<body><div data-value=\"&#161;Hola!\">Z&uuml;rich</div><hr></body>"));
+    assertEquals("Qu’est-ce que c’est?", entities.unescapeAll("Qu&#8217;est-ce que c&#8217;est?"));
+  }
+
+  @Test
+  public void testUnescapeAll_Invalid() {
+    final HtmlEntities entities = BEANS.get(HtmlEntities.class);
+    assertEquals("ä &doesNotExist; é", entities.unescapeAll("&auml; &doesNotExist; &eacute;")); // invalid entity name
+    assertEquals("a &nbsp b", entities.unescapeAll("a &nbsp b")); // not terminated by ';'
+    assertEquals("drag&drop", entities.unescapeAll("drag&drop")); // not an entity name
+    assertEquals("drag & drop", entities.unescapeAll("drag & drop")); // not an entity name
+    assertEquals("& auml;", entities.unescapeAll("& auml;")); // space after '&'
+    assertEquals("&auml ;", entities.unescapeAll("&auml ;")); // space before ';'
+    assertEquals("&auml,", entities.unescapeAll("&auml,")); // not terminated by ';'
+    assertEquals("&#1234567890;", entities.unescapeAll("&#1234567890;")); // not a valid code point
+    assertEquals("&39;", entities.unescapeAll("&39;")); // missing '#'
+    assertEquals("&#a0;", entities.unescapeAll("&#a0;")); // missing 'x'
+  }
+
+  @Test
+  public void testDecodeNamedCharacterReference() {
+    final HtmlEntities entities = BEANS.get(HtmlEntities.class);
+
+    assertEquals("ä", entities.decodeNamedCharacterReference("&auml;"));
+    assertEquals("'", entities.decodeNamedCharacterReference("&apos;"));
+
+    assertNull(entities.decodeNamedCharacterReference(null));
+    assertNull(entities.decodeNamedCharacterReference(""));
+    assertNull(entities.decodeNamedCharacterReference("&doesNotExist;"));
+    assertNull(entities.decodeNamedCharacterReference("&auml;&uuml;"));
+    assertNull(entities.decodeNamedCharacterReference("auml"));
+    assertNull(entities.decodeNamedCharacterReference("&#39;"));
+  }
+
+  @Test
+  public void testDecodeNumericCharacterReference() {
+    final HtmlEntities entities = BEANS.get(HtmlEntities.class);
+
+    assertEquals("ä", entities.decodeNumericCharacterReference("&#228;"));
+    assertEquals("ä", entities.decodeNumericCharacterReference("&#xE4;"));
+    assertEquals("'", entities.decodeNumericCharacterReference("&#39;"));
+    assertEquals("'", entities.decodeNumericCharacterReference("&#x27;"));
+
+    assertNull(entities.decodeNumericCharacterReference(null));
+    assertNull(entities.decodeNumericCharacterReference(""));
+    assertNull(entities.decodeNumericCharacterReference("&doesNotExist;"));
+    assertNull(entities.decodeNumericCharacterReference("&#228;&#39;"));
+    assertNull(entities.decodeNumericCharacterReference("#39;"));
+    assertNull(entities.decodeNumericCharacterReference("&#E4;"));
+    assertNull(entities.decodeNumericCharacterReference("&#hello;"));
+    assertNull(entities.decodeNumericCharacterReference("&#1234567890;"));
+  }
+}

org.eclipse.scout.rt.platform.test/src/test/java/org/eclipse/scout/rt/platform/html/HtmlHelperTest.java

@@ -20,6 +20,7 @@
 /**
  * @since 5.2
  */
+@SuppressWarnings({"SpellCheckingInspection", "ConcatenationWithEmptyString", "TextBlockMigration", "UnnecessaryUnicodeEscape"})
 @RunWith(PlatformTestRunner.class)
 public class HtmlHelperTest {
 
@@ -40,12 +41,13 @@ public void testToPlainText() {
     assertEquals("one two", helper.toPlainText("one\r\ntwo"));
     assertEquals("onetwo", helper.toPlainText("one\rtwo"));
     assertEquals("hell<", helper.toPlainText("hell&lt;"));
-    assertEquals("one   two", helper.toPlainText("one&nbsp;&nbsp; two"));
-    assertEquals("hell&ouml;", helper.toPlainText("hell&ouml;")); // [?] not all entities are replaced
+    assertEquals("hellö", helper.toPlainText("hell&ouml;"));
     assertEquals("one\ttwo", helper.toPlainText("one&#9;two"));
     assertEquals("one \t two", helper.toPlainText("one &#9; two"));
     assertEquals("one\ttwo", helper.toPlainText("one" + StringUtility.HTML_ENCODED_TAB + "two"));
-    assertEquals("one    two", helper.toPlainText("one&#160;&#xa0;&#Xa0;&#xA0;two")); // HTML5 spec allows for mixed case hex values.
+    assertEquals("one two", helper.toPlainText("one   two"));
+    assertEquals("one\u00A0\u00A0 two", helper.toPlainText("one&nbsp;&nbsp; two"));
+    assertEquals("one\u00A0\u00A0\u00A0\u00A0two", helper.toPlainText("one&#160;&#xa0;&#Xa0;&#xA0;two")); // HTML5 spec allows for mixed case hex values.
     assertEquals("one\t\ttwo", helper.toPlainText("one&#x9;&#X9;two")); // HTML5 spec allows for mixed case hex values.
     assertEquals("Unterraschungsfeier", helper.toPlainText("<div class=\"rte-line\">Unter<u>rasch</u>u<span class=\"rte-highlight\" style=\"background-color: rgb(255, 219, 157)\">ngs</span>feier<br></div>")); // Formating tags within a single word.
     assertEquals("Header 1\nHeader 2", helper.toPlainText("<h1>Header 1</h1><h1>Header 2</h1>")); // Headers
@@ -71,15 +73,27 @@ public void testToPlainText() {
     assertEquals("one & two", helper.toPlainText("<html><head>one & two</html>"));
     assertEquals("one & two", helper.toPlainText("<html><head>one &amp; two</html>"));
     assertEquals("one & two\nthree", helper.toPlainText("<html><head>one &amp; two</head><body>three</html>")); // [?] invalid <body>, has no end tag
+    assertEquals("&amp;", helper.toPlainText("&amp;amp;"));
+    assertEquals("&auml;", helper.toPlainText("&amp;auml;"));
+    assertEquals("&#x42;", helper.toPlainText("&amp;#x42;"));
     assertEquals("three", helper.toPlainText("<html><head>one &amp; two</head><body>three</body></html>"));
     assertEquals("Unterraschungsfeier",
         helper.toPlainText("<html><body><div class=\"rte-line\">Unter<u>rasch</u>u<span class=\"rte-highlight\" style=\"background-color: rgb(255, 219, 157)\">ngs</span>feier<br></div></body></html>"));
+    assertEquals("Z1\nZ2", helper.toPlainText("<p>Z1</p><span></span><p>Z2</p>"));
+    assertEquals("Guten Tag\n\n\u00A0\n\nZeile 2", helper.toPlainText("<html><body><div><div><p>Guten Tag<o:p></o:p></p></div><div><p><o:p>&nbsp;</o:p></span></p></div><div><p><span>Zeile 2<o:p></o:p></span></p></div></div></body></html>"));
+    assertEquals("’", helper.toPlainText("&#8217;"));
+    assertEquals("+", helper.toPlainText("&#43;"));
+    assertEquals("+", helper.toPlainText("&#x2B;"));
+    assertEquals("+", helper.toPlainText("&#X2B;"));
+    assertEquals("&#X2Bs;", helper.toPlainText("&#X2Bs;"));
 
     // Line breaks
     assertEquals("a\nb", helper.toPlainText("a<br>b"));
     assertEquals("a\nb", helper.toPlainText("a <br/> b"));
     assertEquals("a\nb", helper.toPlainText("a    <br/> b"));
-    assertEquals("a \nb", helper.toPlainText("a&nbsp;<br/> b"));
+    assertEquals("a\u00A0\nb", helper.toPlainText("a&nbsp;<br/> b"));
+    assertEquals("a \u00A0\nb", helper.toPlainText("a  &nbsp;  <br/>  b  "));
+    assertEquals("a\n\u00A0\u00A0b\n\u00A0\u00A0\u00A0\u00A0c", helper.toPlainText("a<br>&nbsp;&nbsp;b<br>&nbsp;&nbsp;&nbsp;&nbsp;c"));
     assertEquals("line", helper.toPlainText("<br/>line")); // [?]
     assertEquals("line1\nx\nline2", helper.toPlainText("<p>line1<br>\nx</p><p>line2</p>"));
     assertEquals("line1 x\nline2", helper.toPlainText("<div>line1\nx</div><div>line2</div>")); // [?]