fix(core): withhold partial fence and math markers at the end of a streamed prefix

docs/docs/rich-text-formatting/markdown-streaming.md

@@ -52,7 +52,7 @@ It takes **two** conditions, not one: `streamingAnimation` must be on **and** th
 
 | Mode | Behavior |
 | --- | --- |
-| `'progressive'` (default) | Streams the code line-by-line with a visible but non-interactive header (copying is disabled until the block completes); syntax highlighting is deferred until the closing fence so it does not flicker on every token. The block appears once the line that opens the fence has ended, so its language label never changes. |
+| `'progressive'` (default) | Streams the code line-by-line with a visible but non-interactive header (copying is disabled until the block completes); syntax highlighting is deferred until the closing fence so it does not flicker on every token. The block appears once the line that opens the fence has ended, so its language label never changes. The first one or two backticks or tildes of a fence, and the start of its closing fence (including the quote markers or indent before it), are withheld until the next character arrives, so they never flash as text or as a line of code. A fence that starts a list item or quote keeps its bullet or quote bar while it waits. |
 | `'hidden'` | Withholds the whole block until its closing fence arrives, then shows it complete. |
 
 A code block drawn by [`renderers.code`](/react-native/guides/custom-block-renderers#code-blocks) follows the same modes. In `'progressive'` its element re-renders with the growing code and receives `isComplete: false` until the closing fence arrives.
@@ -64,7 +64,7 @@ A code block drawn by [`renderers.code`](/react-native/guides/custom-block-rende
 | `'progressive'` (default) | Shows a trailing image paragraph, link paragraph or `<video>` block once its syntax is complete. A half-streamed `![alt](https://…` or `[label](https://…` is withheld instead of flashing as text, and a partial `<video` tag is not shown. A bare URL on its own line can always grow, so it is withheld until the next line arrives. |
 | `'hidden'` | Withholds the block until a blank line or another block follows it, like `tableMode: 'hidden'`. Applies to native images and to blocks drawn by [`renderers`](/react-native/guides/custom-block-renderers). |
 
-**Block math** has no mode key. An unterminated `$$` block is always withheld, truncated at the opening delimiter, until the closing `$$` arrives. On a page of LLM output that routinely contains LaTeX, this is the third incomplete-block case to expect.
+**Block math** has no mode key. An unterminated `$$` block is always withheld, truncated at the opening delimiter, until the closing `$$` arrives. A lone `$` at the start of a line is held back the same way until the next character shows whether it opens a `$$` block. On a page of LLM output that routinely contains LaTeX, this is the third incomplete-block case to expect.
 
 <LivePreview src={StreamingSrc} unavailable unavailableReason={<>iOS and Android only - <code>streamingAnimation</code> and <code>streamingConfig</code> are stripped on web.</>} />
 

packages/core/cpp/parser/StreamingFilter.cpp

@@ -70,6 +70,11 @@ size_t leadingSpaces(std::string_view text) {
 
 // ---- containers
 
+// Where `part`, a view into `line`, starts in it.
+size_t offsetIn(std::string_view line, std::string_view part) {
+  return static_cast<size_t>(part.data() - line.data());
+}
+
 // A line without its blockquote markers.
 struct Quoted {
   std::string_view body;
@@ -195,6 +200,20 @@ bool parseFenceMarker(std::string_view line, FenceMarker &out) {
   return true;
 }
 
+// Whether `line` is so far only the first `maxRun` or fewer characters of a
+// fence marker: up to three spaces, then a run of one fence character (`ch`, or
+// either kind when it is 0) and nothing after it. On the last line of a stream
+// the next character decides whether it is a fence or text.
+bool isFenceMarkerPrefix(std::string_view line, char ch, size_t maxRun) {
+  size_t i = leadingSpaces(line);
+  if (i > 3 || i >= line.size())
+    return false;
+  char mark = line[i];
+  if ((mark != '`' && mark != '~') || (ch != 0 && mark != ch))
+    return false;
+  return line.size() - i <= maxRun && line.find_first_not_of(mark, i) == npos;
+}
+
 // Reference labels match case-insensitively with whitespace runs collapsed.
 std::string normalizeLabel(std::string_view label) {
   std::string normalized;
@@ -224,6 +243,17 @@ void recordDefinition(std::string_view content, std::unordered_set<std::string>
 
 struct BlockScan {
   size_t openFence = npos; // the opening line of a fence still open at the end
+  // Where that line's fence marker starts, after any quote markers, list marker
+  // and indent: the part before it stays shown while the fence is withheld.
+  size_t openFenceMarker = npos;
+  // Where the last line's characters start when they are only the beginning of
+  // a block delimiter: one or two characters of a fence opener, the start of
+  // the closing fence of an open block, or the first `$` of a `$$` line. Inside
+  // an open fence, a last line that holds only whitespace or container markers
+  // is withheld from its start, since md4c would read it as a blank code line.
+  // The filter withholds them until the next character decides, so they never
+  // show as text or as a line of code that then disappears (#1130).
+  size_t pendingDelimiter = npos;
   std::unordered_set<std::string> definitions;
 };
 
@@ -246,16 +276,22 @@ BlockScan scanBlocks(Lines &lines) {
     bool blank = isBlank(body);
     size_t lead = leadingSpaces(body);
     if (inFence) {
-      // A blank last line has not streamed its quote markers in yet, so it cannot
-      // end a quoted fence.
-      bool undecided = i + 1 == lines.size() && isBlank(lines[i].text);
+      // A last line of only whitespace and quote markers has not streamed all of
+      // its markers in yet, so it cannot end a quoted fence. Until more arrives
+      // it is not shown either: md4c would read it as a blank line of code.
+      bool undecided = i + 1 == lines.size() && blank && quoted.depth <= fence.depth;
       bool containerEnded = !undecided && (quoted.depth < fence.depth || (!blank && lead < fence.indent));
+      if (undecided && !lines[i].text.empty())
+        scan.pendingDelimiter = lines[i].offset;
       if (!containerEnded) {
         lines[i].code = true;
         FenceMarker marker;
-        if (!blank && parseFenceMarker(body.substr(fence.indent), marker) && marker.ch == fence.ch &&
-            marker.length >= fence.length && isBlank(marker.info))
+        std::string_view inner = blank ? std::string_view() : body.substr(fence.indent);
+        if (!blank && parseFenceMarker(inner, marker) && marker.ch == fence.ch && marker.length >= fence.length &&
+            isBlank(marker.info))
           inFence = false;
+        else if (!blank && i + 1 == lines.size() && isFenceMarkerPrefix(inner, fence.ch, fence.length - 1))
+          scan.pendingDelimiter = lines[i].offset + offsetIn(lines[i].text, inner) + leadingSpaces(inner);
         continue;
       }
       inFence = false;
@@ -279,8 +315,15 @@ BlockScan scanBlocks(Lines &lines) {
       fence = {marker.ch, marker.length, quoted.depth, indent};
       lines[i].code = true;
       scan.openFence = i;
+      scan.openFenceMarker = lines[i].offset + offsetIn(lines[i].text, content) + leadingSpaces(content);
       continue;
     }
+    if (i + 1 == lines.size()) {
+      if (isFenceMarkerPrefix(content, 0, 2))
+        scan.pendingDelimiter = lines[i].offset + offsetIn(lines[i].text, content) + leadingSpaces(content);
+      else if (trim(body) == "$") // the block math rule reads `$$` lines the same way
+        scan.pendingDelimiter = lines[i].offset + offsetIn(lines[i].text, body) + leadingSpaces(body);
+    }
     recordDefinition(content, scan.definitions);
   }
   if (!inFence)
@@ -588,6 +631,10 @@ StreamingFilterResult renderableMarkdownForStreaming(const std::string &markdown
   std::string_view text(markdown);
   Lines lines = splitLines(text);
   BlockScan scan = scanBlocks(lines);
+  // The characters of a delimiter still being typed are withheld: the result is
+  // the one the stream had before they arrived.
+  if (scan.pendingDelimiter != npos)
+    return renderableMarkdownForStreaming(markdown.substr(0, scan.pendingDelimiter), options);
   if (scan.openFence == npos)
     return {std::string(removePendingBlocks({text, std::move(lines)}, options, false, scan)), false};
 
@@ -599,8 +646,14 @@ StreamingFilterResult renderableMarkdownForStreaming(const std::string &markdown
   size_t fenceOffset = head.lines[scan.openFence].offset;
   head.cutAtLine(scan.openFence);
   std::string_view kept = removePendingBlocks(std::move(head), options, true, scan);
-  if (options.codeBlockMode == StreamingBlockMode::Hidden || !openerEnded)
-    return {std::string(kept), false};
+  if (options.codeBlockMode == StreamingBlockMode::Hidden || !openerEnded) {
+    // Only the fence is withheld: the quote markers, list marker or indent before
+    // it on its line stay, so a list item or quote that starts with a fence keeps
+    // its bullet or bar while the fence waits.
+    std::string result(kept);
+    result.append(text.substr(fenceOffset, scan.openFenceMarker - fenceOffset));
+    return {std::move(result), false};
+  }
   std::string result;
   result.reserve(kept.size() + text.size() - fenceOffset);
   result.append(kept).append(text.substr(fenceOffset));

packages/core/cpp/parser/StreamingFilter.hpp

@@ -28,13 +28,18 @@ struct StreamingFilterResult {
 //   * tables: Hidden withholds a trailing table until a blank line follows it,
 //     Progressive trims an incomplete trailing row;
 //   * fenced code: Hidden cuts an open fence, Progressive keeps it once its
-//     opening line has ended and filters only the text before it;
+//     opening line has ended and filters only the text before it. Either way a
+//     cut fence keeps the quote markers or list marker before it on its line;
 //   * media (a trailing image paragraph, link-only paragraph or `<video>`
 //     block, the blocks a renderer can replace): Hidden withholds it
 //     until a blank line or another block follows, Progressive withholds it only
 //     while its syntax is still open;
 //   * an inline image or link still being typed at the end of the last
 //     paragraph (`text ![alt](https://`) is cut so it never shows as text.
+//   * the first characters of a delimiter still being typed on the last line
+//     (one or two of a fence opener, the start of an open fence's closing
+//     fence or only its quote markers or indent, a lone `$` before `$$`) are
+//     withheld until the next one decides.
 StreamingFilterResult renderableMarkdownForStreaming(const std::string &markdown,
                                                      const StreamingFilterOptions &options);
 

packages/core/cpp/tests/streaming_filter_fuzz.cpp

@@ -2,16 +2,20 @@
  * Streams every fixture one byte at a time through the filter and parses each
  * prefix with md4c: image, link and video syntax that the whole document
  * parses as syntax must never reach the screen as literal text while it
- * streams. md4c is the oracle, so the filter cannot drift from the parser.
+ * streams, and neither must the first characters of a code fence or a `$$`
+ * line (as prose, or as a line of code before the closing fence completes).
+ * md4c is the oracle, so the filter cannot drift from the parser.
  * See scripts/test-core-parser.sh.
  */
 #include "MD4CParser.hpp"
 #include "StreamingFilter.hpp"
+#include <algorithm>
 #include <cstring>
 #include <fstream>
 #include <iostream>
 #include <sstream>
 #include <string>
+#include <vector>
 
 namespace {
 
@@ -55,6 +59,19 @@ two lines](https://e.com/wrapped) and goes on.
 > const link = "[a](b)";
 > ```
 
+- Install it:
+  ```sh
+  npm i "[pkg](x)"
+  ```
+
+~~~~py
+print("```")
+~~~~
+
+$$
+e^{i\pi} + 1 = 0
+$$
+
 Prices were [about 500 yen][prices], see [notes] and [![thumb](https://e.com/t.jpg)](https://e.com/full.jpg).
 
 [inari]: https://e.com/inari
@@ -91,11 +108,47 @@ bool leaksSyntax(const std::string &fragment, const std::string &wholeText) {
   return false;
 }
 
+// The text of each code block, in document order.
+void collectCode(const MarkdownASTNode &node, std::vector<std::string> &out) {
+  if (node.type == NodeType::CodeBlock) {
+    std::string code;
+    collectText(node, code);
+    out.push_back(code);
+    return;
+  }
+  for (const auto &child : node.children)
+    collectCode(*child, out);
+}
+
+// A text fragment made only of fence characters, or a lone `$`: the start of a
+// fence or `$$` line shown before the rest of it arrived, unless the finished
+// document holds the same text.
+bool flashesDelimiter(const std::string &fragment, const std::string &wholeText) {
+  size_t start = fragment.find_first_not_of(' ');
+  if (start == std::string::npos)
+    return false;
+  size_t end = fragment.find_last_not_of(' ');
+  std::string marks = fragment.substr(start, end - start + 1);
+  bool fenceChars =
+      marks.find_first_not_of('`') == std::string::npos || marks.find_first_not_of('~') == std::string::npos;
+  if (!fenceChars && marks != "$")
+    return false;
+  std::istringstream lines(wholeText);
+  for (std::string line; std::getline(lines, line);) {
+    if (line == fragment)
+      return false;
+  }
+  return true;
+}
+
 int checkDocument(const std::string &name, const std::string &document) {
   Markdown::MD4CParser parser;
   std::string wholeText;
-  if (auto root = parser.parse(document))
+  std::vector<std::string> wholeCode;
+  if (auto root = parser.parse(document)) {
     collectText(*root, wholeText);
+    collectCode(*root, wholeCode);
+  }
   int failures = 0;
   for (StreamingBlockMode mode : {StreamingBlockMode::Progressive, StreamingBlockMode::Hidden}) {
     Markdown::StreamingFilterOptions options;
@@ -111,6 +164,20 @@ int checkDocument(const std::string &name, const std::string &document) {
         continue;
       std::string text;
       collectText(*root, text);
+      // A code block never shows more lines than it ends with: a line that is
+      // only the start of the closing fence, or of its quote markers or indent,
+      // would show as a line of code and then disappear.
+      std::vector<std::string> code;
+      collectCode(*root, code);
+      for (size_t k = 0; k < code.size() && k < wholeCode.size(); k++) {
+        auto lineCount = [](const std::string &s) { return std::count(s.begin(), s.end(), '\n'); };
+        if (lineCount(code[k]) > lineCount(wholeCode[k])) {
+          std::cerr << "FAIL: " << name << (mode == StreamingBlockMode::Hidden ? " (hidden)" : "")
+                    << " shows an extra line in code block " << k << " after " << length << " bytes\n";
+          failures++;
+          break;
+        }
+      }
       std::istringstream fragments(text);
       for (std::string fragment; std::getline(fragments, fragment);) {
         if (leaksSyntax(fragment, wholeText)) {
@@ -119,6 +186,12 @@ int checkDocument(const std::string &name, const std::string &document) {
           failures++;
           break;
         }
+        if (flashesDelimiter(fragment, wholeText)) {
+          std::cerr << "FAIL: " << name << (mode == StreamingBlockMode::Hidden ? " (hidden)" : "")
+                    << " shows the start of a delimiter after " << length << " bytes: [" << fragment << "]\n";
+          failures++;
+          break;
+        }
       }
     }
   }