chore: migrate out smart cell logic to its own package (#6808)

Internal refactor to help share code with vscode-extension for our smart
cells

Author: mscolnick

frontend/package.json

@@ -55,6 +55,7 @@
     "@marimo-team/codemirror-sql": "^0.2.3",
     "@marimo-team/llm-info": "workspace:*",
     "@marimo-team/marimo-api": "workspace:*",
+    "@marimo-team/smart-cells": "workspace:*",
     "@marimo-team/react-slotz": "^0.2.0",
     "@mui/material": "^6.5.0",
     "@open-rpc/client-js": "^1.8.1",

frontend/src/core/codemirror/language/__tests__/ast.test.ts

@@ -174,17 +174,25 @@ describe("switchLanguage", () => {
     // Switch to sql
     switchLanguage(mockEditor, { language: "sql", keepCodeAsIs: false });
     expect(mockEditor.state.doc.toString()).toMatchInlineSnapshot(`
-      "print('Hello')
-      print('Goodbye')"
+      "mo.md(
+          r"""
+      print('Hello')
+      print('Goodbye')
+      """
+      )"
     `);
 
     // Switch back to python
     switchLanguage(mockEditor, { language: "python", keepCodeAsIs: false });
     expect(mockEditor.state.doc.toString()).toMatchInlineSnapshot(`
       "_df = mo.sql(
           f"""
+          mo.md(
+              r\\"""
           print('Hello')
           print('Goodbye')
+          \\"""
+          )
           """
       )"
     `);

frontend/src/core/codemirror/language/__tests__/extension.test.ts

@@ -8,7 +8,10 @@ import { languages } from "@codemirror/language-data";
 import { stexMath } from "@codemirror/legacy-modes/mode/stex";
 import type { Extension } from "@codemirror/state";
 import { type EditorView, ViewPlugin } from "@codemirror/view";
-import dedent from "string-dedent";
+import {
+  type MarkdownMetadata,
+  MarkdownParser,
+} from "@marimo-team/smart-cells";
 import type { CellId } from "@/core/cells/ids";
 import type { CompletionConfig } from "@/core/config/config-schema";
 import type { HotkeyProvider } from "@/core/hotkeys/hotkeys";
@@ -22,183 +25,43 @@ import { parsePython } from "../embedded/embedded-python";
 import { parseLatex } from "../embedded/latex";
 import { languageMetadataField } from "../metadata";
 import type { LanguageAdapter } from "../types";
-import {
-  QUOTE_PREFIX_KINDS,
-  type QuotePrefixKind,
-  splitQuotePrefix,
-} from "../utils/quotes";
-
-export interface MarkdownLanguageAdapterMetadata {
-  quotePrefix: QuotePrefixKind;
-}
 
-const quoteKinds = [
-  ['"""', '"""'],
-  ["'''", "'''"],
-  ['"', '"'],
-  ["'", "'"],
-];
-
-// explode into all combinations
-const pairs = QUOTE_PREFIX_KINDS.flatMap((prefix) =>
-  quoteKinds.map(([start, end]) => [prefix + start, end]),
-);
-
-const regexes = pairs.map(
-  ([start, end]) =>
-    // mo.md( + any number of spaces + start + capture + any number of spaces + end)
-    [
-      start,
-      new RegExp(`^mo\\.md\\(\\s*${start}(.*)${end}\\s*\\)$`, "s"),
-    ] as const,
-);
+export interface MarkdownLanguageAdapterMetadata extends MarkdownMetadata {}
 
 /**
  * Language adapter for Markdown.
  */
 export class MarkdownLanguageAdapter
   implements LanguageAdapter<MarkdownLanguageAdapterMetadata>
 {
+  private parser = new MarkdownParser();
+
   readonly type = "markdown";
-  readonly defaultCode = 'mo.md(r"""\n""")';
-  readonly defaultMetadata: MarkdownLanguageAdapterMetadata = {
-    quotePrefix: "r",
-  };
+  readonly defaultCode = this.parser.defaultCode;
+  readonly defaultMetadata: MarkdownLanguageAdapterMetadata =
+    this.parser.defaultMetadata;
 
   static fromMarkdown(markdown: string) {
-    return `mo.md(r"""\n${markdown}\n""")`;
+    return MarkdownParser.fromMarkdown(markdown);
   }
 
   transformIn(
     pythonCode: string,
   ): [string, number, MarkdownLanguageAdapterMetadata] {
-    pythonCode = pythonCode.trim();
-
-    const metadata = { ...this.defaultMetadata };
-
-    // empty string
-    if (pythonCode === "") {
-      return ["", 0, metadata];
-    }
-
-    for (const [start, regex] of regexes) {
-      const match = pythonCode.match(regex);
-      if (match) {
-        const innerCode = match[1];
-        const [quotePrefix, quoteType] = splitQuotePrefix(start);
-        metadata.quotePrefix = quotePrefix;
-        const unescapedCode = innerCode.replaceAll(`\\${quoteType}`, quoteType);
-
-        const offset = pythonCode.indexOf(innerCode);
-        // string-dedent expects the first and last line to be empty / contain only whitespace, so we pad with \n
-        return [dedent(`\n${unescapedCode}\n`).trim(), offset, metadata];
-      }
-    }
-
-    // no match
-    return [pythonCode, 0, metadata];
+    const result = this.parser.transformIn(pythonCode);
+    return [result.code, result.offset, result.metadata];
   }
 
   transformOut(
     code: string,
     metadata: MarkdownLanguageAdapterMetadata,
   ): [string, number] {
-    // NB. Must be kept consistent with marimo/_convert/utils.py
-    // ::markdown_to_marimo
-
-    // Empty string
-    if (code === "") {
-      // Need at least a space, otherwise the output will be 6 quotes
-      code = " ";
-    }
-
-    const { quotePrefix } = metadata;
-
-    // We always transform back with triple quotes, as to avoid needing to
-    // escape single quotes.
-    const escapedCode = code.replaceAll('"""', String.raw`\"""`);
-
-    // If its one line and not bounded by quotes, write it as single line
-    const isOneLine = !code.includes("\n");
-    const boundedByQuote = code.startsWith('"') || code.endsWith('"');
-    if (isOneLine && !boundedByQuote) {
-      const start = `mo.md(${quotePrefix}"""`;
-      const end = `""")`;
-      return [start + escapedCode + end, start.length];
-    }
-
-    // Multiline code
-    const start = `mo.md(\n    ${quotePrefix}"""\n`;
-    const end = `\n"""\n)`;
-    return [start + escapedCode + end, start.length + 1];
+    const result = this.parser.transformOut(code, metadata);
+    return [result.code, result.offset];
   }
 
   isSupported(pythonCode: string): boolean {
-    pythonCode = pythonCode.trim();
-
-    // Empty strings are supported
-    if (pythonCode === "") {
-      return true;
-    }
-
-    // Must start with mo.md(
-    if (!pythonCode.startsWith("mo.md(")) {
-      return false;
-    }
-
-    // Empty function calls are supported
-    if (pythonCode === "mo.md()") {
-      return true;
-    }
-
-    // Parse the code using Lezer and check for the exact match of mo.md() signature
-    const tree = pythonLanguage.parser.parse(pythonCode);
-
-    // This is the exact match of mo.md() signature
-    const enterOrder: { match: string | RegExp; stop?: boolean }[] = [
-      { match: "Script" },
-      { match: "ExpressionStatement" },
-      { match: "CallExpression" },
-      { match: "MemberExpression" },
-      { match: "VariableName" },
-      { match: "." },
-      { match: "PropertyName" },
-      { match: "ArgList" },
-      { match: "(" },
-      { match: /String|FormatString/, stop: true },
-      { match: ")" },
-    ];
-
-    let isValid = true;
-
-    // Parse the code using Lezer to check for multiple function calls and string content
-    tree.iterate({
-      enter: (node) => {
-        const current = enterOrder.shift();
-        if (current === undefined) {
-          // If our list is empty, but we are still going
-          // then this is not a valid call
-          isValid = false;
-          return false;
-        }
-
-        const match = current.match;
-
-        if (typeof match === "string") {
-          isValid = isValid && match === node.name;
-          return isValid && !current.stop;
-        }
-
-        if (!match.test(node.name)) {
-          isValid = false;
-          return isValid && !current.stop;
-        }
-
-        return isValid && !current.stop;
-      },
-    });
-
-    return isValid;
+    return this.parser.isSupported(pythonCode);
   }
 
   getExtension(