feat: support DataFrames via narwhals (#1570)

Co-authored-by: Barret Schloerke <[email protected]>

Author: machow

.github/workflows/pytest.yaml

@@ -79,7 +79,6 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install https://github.com/rstudio/py-htmltools/tarball/main
           make install-deps
           make install
       - name: "Build Package"

CHANGELOG.md

@@ -15,8 +15,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### New features
 
+* Added [narwhals](https://posit-dev.github.io/py-narwhals) support for `@render.data_frame`. This allows for any eager data frame supported by narwhals to be returned from a `@render.data_frame` output method. All internal methods and helper methods leverage the `narwhals` API to be data frame agnostic. (#1570)
+
 ### Other changes
 
+* Incorporated `orjson` for faster data serialization in `@render.data_frame` outputs. (#1570)
+
 * Added `PageNavbar` class to the list of `shiny.playwright.controllers` for testing `ui.page_navbar()`. (#1668)
 
 * Added `.expect_widths()` to `NavsetPillList` in `shiny.playwright.controllers` for testing `ui.navset_pill_list(widths=)`. (#1668)
@@ -27,15 +31,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 * Small improvements to the default pulse busy indicator to better blend with any background. It's also now slightly smaller by default.(#1707)
 
+* Added [narwhals](https://posit-dev.github.io/py-narwhals) support for `@render.table`. This allows for any eager data frame supported by narwhals to be returned from a `@render.table` output method. (#1570)
+
 ### Bug fixes
 
 * A few fixes for `ui.Chat()`, including:
   * Fixed a bug with `Chat()` sometimes silently dropping errors. (#1672)
   * Fixed a bug with `Chat()` sometimes not removing it's loading icon (on error or a `None` transform). (#1679)
   * `.messages(format="anthropic")` correctly removes non-user starting messages (once again). (#1685)
 
-* `shiny create` now uses the template `id` rather than the directory name as the default directory.
-(#1666)
+* `shiny create` now uses the template `id` rather than the directory name as the default directory. (#1666)
 
 * `ui.Theme()` now works correctly on Windows when the theme requires Sass compilation. (thanks @yuuuxt, #1684)
 
@@ -45,11 +50,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 * Fixed input controller `InputTextArea` in `shiny.playwright.controller` to correctly validate the `resize` style property in `.expect_resize()`. (#1705)
 
-
 * Fixed a bug in `ui.conditional_panel()` that would cause the panel to repeatedly show/hide itself when the provided condition did not evaluate to a boolean value. (#1707)
 
 * Fixed a bug with `ui.input_slider()` when used as a range slider that made it impossible to change the slider value when both handles were at the maximum value. (#1707)
 
+* Fixed bug in `@render.data_frame` where `bool` or `object` columns were not being rendered. (#1570)
+
 
 ## [1.1.0] - 2024-09-03
 

Makefile

@@ -216,17 +216,14 @@ ci-install-wheel: dist FORCE
 install-deps: FORCE ## install dependencies
 	pip install -e ".[dev,test]" --upgrade
 ci-install-deps: FORCE
-	uv pip install "htmltools @ git+https://github.com/posit-dev/py-htmltools.git"
 	uv pip install -e ".[dev,test]"
 
 install-docs: FORCE
 	pip install -e ".[dev,test,doc]"
-	pip install https://github.com/posit-dev/py-htmltools/tarball/main
 	pip install https://github.com/posit-dev/py-shinylive/tarball/main
 ci-install-docs: FORCE
-	uv pip install -e ".[dev,test,doc]"
-	uv pip install "htmltools @ git+https://github.com/posit-dev/py-htmltools.git" \
-			"shinylive @ git+https://github.com/posit-dev/py-shinylive.git"
+	uv pip install -e ".[dev,test,doc]" \
+		"shinylive @ git+https://github.com/posit-dev/py-shinylive.git"
 
 ci-install-rsconnect: FORCE
 	uv pip install "rsconnect-python @ git+https://github.com/rstudio/rsconnect-python.git"

js/data-frame/cell.tsx

@@ -62,9 +62,9 @@ const isShinyHtml = (x: any): x is CellHtmlValue => {
 };
 type CellValue = string | CellHtmlValue | null;
 const getCellValueText = (cellValue: CellValue) => {
-  if (isShinyHtml(cellValue)) return cellValue.obj.html;
   if (cellValue === null) return "";
-  return cellValue as string;
+  if (isShinyHtml(cellValue)) return cellValue.obj.html;
+  return cellValue;
 };
 
 interface TableBodyCellProps {
@@ -472,6 +472,7 @@ export const TableBodyCell: FC<TableBodyCellProps> = ({
 
     // TODO-future; Use faster way to make a deep copy
     const cellValueObjDeepCopy = JSON.parse(JSON.stringify(cellValue.obj));
+    // Render the Shiny content asynchronously to the table's cell
     window.Shiny.renderContentAsync(tdRef.current, cellValueObjDeepCopy);
 
     const curTdRef = tdRef.current;

js/data-frame/index.tsx

@@ -115,6 +115,7 @@ const ShinyDataGrid: FC<ShinyDataGridProps<unknown>> = ({
       fill: false,
       styles: [],
     },
+    htmlDeps,
   } = payload;
   const {
     width,
@@ -197,7 +198,36 @@ const ShinyDataGrid: FC<ShinyDataGridProps<unknown>> = ({
             typeHint,
           },
           cell: ({ getValue }) => {
-            return getValue() as string;
+            const ret = getValue();
+
+            // Regardless of type, if the value is null or undefined,
+            // return an empty string
+            if (ret === null || ret === undefined) {
+              return "";
+            }
+            switch (typeHint?.type) {
+              // Return the value as is
+              case "numeric":
+              case "date":
+              case "datetime":
+              case "duration":
+              case "categorical":
+              case "html":
+                return ret;
+              // Convert the value to a string
+              case "string":
+              case "boolean":
+                return String(ret);
+              // Convert the value to a JSON string if it isn't a string already
+              case "unknown":
+              case "object":
+                if (typeof ret === "string") {
+                  return ret;
+                }
+                return JSON.stringify(ret);
+              default:
+                return ret;
+            }
           },
           enableSorting,
         };
@@ -427,6 +457,12 @@ const ShinyDataGrid: FC<ShinyDataGridProps<unknown>> = ({
     };
   }, [id, selection, tableData]);
 
+  useEffect(() => {
+    if (!htmlDeps) return;
+    // Register the Shiny HtmlDependencies
+    Shiny.renderDependenciesAsync([...htmlDeps]);
+  }, [htmlDeps]);
+
   useEffect(() => {
     const handleColumnSort = (
       event: CustomEvent<{ sort: { col: number; desc: boolean }[] }>

js/data-frame/types.ts

@@ -1,5 +1,7 @@
 import { StyleInfo } from "./style-info";
 
+import type { HtmlDep } from "rstudio-shiny/srcts/types/src/shiny/render";
+
 export type ValueOf<T> = T[keyof T];
 
 export const EditModeEnum = {
@@ -9,7 +11,17 @@ export const EditModeEnum = {
 export type EditMode = ValueOf<typeof EditModeEnum>;
 
 export interface TypeHint {
-  type: "string" | "numeric" | "categorical" | "unknown" | "html";
+  type:
+    | "string"
+    | "numeric"
+    | "boolean"
+    | "date"
+    | "datetime"
+    | "duration"
+    | "object"
+    | "unknown"
+    | "html"
+    | "categorical";
 }
 
 export interface CategoricalTypeHint extends TypeHint {
@@ -34,6 +46,7 @@ export interface PandasData<TIndex> {
   data: unknown[][];
   options: DataGridOptions;
   typeHints?: ReadonlyArray<TypeHint>;
+  htmlDeps?: ReadonlyArray<HtmlDep>;
 }
 
 export interface PatchInfo {

pyproject.toml

@@ -33,7 +33,8 @@ dependencies = [
     "starlette",
     "websockets>=10.0",
     "python-multipart",
-    "htmltools>=0.5.2",
+    # "htmltools>=0.5.3.9002",
+    "htmltools@git+https://github.com/posit-dev/py-htmltools@main",
     "click>=8.1.4;platform_system!='Emscripten'",
     "markdown-it-py>=1.1.0",
     "mdit-py-plugins>=0.3.0",
@@ -46,6 +47,8 @@ dependencies = [
     "prompt-toolkit;platform_system!='Emscripten'",
     "python-multipart>=0.0.7;platform_system!='Emscripten'",
     "setuptools;python_version>='3.12'",
+    "narwhals>=1.9.0,<1.10.0",
+    "orjson>=3.10.7",
 ]
 
 [project.optional-dependencies]

shiny/_deprecated.py

@@ -23,8 +23,8 @@ class ShinyDeprecationWarning(RuntimeWarning):
 warnings.simplefilter("always", ShinyDeprecationWarning)
 
 
-def warn_deprecated(message: str):
-    warnings.warn(message, ShinyDeprecationWarning, stacklevel=3)
+def warn_deprecated(message: str, stacklevel: int = 3):
+    warnings.warn(message, ShinyDeprecationWarning, stacklevel=stacklevel)
 
 
 def render_text():

shiny/render/__init__.py

@@ -15,7 +15,6 @@
 from ._data_frame_utils._selection import CellSelection
 from ._data_frame_utils._types import (  # noqa: F401
     StyleInfo,
-    DataFrameLikeT as _DataFrameLikeT,  # pyright: ignore[reportUnusedImport]
 )
 from ._deprecated import (  # noqa: F401
     RenderFunction,  # pyright: ignore[reportUnusedImport]