Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions frontend/src/components/editor/output/JsonOutput.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ const LEAF_RENDERERS = {
"video/": (value: string) => <VideoOutput src={value} />,
"text/html:": (value: string) => <HtmlOutput html={value} inline={true} />,
"text/plain+float:": (value: string) => <span>{value}</span>,
"text/plain+bigint:": (value: string) => <span>{value}</span>,
"text/plain+set:": (value: string) => <span>set{value}</span>,
"text/plain+tuple:": (value: string) => <span>{value}</span>,
"text/plain:": (value: string) => <CollapsibleTextOutput text={value} />,
Expand Down Expand Up @@ -359,6 +360,11 @@ function pythonJsonReplacer(_key: string, value: unknown): unknown {
if (value.startsWith("text/plain+float:")) {
return `${REPLACE_PREFIX}${leafData(value)}${REPLACE_SUFFIX}`;
}
if (value.startsWith("text/plain+bigint:")) {
// Use BigInt to avoid precision loss
const number = BigInt(leafData(value));
return `${REPLACE_PREFIX}${number}${REPLACE_SUFFIX}`;
}
if (value.startsWith("text/plain+tuple:")) {
// replace first and last characters [] with ()
return `${REPLACE_PREFIX}(${leafData(value).slice(1, -1)})${REPLACE_SUFFIX}`;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,29 @@ describe("getCopyValue", () => {
`,
);
});

it("should handle bigint", () => {
const bigint = 2 ** 64;
const value = `text/plain+bigint:${bigint}`;
const result = getCopyValue(value);
expect(result).toMatchInlineSnapshot(`"18446744073709552000"`);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this seems rounded 2 ** 64

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it should be 18446744073709551616

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oops, yeah, I think the test is inaccurate because 2 ** 64 is going to be rounded. let me think of a way


const nestedBigInt = {
key1: bigint,
key2: `text/plain+bigint:${bigint}`,
key3: true,
};
const nestedResult = getCopyValue(nestedBigInt);
expect(nestedResult).toMatchInlineSnapshot(
`
"{
"key1": 18446744073709552000,
"key2": 18446744073709552000,
"key3": True
}"
`,
);
});
});

describe("determineMaxDisplayLength", () => {
Expand Down
3 changes: 3 additions & 0 deletions frontend/src/components/editor/package-alert.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,7 @@ const ExtrasSelector: React.FC<ExtrasSelectorProps> = ({
<button
className="hover:bg-muted/50 rounded text-sm px-1 transition-colors border border-muted-foreground/30 hover:border-muted-foreground/60 min-w-0 flex-1 truncate text-left"
title={`Selected extras: ${selectedExtras.join(", ")}`}
type="button"
>
{selectedExtras.join(",")}
</button>
Expand Down Expand Up @@ -523,6 +524,7 @@ const ExtrasSelector: React.FC<ExtrasSelectorProps> = ({
!canSelectExtras && "opacity-50 cursor-not-allowed",
)}
title={canSelectExtras ? "Add extras" : "Loading extras..."}
type="button"
>
<PlusIcon className="w-3 h-3 shrink-0" />
</button>
Expand Down Expand Up @@ -619,6 +621,7 @@ const StreamingLogsViewer: React.FC<StreamingLogsViewerProps> = ({
<button
onClick={() => setIsExpanded(!isExpanded)}
className="flex items-center gap-2 text-sm font-medium text-muted-foreground hover:text-foreground transition-colors"
type="button"
>
{isExpanded ? (
<ChevronDownIcon className="w-4 h-4" />
Expand Down
2 changes: 2 additions & 0 deletions frontend/src/custom.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ interface JSON {
text: string,
reviver?: (this: any, key: string, value: any) => any,
): unknown;

rawJSON(value: string): unknown;
}

// Improve type inference for Array.filter with BooleanConstructor
Expand Down
13 changes: 13 additions & 0 deletions frontend/src/utils/__tests__/json-parser.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,19 @@ it("can fail to jsonParseWithSpecialChar", () => {
expect(jsonParseWithSpecialChar("[nan]")).toMatchInlineSnapshot("{}");
});

it("can parse bigInts", () => {
const bigint = JSON.stringify({ bigint: { $bigint: "123456" } });
expect(jsonParseWithSpecialChar(bigint)).toEqual({ bigint: BigInt(123_456) });

const arrayOfBigInts = JSON.stringify([{ $bigint: "123456" }]);
expect(jsonParseWithSpecialChar(arrayOfBigInts)).toEqual([BigInt(123_456)]);

const nestedBigInt = JSON.stringify({ bigint: [{ $bigint: "123456" }] });
expect(jsonParseWithSpecialChar(nestedBigInt)).toEqual({
bigint: [BigInt(123_456)],
});
});

it("can convert json to tsv", () => {
expect(jsonToTSV([])).toEqual("");

Expand Down
31 changes: 28 additions & 3 deletions frontend/src/utils/json/json-parser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,18 @@

import type { JsonString } from "./base64";

declare global {
interface BigInt {
toJSON(): unknown;
}
}

// Treat BigInts as numbers
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON#using_json_numbers
BigInt.prototype.toJSON = function () {
return JSON.rawJSON(this.toString());
};

/**
* Parse an attribute value as JSON.
* This also handles NaN, Infinity, and -Infinity.
Expand All @@ -12,7 +24,7 @@ export function jsonParseWithSpecialChar<T = unknown>(
// This regex handling is expensive and often not needed.
// We try to parse with JSON.parse first, and if that fails, we use the regex.
try {
return JSON.parse(value) as T;
return JSON.parse(value, (_key, value) => sanitizeBigInt(value)) as T;
} catch {
// Do nothing
}
Expand All @@ -36,7 +48,7 @@ export function jsonParseWithSpecialChar<T = unknown>(
);
return JSON.parse(value, (_key, v) => {
if (typeof v !== "string") {
return v;
return sanitizeBigInt(v);
}
if (v === `${CHAR}NaN${CHAR}`) {
return Number.NaN;
Expand All @@ -47,7 +59,7 @@ export function jsonParseWithSpecialChar<T = unknown>(
if (v === `${CHAR}-Infinity${CHAR}`) {
return Number.NEGATIVE_INFINITY;
}
return v;
return sanitizeBigInt(v);
}) as T;
} catch {
return {} as T;
Expand All @@ -63,3 +75,16 @@ export function jsonToTSV(json: Record<string, unknown>[]) {
const values = json.map((row) => keys.map((key) => row[key]).join("\t"));
return `${keys.join("\t")}\n${values.join("\n")}`;
}

/** Adapted from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt#use_within_json */
function sanitizeBigInt(value: unknown): unknown {
if (
value !== null &&
typeof value === "object" &&
"$bigint" in value &&
typeof value.$bigint === "string"
) {
return BigInt(value.$bigint);
}
return value;
}
25 changes: 17 additions & 8 deletions marimo/_output/data/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,20 +185,26 @@ def any_data(data: Union[str, bytes, io.BytesIO], ext: str) -> VirtualFile:
raise ValueError(f"Unsupported data type: {type(data)}")


# JavaScript's safe integer limits
MAX_SAFE_INTEGER = 9007199254740991
MIN_SAFE_INTEGER = -9007199254740991
BIGINT_KEY = "$bigint"


def is_bigint(value: int | float) -> bool:
return value > MAX_SAFE_INTEGER or value < MIN_SAFE_INTEGER


def sanitize_json_bigint(
data: Union[str, dict[str, Any], list[dict[str, Any]]],
) -> str:
"""Sanitize JSON bigint to a string.
"""Sanitize JSON big numbers to a string.

This is necessary because the frontend will round ints larger than
Number.MAX_SAFE_INTEGER to Number.MAX_SAFE_INTEGER.
"""
from json import dumps, loads

# JavaScript's safe integer limits
MAX_SAFE_INTEGER = 9007199254740991
MIN_SAFE_INTEGER = -9007199254740991

def convert_key(key: Any) -> Any:
# Keys must be str, int, float, bool, or None
if key is None:
Expand All @@ -212,9 +218,12 @@ def convert_bigint(obj: Any) -> Any:
return {convert_key(k): convert_bigint(v) for k, v in obj.items()} # type: ignore
elif isinstance(obj, list):
return [convert_bigint(item) for item in obj] # type: ignore
elif isinstance(obj, int) and (
obj > MAX_SAFE_INTEGER or obj < MIN_SAFE_INTEGER
):
elif isinstance(obj, int) and is_bigint(obj):
# If the value is outside the safe integer range, convert it to an object with a $bigint key
# Frontend will convert the object back to an integer.
return {BIGINT_KEY: str(obj)}
elif isinstance(obj, float) and is_bigint(obj):
# Decimals are not handled currently, we just convert them to strings.
return str(obj)
else:
return obj
Expand Down
3 changes: 3 additions & 0 deletions marimo/_output/formatters/structures.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from marimo._messaging.mimetypes import KnownMimeType
from marimo._output import formatting
from marimo._output.data.data import is_bigint
from marimo._output.formatters.formatter_factory import FormatterFactory
from marimo._output.formatters.repr_formatters import maybe_get_repr_formatter
from marimo._plugins.stateless.inspect import inspect
Expand All @@ -26,6 +27,8 @@ def _leaf_formatter(
if isinstance(value, str):
return value
if isinstance(value, int):
if is_bigint(value):
return f"text/plain+bigint:{value}"
return value
# floats are still converted to strings because JavaScript
# can't reliably distinguish between them (eg 1 and 1.0)
Expand Down
18 changes: 15 additions & 3 deletions marimo/_smoke_tests/tables/complex_types.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import marimo

__generated_with = "0.15.5"
__generated_with = "0.16.5"
app = marimo.App(width="medium")


Expand Down Expand Up @@ -110,11 +110,23 @@ def _(df):

@app.cell
def _(mo, pd, pl):
complex = [1 + 2j, 2 + 3j]

additional_types_pd = pd.DataFrame(
{"complex": [1 + 2j, 2 + 3j], "bigint": [2**64, 2**127]}
{
"complex": complex,
"bigint": [2**64, 2**127],
"list_big_ints": [[1253397962952480469], [1253397962952480469]],
"large_floats": [[125339796295248046.9], [-12533979629524804.69]],
}
)
additional_types_pl = pl.DataFrame(
{"complex": [1 + 2j, 2 + 3j], "bigint": [2**64, 2**65]}
{
"complex": complex,
"bigint": [2**64, -(2**65)],
"list_big_ints": [[1253397962952480469], [1253397962952480469]],
"large_floats": [[125339796295248046.9], [-12533979629524804.69]],
}
)
mo.vstack([additional_types_pd, additional_types_pl])
return
Expand Down
Loading
Loading