From 692dc48d7718406d35ae9d13bc67c070dbb5f90f Mon Sep 17 00:00:00 2001 From: Ulises M Date: Thu, 23 Jan 2025 21:57:55 -0800 Subject: [PATCH 1/3] compress graph data to support pako endpoint --- haystack/core/pipeline/draw.py | 13 +++++++++---- .../compress-mermaid-graph-bf23918c3da6e018.yaml | 4 ++++ 2 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 releasenotes/notes/compress-mermaid-graph-bf23918c3da6e018.yaml diff --git a/haystack/core/pipeline/draw.py b/haystack/core/pipeline/draw.py index 2e24bf9acd..d785be602b 100644 --- a/haystack/core/pipeline/draw.py +++ b/haystack/core/pipeline/draw.py @@ -3,6 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 import base64 +import json +import zlib import networkx # type:ignore import requests @@ -68,11 +70,14 @@ def _to_mermaid_image(graph: networkx.MultiDiGraph): """ # Copy the graph to avoid modifying the original graph_styled = _to_mermaid_text(graph.copy()) + json_string = json.dumps({"code": graph_styled}) - graphbytes = graph_styled.encode("ascii") - base64_bytes = base64.b64encode(graphbytes) - base64_string = base64_bytes.decode("ascii") - url = f"https://mermaid.ink/img/{base64_string}?type=png" + # Uses the DEFLATE algorithm at the highest level for smallest size + compressor = zlib.compressobj(level=9) + compressed_data = compressor.compress(json_string.encode("utf-8")) + compressor.flush() + url_safe_base64 = base64.urlsafe_b64encode(compressed_data).decode("utf-8").strip() + + url = f"https://mermaid.ink/img/pako:{url_safe_base64}?type=png" logger.debug("Rendering graph at {url}", url=url) try: diff --git a/releasenotes/notes/compress-mermaid-graph-bf23918c3da6e018.yaml b/releasenotes/notes/compress-mermaid-graph-bf23918c3da6e018.yaml new file mode 100644 index 0000000000..2b2d531924 --- /dev/null +++ b/releasenotes/notes/compress-mermaid-graph-bf23918c3da6e018.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Haystack pipelines with Mermaid graphs are now compressed to reduce the size of the encoded base64 and avoid HTTP 400 errors when the graph is too large. From 7d1c9d72953ccdd5ac6c7b6aabe647324b981394 Mon Sep 17 00:00:00 2001 From: Ulises M <30765968+lbux@users.noreply.github.com> Date: Mon, 27 Jan 2025 09:47:05 -0800 Subject: [PATCH 2/3] Update haystack/core/pipeline/draw.py Co-authored-by: David S. Batista --- haystack/core/pipeline/draw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haystack/core/pipeline/draw.py b/haystack/core/pipeline/draw.py index d785be602b..bf9b1c1809 100644 --- a/haystack/core/pipeline/draw.py +++ b/haystack/core/pipeline/draw.py @@ -75,7 +75,7 @@ def _to_mermaid_image(graph: networkx.MultiDiGraph): # Uses the DEFLATE algorithm at the highest level for smallest size compressor = zlib.compressobj(level=9) compressed_data = compressor.compress(json_string.encode("utf-8")) + compressor.flush() - url_safe_base64 = base64.urlsafe_b64encode(compressed_data).decode("utf-8").strip() + compressed_url_safe_base64 = base64.urlsafe_b64encode(compressed_data).decode("utf-8").strip() url = f"https://mermaid.ink/img/pako:{url_safe_base64}?type=png" From 4516ad8940a492389151db6e735c277e47fee622 Mon Sep 17 00:00:00 2001 From: Ulises M <30765968+lbux@users.noreply.github.com> Date: Mon, 27 Jan 2025 09:47:12 -0800 Subject: [PATCH 3/3] Update haystack/core/pipeline/draw.py Co-authored-by: David S. Batista --- haystack/core/pipeline/draw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haystack/core/pipeline/draw.py b/haystack/core/pipeline/draw.py index bf9b1c1809..b367696d84 100644 --- a/haystack/core/pipeline/draw.py +++ b/haystack/core/pipeline/draw.py @@ -77,7 +77,7 @@ def _to_mermaid_image(graph: networkx.MultiDiGraph): compressed_data = compressor.compress(json_string.encode("utf-8")) + compressor.flush() compressed_url_safe_base64 = base64.urlsafe_b64encode(compressed_data).decode("utf-8").strip() - url = f"https://mermaid.ink/img/pako:{url_safe_base64}?type=png" + url = f"https://mermaid.ink/img/pako:{compressed_url_safe_base64}?type=png" logger.debug("Rendering graph at {url}", url=url) try: