From cba84e89a40b80cf05181c5e2edf20d5ee16f80a Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Fri, 28 Jan 2022 16:16:50 -0500 Subject: [PATCH] Don't persist before computing offset/limits --- dask_sql/physical/rel/logical/limit.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dask_sql/physical/rel/logical/limit.py b/dask_sql/physical/rel/logical/limit.py index 468ac6f0d..a6f6f8c1c 100644 --- a/dask_sql/physical/rel/logical/limit.py +++ b/dask_sql/physical/rel/logical/limit.py @@ -58,7 +58,6 @@ def _apply_offset(self, df: dd.DataFrame, offset: int, end: int) -> dd.DataFrame we need to pass the partition number to the selection function, which is not possible with normal "map_partitions". """ - df = df.persist() if not offset: # We do a (hopefully) very quick check: if the first partition # is already enough, we will just use this