From 0d0a696fe206f58fa64a7d4f4a33bf8fe1ea0b24 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 23 Sep 2024 12:08:48 +0200 Subject: [PATCH 1/3] Update tile retreival ahn.py - On the eclipse server: Skip tile retreival if the link does not exist. - Legacy server: Rasterio merge returned only zero's since 17 sept. All the merging is now done within xarray. --- nlmod/read/ahn.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/nlmod/read/ahn.py b/nlmod/read/ahn.py index ec01c295..773f0b96 100644 --- a/nlmod/read/ahn.py +++ b/nlmod/read/ahn.py @@ -580,7 +580,9 @@ def _get_ahn_ellipsis(extent, identifier="AHN5_5M_M", **kwargs): das = [] for tile in tqdm(tiles.index, desc=f"Downloading tiles of {identifier}"): url = tiles.at[tile, identifier] - if url.endswith(".zip"): + if url == "nan": + continue + elif url.endswith(".zip"): path = url.split("/")[-1].replace(".zip", ".TIF") if path.lower().endswith(".tif.tif"): path = path[:-4] @@ -598,7 +600,7 @@ def _download_and_combine_tiles(tiles, identifier, extent, as_data_array): """Internal method to download and combine ahn-data.""" if tiles.empty: raise (Exception(f"{identifier} has no data for requested extent")) - datasets = [] + das = [] for name in tqdm(tiles.index, desc=f"Downloading tiles of {identifier}"): url = tiles.at[name, identifier] if isinstance(url, pd.Series): @@ -609,14 +611,15 @@ def _download_and_combine_tiles(tiles, identifier, extent, as_data_array): path = url.split("/")[-1].replace(".zip", ".TIF") if path.lower().endswith(".tif.tif"): path = path[:-4] - datasets.append(rasterio.open(f"zip+{url}!/{path}")) - memfile = MemoryFile() - merge.merge(datasets, dst_path=memfile) + das.append(xr.open_dataset(f"zip+{url}!/{path}")) + + dataarray = xr.merge(das)["band_data"] + if as_data_array: - da = rioxarray.open_rasterio(memfile.open(), mask_and_scale=True)[0] - da = da.sel(x=slice(extent[0], extent[1]), y=slice(extent[3], extent[2])) - return da - return memfile + return dataarray.sel( + x=slice(extent[0], extent[1]), y=slice(extent[3], extent[2])) + + return dataarray def _rename_identifier(identifier): From efd78034d4b3ee298fc67342712495770e0a719a Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 23 Sep 2024 12:14:16 +0200 Subject: [PATCH 2/3] Pleasing codacy --- nlmod/read/ahn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nlmod/read/ahn.py b/nlmod/read/ahn.py index 773f0b96..68a38352 100644 --- a/nlmod/read/ahn.py +++ b/nlmod/read/ahn.py @@ -582,7 +582,7 @@ def _get_ahn_ellipsis(extent, identifier="AHN5_5M_M", **kwargs): url = tiles.at[tile, identifier] if url == "nan": continue - elif url.endswith(".zip"): + if url.endswith(".zip"): path = url.split("/")[-1].replace(".zip", ".TIF") if path.lower().endswith(".tif.tif"): path = path[:-4] From bae990fe4fd3b04064dc30e47a09eec369ce4564 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Mon, 23 Sep 2024 16:08:32 +0200 Subject: [PATCH 3/3] Revert changes to legacy functions --- nlmod/read/ahn.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/nlmod/read/ahn.py b/nlmod/read/ahn.py index 68a38352..30575616 100644 --- a/nlmod/read/ahn.py +++ b/nlmod/read/ahn.py @@ -600,7 +600,7 @@ def _download_and_combine_tiles(tiles, identifier, extent, as_data_array): """Internal method to download and combine ahn-data.""" if tiles.empty: raise (Exception(f"{identifier} has no data for requested extent")) - das = [] + datasets = [] for name in tqdm(tiles.index, desc=f"Downloading tiles of {identifier}"): url = tiles.at[name, identifier] if isinstance(url, pd.Series): @@ -611,15 +611,14 @@ def _download_and_combine_tiles(tiles, identifier, extent, as_data_array): path = url.split("/")[-1].replace(".zip", ".TIF") if path.lower().endswith(".tif.tif"): path = path[:-4] - das.append(xr.open_dataset(f"zip+{url}!/{path}")) - - dataarray = xr.merge(das)["band_data"] - + datasets.append(rasterio.open(f"zip+{url}!/{path}")) + memfile = MemoryFile() + merge.merge(datasets, dst_path=memfile) if as_data_array: - return dataarray.sel( - x=slice(extent[0], extent[1]), y=slice(extent[3], extent[2])) - - return dataarray + da = rioxarray.open_rasterio(memfile.open(), mask_and_scale=True)[0] + da = da.sel(x=slice(extent[0], extent[1]), y=slice(extent[3], extent[2])) + return da + return memfile def _rename_identifier(identifier):