Use generic_open for unified compressed/uncompressed file handling

Replaced the custom logic for handling externally zipped NetCDF files
with Satpy's generic_open helper, which transparently supports both
compressed and uncompressed inputs. This simplifies the reader
implementation and removes compression-detection code.

Because generic_open uses a context manager and closes the file as soon
as the reader exits the `with` block, the dataset must now be fully
loaded immediately. The reader therefore calls `.compute()` on the
xarray dataset to ensure all data is available after the context closes.

Author: albertbrotzer

satpy/readers/hsaf_nc.py

@@ -24,32 +24,30 @@
 
 Currently, this reader supports the following products:
   * **H60/H60B** – Blended GEO/IR and LEO/MW instantaneous precipitation over
-  the full Meteosat (0°) disk.
+    the full Meteosat (0°) disk.
   * **H63** – Blended GEO/IR and LEO/MW instantaneous precipitation over the
     Meteosat IODC (Indian Ocean) disk.
   * **H90** – Accumulated Precipitation at ground by blended MW and IR IODC
     over the Meteosat IODC (Indian Ocean) disk.
 
 Notes:
-    - Externally compressed files with the ``.gz`` suffix are automatically
-      uncompressed to the same directory and deleted on reader close.
-    - The reader relies on area definitions provided in the accompanying YAML
-      configuration file.
-    - Supports ``rain_rate`` and ``q_ind`` datasets.
-    - Output coverage and resolution correspond to the MSG-SEVIRI grid.
+  * Externally compressed files (.gz) and uncompressed files are handled
+    transparently via Satpys ``generic_open``, without creating temporary
+    decompressed files on disk.
+  * The reader uses area definitions specified in the associated YAML
+    configuration file to determine projection, coverage, and resolution.
+  * Provides access to the ``rain_rate`` and ``q_ind`` datasets (including
+    accumulated variants where available).
 """
 
 import datetime as dt
-import gzip
 import logging
-import shutil
-import tempfile
-from pathlib import Path
 
 import xarray as xr
 
 from satpy.area import get_area_def
 from satpy.readers.core.file_handlers import BaseFileHandler
+from satpy.readers.core.utils import generic_open
 from satpy.utils import get_legacy_chunk_size
 
 LOG = logging.getLogger(__name__)
@@ -62,58 +60,6 @@
 
 CHUNK_SIZE = get_legacy_chunk_size()
 
-def gunzip(source, destination):
-    """Unzips an externally compressed HSAF file."""
-    with gzip.open(source) as s:
-        with open(destination, "wb") as d:
-            shutil.copyfileobj(s, d, 10 * 1024 * 1024)
-
-def create_named_empty_file(source):
-    """Create an empty file."""
-    # Use delete=False for cross-platform safety
-    tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".nc")
-    tmp.close()  # we only needed the filename
-    LOG.debug(f"Created temporary file {Path(tmp.name)} from {source}")
-    return Path(tmp.name)
-
-
-class HSAFFileWrapper:
-    """Wrapper for a H SAF NetCDF file for handling external compression if an external gzip layer exist."""
-
-    def __init__(self, filename):
-        """Opens the nc file and stores the nc data."""
-        self.filename = filename
-        self._tmp_file = None
-        self._compressed = Path(self.filename).suffix == ".gz"
-
-        if self._compressed:
-            self._tmp_file = create_named_empty_file(self.filename)
-            gunzip(self.filename, self._tmp_file)
-            self.filename = str(self._tmp_file)
-
-        # Open dataset
-        chunks = CHUNK_SIZE
-        LOG.debug(f"Opening HSAF file {self.filename}")
-        self.nc_data = xr.open_dataset(self.filename, decode_cf=True, chunks=chunks)
-
-
-    def close(self):
-        """Close the nc file and clean up temp file if needed."""
-        if self.nc_data is not None:
-            self.nc_data.close()
-            self.nc_data = None
-            LOG.debug("Closed NetCDF dataset")
-
-        if self._compressed and self._tmp_file is not None:
-            try:
-                self._tmp_file.unlink(missing_ok=True)
-                LOG.debug(f"Deleted temporary file {self._tmp_file}")
-            except Exception as e:
-                LOG.warning(f"Failed to delete temp file {self._tmp_file}: {e}")
-            finally:
-                self._tmp_file = None
-
-
 class HSAFNCFileHandler(BaseFileHandler):
     """Handle H SAF NetCDF files, with optional .gz external compression.
 
@@ -129,22 +75,19 @@ class HSAFNCFileHandler(BaseFileHandler):
     def __init__(self, filename, filename_info, filetype_info):
         """Create a wrapper to opens the nc file and store the nc data."""
         super().__init__(filename, filename_info, filetype_info)
-        self._wrapper = HSAFFileWrapper(filename)
         self._area_name = None
         self._lon_0 = None
-
-    def close(self):
-        """Clean up by closing the dataset."""
-        self._wrapper.close()
+        with generic_open(filename, mode="rb", compression="infer") as fp:
+            self._nc_data = xr.open_dataset(fp, chunks="auto", engine="h5netcdf").compute()
 
     def _get_global_attributes(self):
         """Create a dictionary of global attributes to be added to all datasets."""
         attributes = {
             "filename": self.filename,
             "start_time": self.start_time,
             "end_time": self.end_time,
-            "spacecraft_name": platform_translate.get(self._wrapper.nc_data.attrs["satellite_identifier"], "N/A"),
-            "platform_name": platform_translate.get(self._wrapper.nc_data.attrs["satellite_identifier"], "N/A"),
+            "spacecraft_name": platform_translate.get(self._nc_data.attrs["satellite_identifier"], "N/A"),
+            "platform_name": platform_translate.get(self._nc_data.attrs["satellite_identifier"], "N/A"),
         }
 
         return attributes
@@ -163,8 +106,8 @@ def get_dataset(self, dataset_id, dataset_info):
         """Get a dataset from the file."""
         # Get the variable
         var_name = dataset_info.get("file_key", dataset_id["name"])
-        LOG.debug(f"Getting dataset {var_name} from file {self._wrapper.filename}")
-        data = self._wrapper.nc_data[var_name]
+        LOG.debug(f"Getting dataset {var_name} from file {self.filename}")
+        data = self._nc_data[var_name]
 
         data = self._standarize_dims(data)
 
@@ -181,7 +124,7 @@ def get_dataset(self, dataset_id, dataset_info):
 
         data.attrs["resolution"] = self.filetype_info["resolution"]
         self._area_name = self.filetype_info["area"]
-        self._lon_0 = float(self._wrapper.nc_data.attrs["sub_satellite_longitude"].rstrip("f"))
+        self._lon_0 = float(self._nc_data.attrs["sub_satellite_longitude"].rstrip("f"))
 
         # Add global attributes which are shared across datasets
         data.attrs.update(self._get_global_attributes())

satpy/tests/reader_tests/test_hsaf_nc.py

@@ -1,6 +1,8 @@
 """Tests for the H-SAF NC reader."""
 import datetime as dt
+import io
 import os
+from collections import namedtuple
 from unittest import mock
 
 import numpy as np
@@ -35,6 +37,12 @@
     }
 }
 
+# Avoid too many arguments for test_load_datasets
+LoadDatasetsParams = namedtuple(
+    "LoadDatasetsParams",
+    ["file_type", "loadable_ids", "unit", "resolution", "area_name"]
+)
+
 # constants for fake test data
 DEFAULT_SHAPE = (5, 5)
 rng = np.random.default_rng()
@@ -78,48 +86,55 @@ def setup_method(self):
     )
     def test_reader_creation(self, file_type, expected_loadables):
         """Test that the reader can create file handlers."""
-        with mock.patch("satpy.readers.hsaf_nc.xr.open_dataset") as od:
-            od.side_effect = fake_hsaf_dataset
+        with mock.patch("satpy.readers.hsaf_nc.generic_open") as mock_generic_open, \
+                mock.patch("satpy.readers.hsaf_nc.xr.open_dataset") as mock_open_dataset:
+
+            mock_generic_open.return_value.__enter__.return_value = io.BytesIO(b"fake data")
+            mock_open_dataset.side_effect = fake_hsaf_dataset
 
             loadables = file_type["reader"].select_files_from_pathnames([file_type["fake_file"]])
-            file_type["reader"].create_filehandlers(loadables)
+            file_handlers = file_type["reader"].create_filehandlers(loadables)
 
-            assert len(loadables) == expected_loadables
+            assert len(file_handlers) == expected_loadables
             assert file_type["reader"].file_handlers, "No file handlers created"
 
     @pytest.mark.parametrize(
-        ("file_type", "loadable_ids", "unit", "resolution", "area_name"),
+        "params",
         [
-            (FILE_PARAMS[FILE_TYPE_H60], ["rr", "qind"], "mm/h", 3000, "msg_seviri_fes_3km"),
-            (FILE_PARAMS[FILE_TYPE_H63], ["rr", "qind"], "mm/h", 3000, "msg_seviri_iodc_3km"),
-            (FILE_PARAMS[FILE_TYPE_H90], ["acc_rr", "qind"], "mm", 3000, "msg_seviri_iodc_3km"),
+            LoadDatasetsParams(FILE_PARAMS[FILE_TYPE_H60], ["rr", "qind"], "mm/h", 3000, "msg_seviri_fes_3km"),
+            LoadDatasetsParams(FILE_PARAMS[FILE_TYPE_H63], ["rr", "qind"], "mm/h", 3000, "msg_seviri_iodc_3km"),
+            LoadDatasetsParams(FILE_PARAMS[FILE_TYPE_H90], ["acc_rr", "qind"], "mm", 3000, "msg_seviri_iodc_3km"),
         ],
     )
-    def test_load_datasets(self, file_type, loadable_ids, unit, resolution, area_name):
+    def test_load_datasets(self, params):
         """Test that datasets can be loaded correctly."""
-        with mock.patch("satpy.readers.hsaf_nc.xr.open_dataset") as od:
-            od.side_effect = fake_hsaf_dataset
-            loadables = file_type["reader"].select_files_from_pathnames([file_type["fake_file"]])
-            file_type["reader"].create_filehandlers(loadables)
+        with mock.patch("satpy.readers.hsaf_nc.generic_open") as mock_generic_open, \
+                mock.patch("satpy.readers.hsaf_nc.xr.open_dataset") as mock_open_dataset:
 
-            datasets = file_type["reader"].load(loadable_ids)
+            mock_generic_open.return_value.__enter__.return_value = io.BytesIO(b"fake data")
+            mock_open_dataset.side_effect = fake_hsaf_dataset
+
+            loadables = params.file_type["reader"].select_files_from_pathnames([params.file_type["fake_file"]])
+            params.file_type["reader"].create_filehandlers(loadables)
+
+            datasets = params.file_type["reader"].load(params.loadable_ids)
             dataset_names = {d["name"] for d in datasets.keys()}
-            assert dataset_names == set(loadable_ids)
+            assert dataset_names == set(params.loadable_ids)
 
             # check array shapes and types
-            assert datasets[loadable_ids[0]].shape == DEFAULT_SHAPE
-            assert datasets[loadable_ids[1]].shape == DEFAULT_SHAPE
-            assert np.issubdtype(datasets[loadable_ids[0]].dtype, np.floating)
-            assert np.issubdtype(datasets[loadable_ids[1]].dtype, np.integer)
+            assert datasets[params.loadable_ids[0]].shape == DEFAULT_SHAPE
+            assert datasets[params.loadable_ids[1]].shape == DEFAULT_SHAPE
+            assert np.issubdtype(datasets[params.loadable_ids[0]].dtype, np.floating)
+            assert np.issubdtype(datasets[params.loadable_ids[1]].dtype, np.integer)
 
-            data = datasets[loadable_ids[0]]
+            data = datasets[params.loadable_ids[0]]
             assert data.attrs["spacecraft_name"] == "Meteosat-8"
             assert data.attrs["platform_name"] == "Meteosat-8"
-            assert data.attrs["units"] == unit
-            assert data.attrs["resolution"] == resolution
+            assert data.attrs["units"] == params.unit
+            assert data.attrs["resolution"] == params.resolution
             assert data.attrs["start_time"] == dt.datetime(2025, 11, 5, 0, 0)
             assert data.attrs["end_time"] == dt.datetime(2025, 11, 5, 0, 15)
-            assert data.attrs["area"].area_id == area_name
+            assert data.attrs["area"].area_id == params.area_name
             assert data.dims == ("y", "x")
 
 
@@ -152,8 +167,12 @@ def test_real_hsaf_file(self, file_type, loadable_ids):
 
     def test_get_area_def(self):
         """Test that the loaded dataset has a AreaDefinition and overwrite of lon_0 of the area works correctly."""
-        with mock.patch("satpy.readers.hsaf_nc.xr.open_dataset") as od:
-            od.side_effect = fake_hsaf_dataset
+        with mock.patch("satpy.readers.hsaf_nc.generic_open") as mock_generic_open, \
+                mock.patch("satpy.readers.hsaf_nc.xr.open_dataset") as mock_open_dataset:
+
+            mock_generic_open.return_value.__enter__.return_value = io.BytesIO(b"fake data")
+            mock_open_dataset.side_effect = fake_hsaf_dataset
+
             file_type = FILE_PARAMS[FILE_TYPE_H63]
             loadables = file_type["reader"].select_files_from_pathnames([file_type["fake_file"]])
             file_type["reader"].create_filehandlers(loadables)