Adding open_groups to BackendEntryPointEngine, NetCDF4BackendEntrypoint, and H5netcdfBackendEntrypoint

xarray/backends/api.py

@@ -11,6 +11,7 @@
     Callable,
     Final,
     Literal,
+    Optional,
     Union,
     cast,
     overload,
@@ -1720,3 +1721,44 @@ def to_zarr(
         return dask.delayed(_finalize_store)(writes, zstore)
 
     return zstore
+
+
+def open_groups(
+    filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+    engine: T_Engine = None,
+    group: Optional[str] = None,
+    **kwargs,
+) -> dict[str, Dataset]:
+    """
+    Open and decode a file or file-like object, creating a dictionary containing one xarray Dataset for each group in the file.
+
+    Useful when you have e.g. a netCDF file containing many groups, some of which are not alignable with their parents and so the file cannot be opened directly with ``open_datatree``.
+
+    It is encouraged to use this function to inspect your data, then make the necessary changes to make the structure coercible to a `DataTree` object before calling `DataTree.from_dict()` and proceeding with your analysis.
+
+    Parameters
+    ----------
+    filename_or_obj : str, Path, file-like, or DataStore
+        Strings and Path objects are interpreted as a path to a netCDF file or Zarr store.
+    engine : str, optional
+        Xarray backend engine to use. Valid options include `{"netcdf4", "h5netcdf", "zarr"}`.
+    group : str, optional
+        Group to use as the root group to start reading from. Groups above this root group will not be included in the output.
+    **kwargs : dict
+        Additional keyword arguments passed to :py:func:`~xarray.open_dataset` for each group.
+
+    Returns
+    -------
+    dict[str, xarray.Dataset]
+
+    See Also
+    --------
+    open_datatree()
+    DataTree.from_dict()
+    """
+    if engine is None:
+        engine = plugins.guess_engine(filename_or_obj)
+
+    backend = plugins.get_backend(engine)
+
+    return backend.open_groups(filename_or_obj, **kwargs)

xarray/backends/common.py

@@ -535,6 +535,17 @@ def open_datatree(
 
         raise NotImplementedError()
 
+    def open_groups(
+        self,
+        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+        **kwargs: Any,
+    ) -> dict[str, Dataset]:
+        """
+        Backend open_groups method used by Xarray in :py:func:`~xarray.open_groups`.
+        """
+
+        raise NotImplementedError()
+
 
 # mapping of engine name to (module name, BackendEntrypoint Class)
 BACKEND_ENTRYPOINTS: dict[str, tuple[str | None, type[BackendEntrypoint]]] = {}

xarray/backends/h5netcdf_.py

@@ -441,9 +441,9 @@ def open_datatree(
         group: str | Iterable[str] | Callable | None = None,
         **kwargs,
     ) -> DataTree:
-        from xarray.backends.api import open_dataset
+        # TODO: Add function docstring
+
         from xarray.backends.common import _iter_nc_groups
-        from xarray.core.datatree import DataTree
         from xarray.core.treenode import NodePath
         from xarray.core.utils import close_on_error
 
@@ -452,19 +452,20 @@ def open_datatree(
             filename_or_obj,
             group=group,
         )
+        # Check for a group and make it a parent if it exists
         if group:
             parent = NodePath("/") / NodePath(group)
         else:
             parent = NodePath("/")
 
         manager = store._manager
-        ds = open_dataset(store, **kwargs)
-        tree_root = DataTree.from_dict({str(parent): ds})
+
+        groups_dict = {}
         for path_group in _iter_nc_groups(store.ds, parent=parent):
             group_store = H5NetCDFStore(manager, group=path_group, **kwargs)
             store_entrypoint = StoreBackendEntrypoint()
             with close_on_error(group_store):
-                ds = store_entrypoint.open_dataset(
+                group_ds = store_entrypoint.open_dataset(
                     group_store,
                     mask_and_scale=mask_and_scale,
                     decode_times=decode_times,
@@ -474,14 +475,65 @@ def open_datatree(
                     use_cftime=use_cftime,
                     decode_timedelta=decode_timedelta,
                 )
-                new_node: DataTree = DataTree(name=NodePath(path_group).name, data=ds)
-                tree_root._set_item(
-                    path_group,
-                    new_node,
-                    allow_overwrite=False,
-                    new_nodes_along_path=True,
+
+            group_name = NodePath(path_group).name
+            groups_dict[group_name] = group_ds
+
+        return DataTree.from_dict(groups_dict)
+
+    def open_groups(
+        self,
+        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+        *,
+        mask_and_scale=True,
+        decode_times=True,
+        concat_characters=True,
+        decode_coords=True,
+        drop_variables: str | Iterable[str] | None = None,
+        use_cftime=None,
+        decode_timedelta=None,
+        group: str | Iterable[str] | Callable | None = None,
+        **kwargs,
+    ) -> dict[str, Dataset]:
+        # TODO: Add function docstring
+
+        from xarray.backends.common import _iter_nc_groups
+        from xarray.core.treenode import NodePath
+        from xarray.core.utils import close_on_error
+
+        filename_or_obj = _normalize_path(filename_or_obj)
+        store = H5NetCDFStore.open(
+            filename_or_obj,
+            group=group,
+        )
+        # Check for a group and make it a parent if it exists
+        if group:
+            parent = NodePath("/") / NodePath(group)
+        else:
+            parent = NodePath("/")
+
+        manager = store._manager
+
+        groups_dict = {}
+        for path_group in _iter_nc_groups(store.ds, parent=parent):
+            group_store = H5NetCDFStore(manager, group=path_group, **kwargs)
+            store_entrypoint = StoreBackendEntrypoint()
+            with close_on_error(group_store):
+                group_ds = store_entrypoint.open_dataset(
+                    group_store,
+                    mask_and_scale=mask_and_scale,
+                    decode_times=decode_times,
+                    concat_characters=concat_characters,
+                    decode_coords=decode_coords,
+                    drop_variables=drop_variables,
+                    use_cftime=use_cftime,
+                    decode_timedelta=decode_timedelta,
                 )
-        return tree_root
+
+            group_name = NodePath(path_group).name
+            groups_dict[group_name] = group_ds
+
+        return groups_dict
 
 
 BACKEND_ENTRYPOINTS["h5netcdf"] = ("h5netcdf", H5netcdfBackendEntrypoint)

xarray/backends/netCDF4_.py

@@ -682,7 +682,8 @@ def open_datatree(
         group: str | Iterable[str] | Callable | None = None,
         **kwargs,
     ) -> DataTree:
-        from xarray.backends.api import open_dataset
+        # TODO: Add function docstring
+
         from xarray.backends.common import _iter_nc_groups
         from xarray.core.datatree import DataTree
         from xarray.core.treenode import NodePath
@@ -692,19 +693,21 @@ def open_datatree(
             filename_or_obj,
             group=group,
         )
+
+        # Check for a group and make it a parent if it exists
         if group:
             parent = NodePath("/") / NodePath(group)
         else:
             parent = NodePath("/")
 
         manager = store._manager
-        ds = open_dataset(store, **kwargs)
-        tree_root = DataTree.from_dict({str(parent): ds})
+        groups_dict = {}
+
         for path_group in _iter_nc_groups(store.ds, parent=parent):
             group_store = NetCDF4DataStore(manager, group=path_group, **kwargs)
             store_entrypoint = StoreBackendEntrypoint()
             with close_on_error(group_store):
-                ds = store_entrypoint.open_dataset(
+                group_ds = store_entrypoint.open_dataset(
                     group_store,
                     mask_and_scale=mask_and_scale,
                     decode_times=decode_times,
@@ -714,14 +717,65 @@ def open_datatree(
                     use_cftime=use_cftime,
                     decode_timedelta=decode_timedelta,
                 )
-                new_node: DataTree = DataTree(name=NodePath(path_group).name, data=ds)
-                tree_root._set_item(
-                    path_group,
-                    new_node,
-                    allow_overwrite=False,
-                    new_nodes_along_path=True,
+
+            group_name = NodePath(path_group).name
+            groups_dict[group_name] = group_ds
+
+        return DataTree.from_dict(groups_dict)
+
+    def open_groups(
+        self,
+        filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
+        *,
+        mask_and_scale=True,
+        decode_times=True,
+        concat_characters=True,
+        decode_coords=True,
+        drop_variables: str | Iterable[str] | None = None,
+        use_cftime=None,
+        decode_timedelta=None,
+        group: str | Iterable[str] | Callable | None = None,
+        **kwargs,
+    ) -> dict:
+        # TODO: Add function docstring
+
+        from xarray.backends.common import _iter_nc_groups
+        from xarray.core.treenode import NodePath
+
+        filename_or_obj = _normalize_path(filename_or_obj)
+        store = NetCDF4DataStore.open(
+            filename_or_obj,
+            group=group,
+        )
+
+        # Check for a group and make it a parent if it exists
+        if group:
+            parent = NodePath("/") / NodePath(group)
+        else:
+            parent = NodePath("/")
+
+        manager = store._manager
+
+        groups_dict = {}
+        for path_group in _iter_nc_groups(store.ds, parent=parent):
+            group_store = NetCDF4DataStore(manager, group=path_group, **kwargs)
+            store_entrypoint = StoreBackendEntrypoint()
+            with close_on_error(group_store):
+                group_ds = store_entrypoint.open_dataset(
+                    group_store,
+                    mask_and_scale=mask_and_scale,
+                    decode_times=decode_times,
+                    concat_characters=concat_characters,
+                    decode_coords=decode_coords,
+                    drop_variables=drop_variables,
+                    use_cftime=use_cftime,
+                    decode_timedelta=decode_timedelta,
                 )
-        return tree_root
+
+            group_name = NodePath(path_group).name
+            groups_dict[group_name] = group_ds
+
+        return groups_dict
 
 
 BACKEND_ENTRYPOINTS["netcdf4"] = ("netCDF4", NetCDF4BackendEntrypoint)

xarray/backends/plugins.py

@@ -202,6 +202,7 @@ def get_backend(engine: str | type[BackendEntrypoint]) -> BackendEntrypoint:
     if isinstance(engine, str):
         engines = list_engines()
         if engine not in engines:
+            # TODO: Improve error handing message
             raise ValueError(
                 f"unrecognized engine {engine} must be one of: {list(engines)}"
             )