diff --git a/doxygen/aliases b/doxygen/aliases index 0b4908694ad..7d33009efcc 100644 --- a/doxygen/aliases +++ b/doxygen/aliases @@ -392,7 +392,7 @@ ALIASES += indexes=" -
#H5_INDEX_NAMELexicographic order on ALIASES += link_types="
#H5L_TYPE_HARDHard link
#H5L_TYPE_SOFTSoft link
#H5L_TYPE_EXTERNALExternal link
#H5L_TYPE_ERRORError
" ALIASES += mem_types="
#H5FD_MEM_DEFAULTThe default file memory allocation type
#H5FD_MEM_SUPERFile memory allocated for Superblock
#H5FD_MEM_BTREEFile memory allocated for B-tree
#H5FD_MEM_DRAWFile memory allocated for raw data
#H5FD_MEM_GHEAPFile memory allocated for Global Heap
#H5FD_MEM_LHEAPFile memory allocated for Local Heap
#H5FD_MEM_OHDRFile memory allocated for Object Header
" ALIASES += norm_type="
#H5T_NORM_IMPLIED0MSB of mantissa is not stored, always 1
#H5T_NORM_MSBSET1MSB of mantissa is always 1
#H5T_NORM_NONE2Mantissa is not normalized
" -ALIASES += obj_types="
#H5F_OBJ_FILEFiles only
#H5F_OBJ_DATASETDatasets only
#H5F_OBJ_GROUPGroups only
#H5F_OBJ_DATATYPEDatatypes only. If \p file_id is #H5F_OBJ_ALL, this will count committed and transient datatypes. Otherwise, it will only count committed datatypes.
#H5F_OBJ_ATTRAttributes only
#H5F_OBJ_ALLAll of the above
#H5F_OBJ_LOCALRestrict search to objects opened through current file identifier.
\attention #H5F_OBJ_LOCAL does not stand alone; it is effective only when used in combination with one or more of the preceding types. For example, #H5F_OBJ_DATASET | #H5F_OBJ_GROUP | #H5F_OBJ_LOCAL would count all datasets and groups opened through the current file identifier." +ALIASES += obj_types="
#H5F_OBJ_FILEFiles only
#H5F_OBJ_DATASETDatasets only
#H5F_OBJ_GROUPGroups only
#H5F_OBJ_DATATYPEDatatypes only. If \p file_id is #H5F_OBJ_ALL, this will count committed and transient datatypes. Otherwise, it will only count committed datatypes.
#H5F_OBJ_ATTRAttributes only
#H5F_OBJ_ALLAll of the above
#H5F_OBJ_LOCALRestrict search to objects opened through current file identifier.
\attention #H5F_OBJ_LOCAL does not stand alone; it is effective only when used in combination with one or more of the preceding types. For example, #H5F_OBJ_DATASET | #H5F_OBJ_GROUP | #H5F_OBJ_LOCAL would count all datasets and groups opened through the current file identifier." ALIASES += orders="
#H5_ITER_INCIncreasing order
#H5_ITER_DECDecreasing order
#H5_ITER_NATIVEFastest available order
" ALIASES += padding_type="
#H5T_PAD_ZERO0Set background to zeros
#H5T_PAD_ONE1Set background to ones
#H5T_PAD_BACKGROUND2Leave background alone
" ALIASES += scopes="
#H5F_SCOPE_GLOBALFlushes the entire virtual file
#H5F_SCOPE_LOCALFlushes only the specified file
" diff --git a/doxygen/dox/CollectiveCalls.dox b/doxygen/dox/CollectiveCalls.dox index 7e6d4756ddc..31f75000cc3 100644 --- a/doxygen/dox/CollectiveCalls.dox +++ b/doxygen/dox/CollectiveCalls.dox @@ -930,7 +930,6 @@ #H5Topen1 at Release 1.8.0.
* * \section sec_collective_calls_props Properties diff --git a/doxygen/dox/CollectiveMetadataIO.dox b/doxygen/dox/CollectiveMetadataIO.dox index 958025b6493..1d3080661cd 100644 --- a/doxygen/dox/CollectiveMetadataIO.dox +++ b/doxygen/dox/CollectiveMetadataIO.dox @@ -239,7 +239,6 @@ * * #H5Zunregister * - * * * * In addition, most deprecated functions fall into this category. diff --git a/doxygen/dox/FileFormatDisc.dox b/doxygen/dox/FileFormatDisc.dox index 66c39107b62..87abc63dfa3 100644 --- a/doxygen/dox/FileFormatDisc.dox +++ b/doxygen/dox/FileFormatDisc.dox @@ -849,7 +849,7 @@ Symbol 0: Flags: 0x01 - Reserved: 0 + Reserved: 0 diff --git a/doxygen/dox/H5.format.1.0.dox b/doxygen/dox/H5.format.1.0.dox index 9e3d3d8d169..82f5a18b421 100644 --- a/doxygen/dox/H5.format.1.0.dox +++ b/doxygen/dox/H5.format.1.0.dox @@ -873,7 +873,7 @@ as an atomic object. Version Reserved - + Collection Size diff --git a/doxygen/dox/H5.format.1.1.dox b/doxygen/dox/H5.format.1.1.dox index 80c5e7ddc42..d4d57b86af2 100644 --- a/doxygen/dox/H5.format.1.1.dox +++ b/doxygen/dox/H5.format.1.1.dox @@ -1107,7 +1107,7 @@ variable-length datatype elements, etc). Version Reserved - + Collection SizeL diff --git a/doxygen/dox/H5.format.3.0.dox b/doxygen/dox/H5.format.3.0.dox index 353b0b4f64a..d6ac4a6893b 100644 --- a/doxygen/dox/H5.format.3.0.dox +++ b/doxygen/dox/H5.format.3.0.dox @@ -7995,7 +7995,6 @@ information for the virtual layout class as well as indexing information for the Implicit index Address of the array of dataset chunks. - Fixed Array index @@ -8004,7 +8003,6 @@ information for the virtual layout class as well as indexing information for the Extensible Array index Address of the index. - Version 2 B-tree index @@ -8111,6 +8109,7 @@ The following information exists only when the chunk is filtered. In other words Min Pointers Min Elements + Page Bits This space inserted only to align table nicely @@ -8194,7 +8193,7 @@ The following information exists only when the chunk is filtered. In other words byte byte byte - <>byte + byte
AddressO

@@ -12465,7 +12464,6 @@ Reference Block stored in the global heap:
Dataspace Selection Information (variable size)

- Length of Attribute Name diff --git a/doxygen/dox/H5.format.4.0.dox b/doxygen/dox/H5.format.4.0.dox index fa0243abbc1..ba3695170c1 100644 --- a/doxygen/dox/H5.format.4.0.dox +++ b/doxygen/dox/H5.format.4.0.dox @@ -8165,7 +8165,6 @@ information for the virtual layout class as well as indexing information for the Implicit index Address of the array of dataset chunks. - Fixed Array index @@ -8174,7 +8173,6 @@ information for the virtual layout class as well as indexing information for the Extensible Array index Address of the index. - Version 2 B-tree index @@ -8281,6 +8279,7 @@ The following information exists only when the chunk is filtered. In other words Min Pointers Min Elements + Page Bits This space inserted only to align table nicely @@ -8364,7 +8363,7 @@ The following information exists only when the chunk is filtered. In other words byte byte byte - <>byte + byte
AddressO

@@ -12710,7 +12709,6 @@ Reference Block stored in the global heap:
Dataspace Selection Information (variable size)

- Length of Attribute Name diff --git a/doxygen/dox/UsersGuide.dox b/doxygen/dox/UsersGuide.dox index ead8709831d..a9500290b01 100644 --- a/doxygen/dox/UsersGuide.dox +++ b/doxygen/dox/UsersGuide.dox @@ -113,6 +113,15 @@ Navigate back: \ref index "Main" \li \ref subsec_group_examples +\ref sec_link +\li \ref subsec_link_intro +\li \ref subsec_link_types +\li \ref subsec_link_create +\li \ref subsec_link_ops +\li \ref subsec_link_traverse +\li \ref subsec_link_user +\li \ref subsec_link_summary + \ref sec_dataset \li \ref subsec_dataset_intro \li \ref subsec_dataset_function @@ -228,6 +237,25 @@ Navigate back: \ref index "Main" \li \ref subsec_attribute_special +\ref sec_object +\li \ref subsec_object_intro +\li \ref subsec_object_ops +\li \ref subsec_object_visit +\li \ref subsec_object_comments +\li \ref subsec_object_refcount +\li \ref subsec_object_cache +\li \ref subsec_object_summary + +\ref sec_identifier +\li \ref subsec_identifier_intro +\li \ref subsec_identifier_types +\li \ref subsec_identifier_refcount +\li \ref subsec_identifier_valid +\li \ref subsec_identifier_query +\li \ref subsec_identifier_user +\li \ref subsec_identifier_iteration +\li \ref subsec_identifier_summary + \ref sec_error \li \ref subsec_error_intro \li \ref subsec_error_program @@ -266,6 +294,17 @@ Navigate back: \ref index "Main" \li \ref subsec_plist_resources \li \ref subsec_plist_notes +\ref sec_vfd +\li \ref subsec_vfd_intro +\li \ref subsec_vfd_purpose +\li \ref subsec_vfd_drivers +\li \ref subsec_vfd_selection +\li \ref subsec_vfd_custom +\li \ref subsec_vfd_parallel +\li \ref subsec_vfd_performance +\li \ref subsec_vfd_query +\li \ref subsec_vfd_summary + \ref sec_vol \li \ref subsec_vol_intro \li \ref subsec_vol_abstract_layer @@ -338,10 +377,24 @@ Navigate back: \ref index "Main" \li \ref subsec_async_intro \ref sec_map +\li \ref subsec_map_intro +\li \ref subsec_map_lifecycle +\li \ref subsec_map_plist +\li \ref subsec_map_datatypes +\li \ref subsec_map_example +\li \ref subsec_map_notes \ref sec_reference -\ref sec_filter_plugins +\ref sec_filter +\li \ref subsec_filter_intro +\li \ref subsec_filter_builtin +\li \ref subsec_filter_usage +\li \ref subsec_filter_pipeline +\li \ref subsec_filter_custom +\li \ref subsec_filter_query +\li \ref subsec_filter_plugins +\li \ref subsec_filter_summary \section sec_hl HDF5 High Level APIs \li \ref H5DO_UG diff --git a/doxygen/dox/VFLTechNote.dox b/doxygen/dox/VFLTechNote.dox index 3438cae66a7..6f56e6bbafe 100644 --- a/doxygen/dox/VFLTechNote.dox +++ b/doxygen/dox/VFLTechNote.dox @@ -521,7 +521,6 @@ The library makes many types of allocation requests: #H5FD_MEM_DRAWAn allocation request for the raw data of a dataset. - #H5FD_MEM_GHEAPAn allocation request for a global heap collection. Global heaps are used to store certain types of references such as dataset region references. diff --git a/doxygen/dox/ViewTools2.dox b/doxygen/dox/ViewTools2.dox index 92f0ba2b564..2c5f6a8a28b 100644 --- a/doxygen/dox/ViewTools2.dox +++ b/doxygen/dox/ViewTools2.dox @@ -159,7 +159,7 @@ you will see a detailed usage statement with examples of modifying a filter. The numerous filters that you can apply to a dataset: -Options +
FilterFilterOptions
GZIP compression (levels 1-9) diff --git a/doxygen/dox/hdf5_1_8.dox b/doxygen/dox/hdf5_1_8.dox index 5f4fc80c5bd..5eb173242e0 100644 --- a/doxygen/dox/hdf5_1_8.dox +++ b/doxygen/dox/hdf5_1_8.dox @@ -54,12 +54,13 @@ This section lists interface-level changes and other user-visible changes in beh \subsubsection subsubsec_rel_spec_18_change_22_new New and Changed Functions, Classes, Subroutines, Wrappers, and Macros

In the C Interface (main library)

The following are new C functions in this release: - +
FunctionDescription
+ -
FunctionDescription
#H5Pget_fapl_hdfsGets the information of the given Read-Only HDFS virtual file driver
#H5Pget_fapl_ros3Gets the information of the given Read-Only S3 virtual file driver
#H5Pset_fapl_hdfsSets up Read-Only HDFS virtual file driver
#H5Pset_fapl_ros3Sets up Read-Only S3 virtual file driver
+

In the C High Level Interface

\li None. @@ -225,13 +226,14 @@ The \#ifdef VLPT_REMOVED blocks have been removed from the packet t Changed shared library interface version numbers (soname) For HDF5 Release 1.8.17, the shared object version numbers have changed as follows. - +
LibraryRelease 1.8.16Release 1.8.17Reason for Change
+ -
LibraryRelease 1.8.16Release 1.8.17Reason for Change
HDF5 C Library10.1.010.2.0Added APIs
HDF5 High-level C Library10.0.210.1.0Added APIs
C++ Library11.0.012.0.0Changes in existing APIs
High-level C++ Library11.0.011.1.0Added APIs
All others10.0.210.0.3Code changes that did not result in interface changes
+

Compatibility report for Release 1.8.17 versus Release 1.8.16

The following interface compatibility report provides a full list of the changed symbols: @@ -274,12 +276,13 @@ This function reports whether the linked version of the HDF5 Library was built w and #H5Pset_attr_creation_order/#H5Pget_attr_creation_order
The following wrappers were added to the class H5::ObjCreatPropList for the respective C functions: - +
C FunctionC++ Wrapper
+ -
C FunctionC++ Wrapper
#H5Pset_attr_phase_changeH5::ObjCreatPropList::setAttrPhaseChange
#H5Pget_attr_phase_changeH5::ObjCreatPropList::getAttrPhaseChange
#H5Pset_attr_creation_orderH5::ObjCreatPropList::setAttrCrtOrder
#H5Pget_attr_creation_orderH5::ObjCreatPropList::getAttrCrtOrder
+

Library Configuration

\li Use of thread-safety with the HDF5 High-level Library has been declared unsupported in @@ -316,12 +319,13 @@ changes specific to each wrapper library.

Changed shared library interface version numbers (soname)

For HDF5 Release 1.8.16, the shared object version numbers have changed as follows. - +
LibraryRelease 1.8.15Release 1.8.16Reason for Change
+ -
LibraryRelease 1.8.15Release 1.8.16Reason for Change
HDF5 Library10.0.110.1.0Added APIs
C++ Library10.0.111.0.0Changes in existing APIs
High-level C++ Library10.0.111.0.0Changes in existing APIs
All others10.0.110.0.2Code changes that did not result in interface changes
+

Compatibility report for Release 1.8.16 versus Release 1.8.15

The following interface compatibility report provides a full list of the changed symbols: diff --git a/doxygen/examples/tables/predefinedDatatypes.dox b/doxygen/examples/tables/predefinedDatatypes.dox index 4ec666dea08..fefb8a349cb 100644 --- a/doxygen/examples/tables/predefinedDatatypes.dox +++ b/doxygen/examples/tables/predefinedDatatypes.dox @@ -254,7 +254,6 @@ #H5T_C_S1 String datatype in C (size defined in bytes rather than in bits) - #H5T_FORTRAN_S1 String datatype in Fortran (as defined for the HDF5 C library) diff --git a/hl/src/H5DOpublic.h b/hl/src/H5DOpublic.h index 97ee719d7cb..11e7cf4fef7 100644 --- a/hl/src/H5DOpublic.h +++ b/hl/src/H5DOpublic.h @@ -22,6 +22,13 @@ extern "C" { * Navigate back: \ref index "Main" / \ref UG *
* + * \section sec_hldo_intro Introduction + * + * The HDF5 Dataset Optimization (H5DO) interface provides high-performance functions for specialized + * dataset I/O operations that bypass standard HDF5 processing layers when appropriate. + * + * @see H5DO Reference Manual + * * Since version 1.10.3 these functions are deprecated in favor of #H5Dwrite_chunk and #H5Dread_chunk. * * \section sec_hldo_direct_chunk Direct Chunk Write Function diff --git a/hl/src/H5DSpublic.h b/hl/src/H5DSpublic.h index 15b76a83920..b727c479c78 100644 --- a/hl/src/H5DSpublic.h +++ b/hl/src/H5DSpublic.h @@ -526,15 +526,11 @@ extern "C" { * * These relationships are represented in the file by attributes of the Dataset D and the Dimension * Scale Datasets. Figure 5 shows the values that are stored for the DIMENSION_LIST attribute of - * Dataset D. This - * - * - * - * - *
+ * Dataset D. + * * \image html UML_Attribute.jpg "The UML model for an HDF5 attribute" - *
- * attribute is a one-dimensional array with the HDF5 datatype variable length + * + * This attribute is a one-dimensional array with the HDF5 datatype variable length * #H5T_STD_REF_OBJ. Each row of the array is zero or more object references for Dimension Scale datasets. * * Table 6 shows the DIMENSION_LABELLIST for Dataset D. This is a one dimensional array with some empty @@ -559,30 +555,11 @@ extern "C" { * an item from the table, because the entries at both ends of the association must be updated * at the same time. * - * - * - * - * - *
* \image html H5DS_fig3.png "Figure 3. Example dataset and scales." - *
* - * - * - * - * - *
* \image html H5DS_fig4.png "Figure 4. Example labels, names, and attached scales." - *
* - * - * - * - * - *
- * \image html H5DS_fig5.png "Figure 5. The table of dimension references, stored as an attribute of the - * Dataset." - *
+ * \image html H5DS_fig5.png "Figure 5. The table of dimension references." * * * @@ -750,10 +727,12 @@ extern "C" { * 3. For each scale, detach the Dimension Scale S from dimension j of Dataset D with H5DSdetach_scale * 4. Delete the Dataset, with H5Gunlink. * - * \subsection subsec_dim_scales_api_func Programming API: H5DS - * @see H5DS Reference Manual + * \subsection subsec_dim_scales_ops Dimension Scale Operations * - * @todo Under Construction + * Attach dimension scales to datasets using #H5DSattach_scale and detach with #H5DSdetach_scale. + * Set and retrieve dimension scale names with #H5DSset_scale and #H5DSget_scale_name. Query + * if a dataset is a dimension scale with #H5DSis_scale and check attachments with #H5DSis_attached. + * Iterate through scales with #H5DSiterate_scales. * * Previous Chapter \ref sec_hldo_direct_chunk - Next Chapter \ref sec_hl_images * diff --git a/hl/src/H5IMpublic.h b/hl/src/H5IMpublic.h index 01b11744c9d..7cad53aa5b1 100644 --- a/hl/src/H5IMpublic.h +++ b/hl/src/H5IMpublic.h @@ -24,7 +24,20 @@ extern "C" { * * \section sec_hl_images HDF5 Images * - * @todo Under Construction + * \subsection subsec_hl_images_intro Introduction + * + * The HDF5 Image API (H5IM) provides functions for storing and retrieving image data + * in HDF5 files. The API supports common image formats and includes functions for + * working with image attributes such as palettes. + * + * @see H5IM Reference Manual + * + * \subsection subsec_hl_images_ops Image Operations + * + * Create images with #H5IMmake_image_8bit or #H5IMmake_image_24bit, and read them with + * #H5IMread_image. Check if a dataset is an image using #H5IMis_image. Get image + * information with #H5IMget_image_info. Work with palettes using #H5IMmake_palette, + * #H5IMlink_palette, #H5IMget_palette_info, and #H5IMget_palette. * * Previous Chapter \ref sec_dim_scales_stand - Next Chapter \ref sec_hl_lite_api * diff --git a/hl/src/H5LTpublic.h b/hl/src/H5LTpublic.h index 991000c9793..c6099ac38a0 100644 --- a/hl/src/H5LTpublic.h +++ b/hl/src/H5LTpublic.h @@ -41,7 +41,30 @@ extern "C" { * * \section sec_hl_lite_api HDF5 Lite APIs * - * @todo Under Construction + * \subsection subsec_hl_lite_intro Introduction + * + * The HDF5 Lite API (H5LT) provides simplified functions for common HDF5 operations, combining + * multiple low-level calls into single high-level functions. This reduces code complexity and + * makes HDF5 more accessible for straightforward data storage tasks. + * + * @see H5LT Reference Manual + * + * \subsection subsec_hl_lite_dataset Dataset Operations + * + * Create and access datasets with single function calls using type-specific functions like + * #H5LTmake_dataset_int, #H5LTmake_dataset_float, #H5LTread_dataset_int, etc. Query dataset + * properties with #H5LTget_dataset_ndims, #H5LTget_dataset_info, and #H5LTfind_dataset. + * + * \subsection subsec_hl_lite_attr Attribute Operations + * + * Set and read attributes using type-specific functions such as #H5LTset_attribute_int, + * #H5LTset_attribute_string, #H5LTget_attribute_int, etc. Query attribute information + * with #H5LTget_attribute_ndims and #H5LTget_attribute_info. + * + * \subsection subsec_hl_lite_util Utility Functions + * + * Convert between datatypes and text representations using #H5LTtext_to_dtype and #H5LTdtype_to_text. + * Validate paths with #H5LTpath_valid, and work with file images using #H5LTopen_file_image. * * Previous Chapter \ref sec_hl_images - Next Chapter \ref sec_hl_table_api * diff --git a/hl/src/H5PTpublic.h b/hl/src/H5PTpublic.h index 5ce19afe60a..8c1381ed9ae 100644 --- a/hl/src/H5PTpublic.h +++ b/hl/src/H5PTpublic.h @@ -22,9 +22,21 @@ extern "C" { * Navigate back: \ref index "Main" / \ref UG *
* - * \section sec_hl_packet_table_api HDF5 Table APIs + * \section sec_hl_packet_table_api HDF5 Packet Table APIs * - * @todo Under Construction + * \subsection subsec_hl_packet_intro Introduction + * + * The HDF5 Packet Table API (H5PT) provides an interface for storing and retrieving + * fixed-length packets of data in append-only datasets. Packet tables are optimized + * for streaming data collection where records are added sequentially. + * + * @see H5PT Reference Manual + * + * \subsection subsec_hl_packet_ops Packet Table Operations + * + * Create packet tables with #H5PTcreate or open existing ones with #H5PTopen. Append + * packets using #H5PTappend and read packets with #H5PTread_packets. Get the number + * of packets with #H5PTget_num_packets. Close with #H5PTclose. * * Previous Chapter \ref sec_hl_table_api * diff --git a/hl/src/H5TBpublic.h b/hl/src/H5TBpublic.h index 5b8c8cea578..7879712b361 100644 --- a/hl/src/H5TBpublic.h +++ b/hl/src/H5TBpublic.h @@ -24,7 +24,20 @@ extern "C" { * * \section sec_hl_table_api HDF5 Table APIs * - * @todo Under Construction + * \subsection subsec_hl_table_intro Introduction + * + * The HDF5 Table API (H5TB) provides functions for creating and manipulating HDF5 datasets + * as tables with named fields, similar to database tables or spreadsheets. Tables organize + * data in rows and columns, making them ideal for storing structured records. + * + * @see H5TB Reference Manual + * + * \subsection subsec_hl_table_ops Table Operations + * + * Create tables with #H5TBmake_table, append rows with #H5TBappend_records, write data with + * #H5TBwrite_records, and read with #H5TBread_records. Insert or delete records using + * #H5TBinsert_record and #H5TBdelete_record. Add or delete fields with #H5TBinsert_field + * and #H5TBdelete_field. Query table properties with #H5TBget_table_info and #H5TBget_field_info. * * Previous Chapter \ref sec_hl_lite_api - Next Chapter \ref sec_hl_packet_table_api * diff --git a/src/H5ESmodule.h b/src/H5ESmodule.h index 661f253360f..c7e1bec928d 100644 --- a/src/H5ESmodule.h +++ b/src/H5ESmodule.h @@ -30,11 +30,9 @@ * Navigate back: \ref index "Main" / \ref UG *
* - * @todo Under Construction - * * \section sec_async The HDF5 Event Set Interface * - * \section subsec_async_intro Introduction + * \subsection subsec_async_intro Introduction * HDF5 provides asynchronous APIs for the HDF5 VOL connectors that support asynchronous HDF5 * operations using the HDF5 Event Set (H5ES) API. This allows I/O to proceed in the background * while the application is performing other tasks. @@ -51,7 +49,7 @@ * * * - * * * @@ -82,7 +80,7 @@ * * * - * * * @@ -92,7 +90,7 @@ * * * - * * *
Table 6. The table of dimension labels.
H5F#H5Fcreate, #H5Fflush, #H5Fis_accessible, #H5Fopen, #H5Fclose + * #H5Fcreate, #H5Fflush, #H5Fopen, #H5Fclose *
H5R#H5Ropen_attr, #H5Ropen_object #H5Ropen_region, #H5Rdereference + * #H5Ropen_attr, #H5Ropen_object, #H5Ropen_region *
H5T#H5Tcommit, #H5Topen, #H5Tcopy, #H5Tclose + * #H5Tcommit, #H5Topen, #H5Tclose *
diff --git a/src/H5Emodule.h b/src/H5Emodule.h index 4c5d2aab7c0..87996c250b0 100644 --- a/src/H5Emodule.h +++ b/src/H5Emodule.h @@ -64,7 +64,51 @@ * see @ref H5E reference manual * * \subsection subsec_error_program Programming Model for Error Handling - * This section is under construction. + * + * The HDF5 error handling programming model provides a structured approach for detecting, reporting, + * and managing errors in applications. The model is based on error stacks that accumulate contextual + * information as errors propagate through the library call stack. + * + * \subsubsection subsubsec_error_program_model Basic Programming Model + * + * Applications typically use one of two approaches for error handling: + * + * \li \Bold{Automatic Error Handling}: By default, HDF5 automatically prints error information to + * stderr when errors occur. This is suitable for development and debugging but may not be + * appropriate for production applications. + * + * \li \Bold{Custom Error Handling}: Applications can register custom error handling functions using + * #H5Eset_auto to gain full control over error reporting and recovery. Custom handlers can log + * errors, display messages to users, or implement application-specific recovery strategies. + * + * \subsubsection subsubsec_error_program_detection Error Detection + * + * HDF5 functions indicate errors through their return values: + * \li Functions returning \c hid_t (identifiers) return a negative value on error + * \li Functions returning \c herr_t (error codes) return a negative value on error + * \li Functions returning pointers return NULL on error + * \li Functions returning \c htri_t (tri-state) return negative on error, zero for false, positive for true + * + * Applications should check return values from all HDF5 API calls to detect errors. + * + * \subsubsection subsubsec_error_program_handling Error Handling Strategies + * + * Common error handling strategies include: + * + * \li \Bold{Print and Continue}: Use default error handling to print diagnostic information and + * continue execution where possible. This is the default behavior. + * + * \li \Bold{Silent Error Handling}: Disable automatic error printing with #H5Eset_auto(H5E_DEFAULT, NULL, + *NULL) and check return values explicitly. This is common in production code. + * + * \li \Bold{Custom Error Handler}: Install a custom error handler to implement application-specific + * error logging, user notification, or recovery procedures. + * + * \li \Bold{Error Stack Inspection}: Walk the error stack with #H5Ewalk to examine detailed error + * information and make programmatic decisions based on specific error conditions. + * + * @see \ref subsec_error_ops for details on error stack operations and \ref subsec_error_adv for + * advanced error handling features including custom error classes and messages. * * \subsection subsec_error_ops Basic Error Handling Operations * Let us first try to understand the error stack. An error stack is a collection of error records. Error diff --git a/src/H5FDlog.h b/src/H5FDlog.h index d4779d682c0..d643f99797a 100644 --- a/src/H5FDlog.h +++ b/src/H5FDlog.h @@ -252,7 +252,6 @@ H5_DLLVAR hid_t H5FD_LOG_id_g; * Table2: Logging Output * * FlagVFD CallOutput and Comments - * * * * #H5FD_LOG_LOC_READ @@ -395,7 +394,6 @@ H5_DLLVAR hid_t H5FD_LOG_id_g; * See also: #H5FD_LOG_LOC_READ * * - * * * #H5FD_LOG_TIME_WRITE * Close, Write @@ -448,7 +446,6 @@ H5_DLLVAR hid_t H5FD_LOG_id_g; * Table3: Flavors of logged data * * FlavorDescription - * * * * #H5FD_MEM_NOLIST diff --git a/src/H5FDmodule.h b/src/H5FDmodule.h index 2930f19b882..508a8bbd97f 100644 --- a/src/H5FDmodule.h +++ b/src/H5FDmodule.h @@ -24,8 +24,189 @@ #define H5FD_MODULE #define H5_MY_PKG H5FD #define H5_MY_PKG_INIT YES + +/** \page H5FD_UG HDF5 Virtual File Drivers + * + * Navigate back: \ref index "Main" / \ref UG + *
+ * + * \section sec_vfd The HDF5 Virtual File Driver Interface + * + * \subsection subsec_vfd_intro Introduction + * + * The HDF5 Virtual File Driver (VFD) interface provides an abstraction layer for file I/O operations, + * enabling HDF5 to work with different file storage mechanisms. The VFD layer intercepts all low-level + * file access operations and forwards them to a specific driver implementation, allowing HDF5 files + * to be stored in various ways beyond simple POSIX files. + * + * @see H5FD Reference Manual + * + * \subsection subsec_vfd_purpose Purpose and Benefits + * + * The Virtual File Driver interface serves several important purposes: + * + * \li \Bold{Storage Flexibility}: Enables HDF5 to work with different storage backends including + * local files, parallel file systems, memory, and cloud storage. + * + * \li \Bold{Performance Optimization}: Allows selection of file drivers optimized for specific + * computing environments and I/O patterns. + * + * \li \Bold{Parallel I/O}: Provides support for MPI-based parallel I/O through specialized drivers. + * + * \li \Bold{Custom Storage}: Enables development of custom file drivers for specialized storage + * requirements. + * + * \subsection subsec_vfd_drivers Built-in File Drivers + * + * HDF5 includes several standard Virtual File Drivers: + * + * \li \Bold{SEC2 Driver}: The default POSIX I/O driver using standard system calls like read() and + * write(). Suitable for most serial applications on local file systems. Set with #H5Pset_fapl_sec2. + * + * \li \Bold{STDIO Driver}: Uses buffered I/O from the C standard library (fread/fwrite). May provide + * better performance for some applications. Set with #H5Pset_fapl_stdio. + * + * \li \Bold{Core Driver}: Stores the HDF5 file entirely in memory, with optional backing store to disk. + * Provides fastest I/O for temporary files or small datasets. Set with #H5Pset_fapl_core. + * + * \li \Bold{Family Driver}: Splits a logical HDF5 file across multiple physical files of equal size. + * Useful for circumventing file system limitations. Set with #H5Pset_fapl_family. + * + * \li \Bold{Multi Driver}: Stores different types of HDF5 data in separate files (metadata, raw data, + * etc.). Can optimize I/O by placing different data types on different storage devices. Set with + * #H5Pset_fapl_multi. + * + * \li \Bold{Split Driver}: A simplified version of the Multi driver that separates metadata and raw + * data into two files. Set with #H5Pset_fapl_split. + * + * \li \Bold{Log Driver}: Wraps another driver and logs all file access operations. Useful for + * debugging and I/O profiling. Set with #H5Pset_fapl_log. + * + * \li \Bold{MPI-IO Driver}: Enables parallel I/O using MPI-IO for HPC applications. Required for + * parallel HDF5 operations. Set with #H5Pset_fapl_mpio. + * + * \li \Bold{Subfiling Driver}: A parallel I/O driver that improves parallel I/O performance on parallel + * file systems by splitting the logical HDF5 file into multiple subfiles distributed across I/O + * concentrator nodes. Reduces contention and improves scalability for large-scale parallel + * applications. Set with #H5Pset_fapl_subfiling. + * + * \li \Bold{Direct Driver}: Uses direct I/O (O_DIRECT) to bypass OS caching. Can improve performance + * for large sequential I/O. Set with #H5Pset_fapl_direct. + * + * \li \Bold{Onion Driver}: Provides revision control for HDF5 files by storing file modifications + * as separate revisions. Enables tracking changes over time and accessing previous versions. + * Set with #H5Pset_fapl_onion. + * + * \li \Bold{Splitter Driver}: Writes file operations simultaneously to two different channels using + * different VFDs. Useful for creating redundant copies or logging I/O to separate locations. + * Set with #H5Pset_fapl_splitter. + * + * \li \Bold{Mirror Driver}: Mirrors all file operations to a remote server in real-time over a + * network connection. Enables remote backup and replication scenarios. Set with + * #H5Pset_fapl_mirror. + * + * \li \Bold{ROS3 Driver}: Read-only driver for accessing HDF5 files in S3-compatible object storage. + * Set with #H5Pset_fapl_ros3. + * + * \li \Bold{HDFS Driver}: Read-only driver for accessing HDF5 files in Hadoop Distributed File System. + * Set with #H5Pset_fapl_hdfs. + * + * \subsection subsec_vfd_selection Selecting a File Driver + * + * File drivers are selected through the file access property list when opening or creating a file. + * The basic pattern is: + * + * \li Create a file access property list with #H5Pcreate + * \li Set the desired file driver using the appropriate H5Pset_fapl_* function + * \li Pass the property list to #H5Fcreate or #H5Fopen + * + * \subsection subsec_vfd_custom Custom File Drivers + * + * Applications can implement custom file drivers by: + * + * \li Defining a #H5FD_class_t structure with function pointers for all required operations + * \li Implementing the driver callbacks (open, close, read, write, etc.) + * \li Registering the driver with #H5FDregister + * \li Setting the driver in a file access property list with #H5Pset_driver + * + * Custom drivers enable specialized I/O strategies such as: + * \li Integration with custom storage systems + * \li Transparent encryption or compression at the I/O layer + * \li Specialized caching strategies + * \li Network-based storage protocols + * + * \subsection subsec_vfd_parallel Parallel File Drivers + * + * For parallel HDF5 applications, the MPI-IO file driver is required (see \ref IntroParHDF5 for + * details on parallel HDF5 programming). This driver coordinates file access across multiple + * MPI processes, enabling collective I/O operations and preventing conflicts. Parallel applications must: + * + * \li Build HDF5 with parallel support enabled + * \li Use the MPI-IO file driver via #H5Pset_fapl_mpio + * \li Provide MPI communicator and info objects + * \li Coordinate file access across processes + * + * The Subfiling driver provides additional performance benefits for + * large-scale parallel applications on parallel file systems. It works by: + * + * \li Distributing the HDF5 file across multiple subfiles + * \li Designating I/O concentrator processes (typically one per node) + * \li Striping data across subfiles to reduce contention + * \li Enabling better parallel I/O scaling on Lustre, GPFS, and similar file systems + * + * The Subfiling driver is particularly beneficial when running at scale on parallel file systems + * where a single shared file can become a bottleneck. + * + * \subsection subsec_vfd_performance Performance Considerations + * + * Choosing the right file driver can significantly impact I/O performance: + * + * \li \Bold{Local Files}: SEC2 or STDIO drivers typically provide good performance + * \li \Bold{Temporary Data}: Core driver provides fastest access by avoiding disk I/O + * \li \Bold{Large Files}: Family driver can work around file size limitations + * \li \Bold{Parallel Applications}: MPI-IO driver required for coordinated parallel access + * \li \Bold{Large-Scale Parallel}: Subfiling driver can dramatically improve performance on shared + * parallel file systems by reducing metadata contention and enabling better striping + * \li \Bold{Network Storage}: Consider drivers optimized for network protocols + * \li \Bold{Redundancy}: Splitter or Mirror drivers enable real-time backup and replication + * \li \Bold{Versioning}: Onion driver enables tracking file revisions for provenance and rollback + * + * \subsection subsec_vfd_query Querying File Driver Information + * + * Applications can query the current file driver: + * + * \li #H5Pget_driver retrieves the driver identifier from a file access property list + * \li #H5FDis_driver_registered_by_name checks if a specific driver is available by name + * \li #H5FDis_driver_registered_by_value checks if a specific driver is available by value + * \li Driver-specific property list functions retrieve driver parameters + * + * \subsection subsec_vfd_summary Summary + * + * The HDF5 Virtual File Driver interface provides: + * \li Abstraction of file I/O operations for flexibility and portability + * \li Multiple built-in drivers for common storage scenarios + * \li Support for parallel I/O via MPI-IO + * \li Extensibility through custom driver implementation + * \li Performance optimization opportunities through driver selection + * + * Proper selection and configuration of file drivers is essential for optimal HDF5 + * performance in different computing environments. + * + *
+ * Navigate back: \ref index "Main" / \ref UG + */ + /** + * \defgroup H5FD File Drivers (H5FD) + * + * Use the functions in this module to manage HDF5 Virtual File Drivers (VFDs). + * + * Virtual File Drivers (VFDs) provide an abstraction layer for file I/O, enabling HDF5 to work + * with different storage mechanisms. VFDs can be selected to optimize performance for specific + * environments or to enable specialized storage backends. + * * \defgroup H5VFD Virtual File Driver Features + * \ingroup H5FD * */ diff --git a/src/H5FDsubfiling/H5FDioc.c b/src/H5FDsubfiling/H5FDioc.c index f178b45ac6f..c9717592eff 100644 --- a/src/H5FDsubfiling/H5FDioc.c +++ b/src/H5FDsubfiling/H5FDioc.c @@ -12,7 +12,7 @@ /* * Purpose: The IOC VFD implements a file driver which relays all the - * VFD calls to an underlying VFD, and send all the write calls to + * VFD calls to an underlying VFD, and sends all the write calls to * another underlying VFD. Maintains two files simultaneously. */ @@ -43,7 +43,7 @@ static bool H5FD_mpi_self_initialized_s = false; /* Pointer to value for MPI_TAG_UB */ int *H5FD_IOC_tag_ub_val_ptr = NULL; -/* The information of this ioc */ +/* The information of this IOC */ typedef struct H5FD_ioc_t { H5FD_t pub; /* public stuff, must be first */ int fd; /* the filesystem file descriptor */ @@ -305,7 +305,7 @@ H5FD__ioc_term(void) * Function: H5Pset_fapl_ioc * * Purpose: Sets the file access property list to use the - * ioc driver. + * IOC driver. * * Return: SUCCEED/FAIL *------------------------------------------------------------------------- @@ -347,7 +347,7 @@ H5Pset_fapl_ioc(hid_t fapl_id, H5FD_ioc_config_t *vfd_config) /*------------------------------------------------------------------------- * Function: H5Pget_fapl_ioc * - * Purpose: Returns information about the ioc file access property + * Purpose: Returns information about the IOC file access property * list through the structure config_out. * * Will fail if config_out is received without pre-set valid diff --git a/src/H5Gmodule.h b/src/H5Gmodule.h index dc1fe453e97..3d529200b04 100644 --- a/src/H5Gmodule.h +++ b/src/H5Gmodule.h @@ -415,7 +415,6 @@ * #H5Lcreate_ud * Creates a link of a user-defined type. * - * * * #H5Lget_val * Returns the value of a symbolic link. Replaces H5Gget_linkval. diff --git a/src/H5Imodule.h b/src/H5Imodule.h index 91335b843c1..35110a057c1 100644 --- a/src/H5Imodule.h +++ b/src/H5Imodule.h @@ -30,7 +30,96 @@ * Navigate back: \ref index "Main" / \ref UG *
* - * @todo Under Construction + * \section sec_identifier The HDF5 Identifier Interface + * + * \subsection subsec_identifier_intro Introduction + * + * The HDF5 Identifier interface (H5I) manages identifiers (type #hid_t) that serve as handles + * to HDF5 objects and resources. Identifiers provide an abstraction layer between applications + * and the internal HDF5 library structures, enabling safe and efficient object management. + * + * Every HDF5 object—file, group, dataset, datatype, dataspace, attribute, property list—is + * accessed through an identifier. The H5I interface provides functions to query, validate, + * and manage these identifiers and their associated resources. + * + * \subsection subsec_identifier_types Identifier Types + * + * HDF5 defines several built-in identifier types: + * + * \li #H5I_FILE - File identifiers + * \li #H5I_GROUP - Group identifiers + * \li #H5I_DATATYPE - Datatype identifiers + * \li #H5I_DATASPACE - Dataspace identifiers + * \li #H5I_DATASET - Dataset identifiers + * \li #H5I_ATTR - Attribute identifiers + * \li #H5I_MAP - Map identifiers + * \li #H5I_VFL - Virtual File Layer driver identifiers + * \li #H5I_VOL - Virtual Object Layer connector identifiers + * \li Property list identifiers (various classes) + * + * Each identifier type is managed independently with its own reference counting system. + * + * \subsection subsec_identifier_refcount Reference Counting + * + * Identifiers use reference counting to manage object lifetimes. When an identifier + * is created, its reference count is set to 1. The reference count can be manipulated + * to control when resources are released: + * + * \li #H5Iget_ref retrieves the current reference count + * \li #H5Iinc_ref increments the reference count + * \li #H5Idec_ref decrements the reference count + * + * When a reference count reaches zero, the associated resources are automatically released. + * + * \subsection subsec_identifier_valid Identifier Validation + * + * The H5I interface provides functions to validate identifiers: + * + * \li #H5Iis_valid checks if an identifier is valid and has a positive reference count + * \li #H5Iget_type retrieves the type of an identifier + * \li #H5Itype_exists checks if an identifier type is registered + * + * \subsection subsec_identifier_query Querying Identifiers + * + * Applications can retrieve information about objects through their identifiers: + * + * \li #H5Iget_name retrieves the name of an object (for files, groups, datasets, etc.) + * \li #H5Iget_file_id retrieves the file identifier associated with any object + * + * These functions are particularly useful for debugging and logging. + * + * \subsection subsec_identifier_user User-Defined Identifier Types + * + * Advanced applications can register custom identifier types using #H5Iregister. + * This allows user-defined objects to benefit from HDF5's identifier management: + * + * \li Automatic reference counting + * \li Type safety through identifier types + * \li Integration with HDF5's resource management + * + * Custom identifier types can be destroyed using #H5Idestroy_type when no longer needed. + * + * \subsection subsec_identifier_iteration Identifier Iteration + * + * The H5I interface allows iteration over all identifiers of a given type: + * + * \li #H5Iiterate iterates over identifiers of a specified type with a callback function + * \li #H5Inmembers returns the number of identifiers of a given type + * + * This is useful for resource tracking, debugging, and cleanup operations. + * + * \subsection subsec_identifier_summary Summary + * + * The H5I identifier interface provides essential functionality: + * \li Safe handles to HDF5 objects and resources + * \li Reference counting for automatic resource management + * \li Type checking and validation + * \li Object information retrieval + * \li Support for user-defined identifier types + * \li Iteration capabilities for resource tracking + * + * Proper identifier management is fundamental to writing robust HDF5 applications + * that efficiently manage memory and avoid resource leaks. * *
* Navigate back: \ref index "Main" / \ref UG diff --git a/src/H5Lmodule.h b/src/H5Lmodule.h index eba86d0fb8d..265482a60fa 100644 --- a/src/H5Lmodule.h +++ b/src/H5Lmodule.h @@ -30,7 +30,70 @@ * Navigate back: \ref index "Main" / \ref UG *
* - * @todo Under Construction + * \section sec_link The HDF5 Link Interface + * + * \subsection subsec_link_intro Introduction + * + * The HDF5 Link interface (H5L) provides functions for creating, querying, and manipulating + * links between objects in an HDF5 file. Links are the primary mechanism for organizing objects + * in the HDF5 file hierarchy and enabling navigation between related data. + * + * \subsection subsec_link_types Link Types + * + * HDF5 supports three types of links: + * + * \li \Bold{Hard Links}: Direct references to objects within the same file. An object exists + * as long as at least one hard link points to it. Hard links cannot cross file boundaries. + * + * \li \Bold{Soft Links (Symbolic Links)}: Path-based references that store the path to an object + * as a string. Soft links can "dangle" if the target object is deleted or moved. + * + * \li \Bold{External Links}: References to objects in other HDF5 files. External links store + * both the filename and the path to the object within that file. + * + * \subsection subsec_link_create Creating Links + * + * Links can be created using several methods: + * + * \li #H5Lcreate_hard creates hard links to objects + * \li #H5Lcreate_soft creates soft (symbolic) links using path strings + * \li #H5Lcreate_external creates links to objects in other HDF5 files + * + * \subsection subsec_link_ops Link Operations + * + * The H5L interface provides functions for: + * + * \li \Bold{Querying}: #H5Lexists checks if a link exists, #H5Lget_info retrieves link information + * \li \Bold{Moving/Copying}: #H5Lmove and #H5Lcopy relocate or duplicate links + * \li \Bold{Deleting}: #H5Ldelete removes links from the file + * \li \Bold{Iterating}: #H5Literate and #H5Lvisit traverse links in groups + * + * \subsection subsec_link_traverse Link Traversal + * + * The H5L interface includes powerful traversal functions that can iterate over all links + * in a group or recursively visit all links in a group hierarchy: + * + * \li #H5Literate iterates over links in a group using a group identifier + * \li #H5Lvisit recursively visits all links in a group and its subgroups using a group identifier + * \li #H5Literate_by_name and #H5Lvisit_by_name provide the same functionality but accept a + * location identifier and group name, allowing traversal without first opening the group + * + * These functions accept callbacks that are invoked for each link, enabling custom processing. + * + * \subsection subsec_link_user User-Defined Link Types + * + * HDF5 allows registration of custom link types through #H5Lregister. User-defined links + * can implement custom traversal and access behaviors. Use #H5Lunregister to remove + * user-defined link classes and #H5Lis_registered to check if a link class is registered. + * + * \subsection subsec_link_summary Summary + * + * The H5L interface provides flexible mechanisms for organizing and navigating HDF5 files: + * \li Hard links provide reference-counted object management + * \li Soft links enable flexible, path-based references + * \li External links connect objects across multiple files + * \li Iteration functions enable systematic traversal of file hierarchies + * \li User-defined links support extensibility for custom use cases * *
* Navigate back: \ref index "Main" / \ref UG diff --git a/src/H5Mmodule.h b/src/H5Mmodule.h index 3b462dcb29e..5e71ef6146b 100644 --- a/src/H5Mmodule.h +++ b/src/H5Mmodule.h @@ -36,9 +36,234 @@ * * \section sec_map HDF5 Map Object * - * \todo Describe the map life cycle. + * \subsection subsec_map_intro Introduction * - * \todo How does MAPL fit into \ref subsubsec_plist_class. + * While the HDF5 data model provides a flexible way to store structured data, some applications + * require a more general mechanism for indexing information with arbitrary keys. HDF5 Map objects + * address this need by providing application-defined key-value stores where key-value pairs can + * be added, retrieved by key, and iterated over. + * + * Map objects are available only through VOL connectors that implement the map interface. The + * native HDF5 file format does not support map objects. VOL connectors with map support include + * DAOS and other storage systems optimized for key-value operations. + * + * @see H5M Reference Manual + * + * \subsection subsec_map_lifecycle Map Object Life Cycle + * + * Map objects follow a well-defined life cycle from creation through cleanup: + * + *
    + *
  1. Creation + *
      + *
    • Create a new map with #H5Mcreate() or #H5Mcreate_anon(): + *
        + *
      • #H5Mcreate() creates a map and links it into the file hierarchy with a specified name
      • + *
      • #H5Mcreate_anon() creates an anonymous map that must be linked with #H5Olink before + * closing
      • + *
      + *
    • + *
    • Specify the key datatype and value datatype during creation - these define how keys and values + * are stored in the map
    • + *
    • Optionally provide creation and access property lists (MCPL and MAPL) to control map behavior
    • + *
    • The function returns a map identifier (hid_t) used for all subsequent operations
    • + *
    + *
  2. + * + *
  3. Opening + *
      + *
    • Open an existing map with #H5Mopen() by providing its location and name
    • + *
    • Optionally specify a map access property list (MAPL) to control access behavior
    • + *
    • Multiple opens of the same map are allowed and each returns a separate identifier
    • + *
    + *
  4. + * + *
  5. Data Operations + *
      + *
    • Adding/Updating: Use #H5Mput() to add new key-value pairs or update existing values + *
        + *
      • Specify memory datatypes for the key and value being written
      • + *
      • If the key already exists, its value is updated
      • + *
      • If the key is new, a new key-value pair is added
      • + *
      + *
    • + *
    • Retrieving: Use #H5Mget() to retrieve a value by its key + *
        + *
      • Provide a buffer to receive the value
      • + *
      • Specify memory datatypes for key and value
      • + *
      • Type conversion is performed if memory and stored datatypes differ
      • + *
      + *
    • + *
    • Checking Existence: Use #H5Mexists() to check if a key exists without retrieving its value + *
    • + *
    • Deleting: Use #H5Mdelete() to remove a key-value pair from the map + *
    • + *
    • Iterating: Use #H5Miterate() or #H5Miterate_by_name() to process all key-value pairs + *
        + *
      • Provide a callback function that is invoked for each key-value pair
      • + *
      • Iteration order is determined by the VOL connector implementation
      • + *
      • Can start iteration from a specific index position
      • + *
      + *
    • + *
    + *
  6. + * + *
  7. Query Operations + *
      + *
    • #H5Mget_count() retrieves the number of key-value pairs stored in the map
    • + *
    • #H5Mget_key_type() returns the datatype used for keys
    • + *
    • #H5Mget_val_type() returns the datatype used for values
    • + *
    • #H5Mget_create_plist() retrieves the map creation property list (MCPL)
    • + *
    • #H5Mget_access_plist() retrieves the map access property list (MAPL)
    • + *
    + *
  8. + * + *
  9. Closing + *
      + *
    • Close the map with #H5Mclose() when finished
    • + *
    • All map identifiers should be closed before closing the file
    • + *
    • Closing a map does not delete it from the file
    • + *
    • The library maintains reference counts and will not actually close the map until all + * identifiers are closed
    • + *
    + *
  10. + *
+ * + * \subsection subsec_map_plist Map Property Lists + * + * Map objects use two types of property lists that fit into the HDF5 property list class hierarchy + * (see \ref subsubsec_plist_class): + * + * + * + * These property list classes follow the same inheritance hierarchy as other HDF5 objects. + * The MCPL inherits properties relevant to object creation (like character encoding for names), + * while the MAPL controls access-specific settings. + * + * \subsection subsec_map_datatypes Key and Value Datatypes + * + * Map objects require two datatypes: + * + * + * + * During #H5Mput() and #H5Mget() operations, memory datatypes can differ from the stored datatypes. + * The HDF5 library will perform type conversion as needed, similar to dataset I/O operations. + * + * \subsection subsec_map_example Example Usage + * + * The following example demonstrates creating a map, adding key-value pairs, and retrieving values. + * Note that this example requires a VOL connector with map support (e.g., DAOS). + * See \ref H5VL_UG for VOL connector configuration details. + * + * \code + * hid_t file_id, map_id, vls_type_id, fapl_id; + * const char *names[2] = {"Alice", "Bob"}; + * uint64_t IDs[2] = {25385486, 34873275}; + * uint64_t val_out; + * herr_t ret; + * + * // Setup file access property list with VOL connector that supports maps + * // (Configuration depends on specific VOL connector - see connector documentation) + * fapl_id = H5Pcreate(H5P_FILE_ACCESS); + * // ... configure VOL connector in fapl_id ... + * + * // Create variable-length string datatype for keys + * vls_type_id = H5Tcopy(H5T_C_S1); + * H5Tset_size(vls_type_id, H5T_VARIABLE); + * + * // Create file + * file_id = H5Fcreate("file.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + * + * // Create map with string keys and uint64 values + * map_id = H5Mcreate(file_id, "map", vls_type_id, H5T_NATIVE_UINT64, + * H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + * + * // Add key-value pairs + * H5Mput(map_id, vls_type_id, &names[0], H5T_NATIVE_UINT64, &IDs[0], H5P_DEFAULT); + * H5Mput(map_id, vls_type_id, &names[1], H5T_NATIVE_UINT64, &IDs[1], H5P_DEFAULT); + * + * // Retrieve a value by key + * ret = H5Mget(map_id, vls_type_id, &names[0], H5T_NATIVE_UINT64, &val_out, H5P_DEFAULT); + * if(ret < 0 || val_out != IDs[0]) { + * fprintf(stderr, "Failed to retrieve correct value from map\n"); + * goto error; + * } + * + * // Close map and other objects + * H5Mclose(map_id); + * H5Tclose(vls_type_id); + * H5Pclose(fapl_id); + * H5Fclose(file_id); + * return 0; + * + * error: + * H5E_BEGIN_TRY { + * H5Mclose(map_id); + * H5Tclose(vls_type_id); + * H5Pclose(fapl_id); + * H5Fclose(file_id); + * } H5E_END_TRY; + * return -1; + * \endcode + * + * \subsection subsec_map_notes Important Notes + * + * * * Previous Chapter \ref sec_async - Next Chapter \ref sec_reference * @@ -68,24 +293,32 @@ * * \par Example: * \code + * // NOTE: This example requires a VOL connector with map support (e.g., DAOS) * hid_t file_id, fapl_id, map_id, vls_type_id; - * const char *names[2] = ["Alice", "Bob"]; - * uint64_t IDs[2] = [25385486, 34873275]; + * const char *names[2] = {"Alice", "Bob"}; + * uint64_t IDs[2] = {25385486, 34873275}; * uint64_t val_out; + * herr_t ret; * - * + * // Setup file access property list with VOL connector that supports maps + * fapl_id = H5Pcreate(H5P_FILE_ACCESS); + * // ... configure VOL connector in fapl_id (see connector documentation) ... * * vls_type_id = H5Tcopy(H5T_C_S1); * H5Tset_size(vls_type_id, H5T_VARIABLE); * file_id = H5Fcreate("file.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); - * map_id = H5Mcreate(file_id, "map", vls_type_id, H5T_NATIVE_UINT64, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + * map_id = H5Mcreate(file_id, "map", vls_type_id, H5T_NATIVE_UINT64, + * H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); * H5Mput(map_id, vls_type_id, &names[0], H5T_NATIVE_UINT64, &IDs[0], H5P_DEFAULT); * H5Mput(map_id, vls_type_id, &names[1], H5T_NATIVE_UINT64, &IDs[1], H5P_DEFAULT); - * H5Mget(map_id, vls_type_id, &names[0], H5T_NATIVE_UINT64, &val_out, H5P_DEFAULT); - * if(val_out != IDs[0]) - * ERROR; + * ret = H5Mget(map_id, vls_type_id, &names[0], H5T_NATIVE_UINT64, &val_out, H5P_DEFAULT); + * if(ret < 0 || val_out != IDs[0]) { + * fprintf(stderr, "Map retrieval failed\n"); + * // Handle error... + * } * H5Mclose(map_id); * H5Tclose(vls_type_id); + * H5Pclose(fapl_id); * H5Fclose(file_id); * \endcode * diff --git a/src/H5Omodule.h b/src/H5Omodule.h index b82de25c520..e62eb520283 100644 --- a/src/H5Omodule.h +++ b/src/H5Omodule.h @@ -30,7 +30,79 @@ * Navigate back: \ref index "Main" / \ref UG *
* - * @todo Under Construction + * \section sec_object The HDF5 Object Interface + * + * \subsection subsec_object_intro Introduction + * + * The HDF5 Object interface (H5O) provides functions for managing HDF5 objects at a generic level. + * While specific object types (groups, datasets, datatypes) have their own interfaces (\ref H5G, + * \ref H5D, \ref H5T), the H5O interface enables operations that apply to all object types uniformly. + * + * @see H5O Reference Manual + * + * \subsection subsec_object_ops Object Operations + * + * The H5O interface provides several categories of operations: + * + * \li \Bold{Object Information}: Retrieve metadata about objects using #H5Oget_info, #H5Oget_info_by_name, + * #H5Oget_info_by_idx, and #H5Oget_native_info. These functions return information such as + * reference counts, modification times, and storage details. + * + * \li \Bold{Object Copying}: Copy objects between locations or files using #H5Ocopy. This creates + * a complete copy of the source object including all its attributes and, for groups, can + * optionally copy contained objects recursively. + * + * \li \Bold{Object Opening}: Open objects by address or index using #H5Oopen, #H5Oopen_by_idx, + * or #H5Oopen_by_token. This enables access to objects when the specific type is unknown. + * + * \li \Bold{Object Linking}: Create links to objects using #H5Olink, enabling multiple names + * to reference the same object. + * + * \subsection subsec_object_visit Object Traversal + * + * The H5O interface provides powerful traversal capabilities: + * + * \li #H5Ovisit recursively visits all objects in a group hierarchy + * \li #H5Ovisit_by_name performs recursive visitation starting from a named location + * + * These functions invoke a user-supplied callback for each object encountered, enabling + * custom processing, cataloging, or analysis of file contents. + * + * \subsection subsec_object_comments Object Comments + * + * Objects can have associated comment strings for documentation purposes: + * + * \li #H5Oset_comment and #H5Oset_comment_by_name attach comments to objects + * \li #H5Oget_comment and #H5Oget_comment_by_name retrieve object comments + * + * \subsection subsec_object_refcount Reference Counting + * + * HDF5 uses reference counting to manage object lifetimes. The library automatically + * deletes objects when their reference count reaches zero: + * + * \li #H5Oincr_refcount manually increments an object's reference count + * \li #H5Odecr_refcount manually decrements an object's reference count + * + * \attention Manual manipulation of reference counts should be used with extreme caution. + * Improper use can lead to memory leaks or premature object deletion. + * + * \subsection subsec_object_cache Cache Control + * + * For performance optimization, the H5O interface provides cache management functions: + * + * \li #H5Orefresh reloads object metadata from disk, discarding cached data + * \li #H5Oflush writes object metadata to disk immediately + * + * These functions are particularly useful in SWMR (Single Writer Multiple Reader) scenarios. + * + * \subsection subsec_object_summary Summary + * + * The H5O interface provides essential generic object operations: + * \li Query object metadata and properties across all object types + * \li Copy objects within and between HDF5 files + * \li Traverse object hierarchies with custom processing + * \li Manage object comments for documentation + * \li Control object caching and persistence * *
* Navigate back: \ref index "Main" / \ref UG diff --git a/src/H5Pmodule.h b/src/H5Pmodule.h index 0c6c5098dab..124ea485a62 100644 --- a/src/H5Pmodule.h +++ b/src/H5Pmodule.h @@ -293,13 +293,36 @@ * See \ref subsec_attribute_work. * * + * + * + * Map creation (MCPL) + * + * + * \ref H5P_MAP_CREATE + * + * + * See \ref subsec_map_plist. Only available with VOL connectors that support maps. + * + * + * + * + * Map access (MAPL) + * + * + * \ref H5P_MAP_ACCESS + * + * + * See \ref subsec_map_plist. Only available with VOL connectors that support maps. + * + * * * * Note: In the table above, the abbreviations to the right of each property list class name in this * table are widely used in both HDF5 programmer documentation and HDF5 source code. For * example, \ref FCPL (FCPL) is the file creation property list, \ref OCPL (OCPL) is the object creation - * property list, \ref OCPYPL (OCPYPL) is object copy property list, and \ref STRCPL (STRCPL) is the string - * creation property list. These abbreviations may appear in either uppercase or lowercase. + * property list, \ref OCPYPL (OCPYPL) is object copy property list, \ref STRCPL (STRCPL) is the string + * creation property list, and \ref MCPL (MCPL) and \ref MAPL (MAPL) are the map creation and access + * property lists. These abbreviations may appear in either uppercase or lowercase. * * The “HDF5 property list class inheritance hierarchy” figure, immediately following, illustrates * the inheritance hierarchy of HDF5's property list classes. Properties are defined at the root of the @@ -1121,6 +1144,11 @@ * TAPL isn't supported yet. * * + * \defgroup MCPL Map Creation Properties + * \ingroup H5P + * Properties for map object creation. + * + * * \defgroup MAPL VOL Data Mapping Properties * \ingroup H5P * Empty property class. diff --git a/src/H5Smodule.h b/src/H5Smodule.h index 923542d5d74..5ea055c043d 100644 --- a/src/H5Smodule.h +++ b/src/H5Smodule.h @@ -974,7 +974,29 @@ * * \subsection subsec_dataspace_select Dataspace Selection Operations and Data Transfer * - * This section is under construction. + * Dataspace selections play a critical role in HDF5 data transfer operations. When reading or writing + * data with #H5Dread or #H5Dwrite, selections determine which elements are transferred between memory + * and the dataset. The HDF5 Library supports independent specification of selections for both the + * dataset (file dataspace) and the memory buffer (memory dataspace). + * + * During data transfer, selections work as follows: + * \li The selection in the file dataspace identifies which elements to read from or write to in the dataset. + * \li The selection in the memory dataspace defines where to place the read data or where to retrieve the + write data. + * \li Both selections must contain the same number of elements and should not exceed the dataset dimensions. + * + * Additionally, as data is transferred, HDF5 automatically performs data type conversion between the + * file and memory representations if the source and destination data types differ. + * + * Selection operations are designed to work with commonly structured patterns while also allowing for + * arbitrary point and hyperslab selections to provide maximum flexibility. These selections can be combined + * using set operations, such as #H5S_SELECT_OR for a union and #H5S_SELECT_AND for an intersection. You can + * then pass these combined selections to #H5Sselect_hyperslab or #H5Sselect_elements to efficiently create + * complex selection patterns. + * + * For parallel I/O operations, collective data transfers can optimize performance when multiple processes + * access different selections of the same dataset simultaneously. See the parallel HDF5 documentation for + * details on collective I/O with selections. * * \subsection subsec_dataspace_refer References * diff --git a/src/H5Tmodule.h b/src/H5Tmodule.h index 453e2ec82b8..b095bae08dd 100644 --- a/src/H5Tmodule.h +++ b/src/H5Tmodule.h @@ -2790,10 +2790,9 @@ filled according to the value of this property. The padding can be: * Easy to access each plane, can select any plane(s) * * - * Less efficient to access a ‘column’ through the planes + * Less efficient to access a 'column' through the planes * * - * * * * N+1 Dimension @@ -2806,7 +2805,6 @@ filled according to the value of this property. The padding can be: * The added dimension may not make sense in the scientific model * * - * * * * Compound Datatype @@ -2819,7 +2817,6 @@ filled according to the value of this property. The padding can be: * Not a natural representation for a matrix * * - * * * * Array diff --git a/src/H5VLmodule.h b/src/H5VLmodule.h index 7bb6681a138..638ec509462 100644 --- a/src/H5VLmodule.h +++ b/src/H5VLmodule.h @@ -641,7 +641,101 @@ /** *\defgroup H5VL VOL connector (H5VL) * - * \todo Describe the VOL plugin life cycle. + * Use the functions in this module to manage HDF5 Virtual Object Layer (VOL) connectors. + * + * VOL connectors provide an abstraction layer for storage operations, enabling HDF5 to work + * with different storage backends while maintaining the familiar HDF5 API. This module provides + * functions for registering, configuring, and managing VOL connector plugins. + * + *

VOL Connector Life Cycle

+ * + * VOL connectors can be implemented as dynamically loaded plugins, statically linked libraries, + * or built into the HDF5 library itself. Throughout this documentation, "VOL connector" refers + * to the connector implementation regardless of how it is deployed. + * + * VOL connectors follow a well-defined life cycle from loading to cleanup: + * + *
    + *
  1. Discovery and Loading + *
      + *
    • For dynamically loaded plugins: The library searches the plugin path + * (default: /usr/local/hdf5/lib/plugin on POSIX, %ALLUSERSPROFILE%/hdf5/lib/plugin on Windows)
    • + *
    • Plugin path can be overridden using the #HDF5_PLUGIN_PATH environment variable
    • + *
    • For statically linked connectors: The connector is available immediately when the library + *loads
    • For internal connectors: Built into the HDF5 library (e.g., the native VOL connector)
    • + *
    + *
  2. + * + *
  3. Registration + *
      + *
    • Before use, connectors must be registered with the library using: + *
        + *
      • #H5VLregister_connector() - Registers a new VOL connector
      • + *
      • #H5VLregister_connector_by_name() - Register by connector name
      • + *
      • #H5VLregister_connector_by_value() - Register by connector-specific value
      • + *
      + *
    • + *
    • Registration loads the connector and returns an HDF5 identifier (#hid_t)
    • + *
    • Many connectors provide initialization functions that handle registration automatically
    • + *
    • Registration can also occur automatically via the #HDF5_VOL_CONNECTOR environment variable
    • + *
    + *
  4. + * + *
  5. Configuration + *
      + *
    • The connector is set in a file access property list (fapl) using: + *
        + *
      • #H5Pset_vol() - Generic method using connector ID and optional configuration
      • + *
      • Connector-specific functions (e.g., H5Pset_fapl_()) - Often more convenient
      • + *
      • #HDF5_VOL_CONNECTOR environment variable - Sets the default connector for all file opens
      • + *
      + *
    • + *
    • Configuration may include connector-specific parameters passed via info structs or parameter + *strings
    • Parameter strings can be parsed using #H5VLconnector_str_to_info()
    • + *
    + *
  6. + * + *
  7. Active Use + *
      + *
    • When a file is opened or created with a configured fapl, the specified VOL connector handles all + * storage operations
    • + *
    • All API calls that manipulate storage (create, open, read, write, etc.) are forwarded to the + * connector's callbacks
    • + *
    • The connector remains active for the lifetime of all files opened with it
    • + *
    • Multiple different connectors can be active simultaneously for different files
    • + *
    + *
  8. + * + *
  9. Query and Introspection + *
      + *
    • #H5Pget_vol_id() retrieves the connector ID from a fapl
    • + *
    • #H5Pget_vol_info() retrieves connector-specific configuration information
    • + *
    • #H5VLget_connector_name() retrieves the connector's registered name
    • + *
    • #H5VLis_connector_registered_by_name() checks if a connector is registered by name
    • + *
    • #H5VLis_connector_registered_by_value() checks if a connector is registered by value
    • + *
    • #H5VLquery_optional() determines support for optional operations
    • + *
    + *
  10. + * + *
  11. Cleanup and Unregistration + *
      + *
    • Connector IDs can be closed using: + *
        + *
      • #H5VLunregister_connector() - Unregister a connector
      • + *
      • #H5VLclose() - Close a connector ID (same internal implementation)
      • + *
      + *
    • + *
    • The library maintains reference counts on connector IDs and will not actually close them until + * the reference count reaches zero
    • + *
    • It is safe to close connector IDs after use, even while files opened with that connector remain + *open
    • The library automatically unloads all connectors when it shuts down
    • The native VOL + *connector cannot be unloaded and is always available
    • Connector-specific info structures should be + *freed using #H5VLfree_connector_info()
    • + *
    + *
  12. + *
+ * + * @see H5VL_UG * * \defgroup ASYNC Asynchronous Functions * \brief List of the asynchronous functions. diff --git a/src/H5Zmodule.h b/src/H5Zmodule.h index c1430bd3123..9020e333f04 100644 --- a/src/H5Zmodule.h +++ b/src/H5Zmodule.h @@ -30,7 +30,86 @@ * Navigate back: \ref index "Main" / \ref UG *
* - * @todo Under Construction + * \section sec_filter The HDF5 Filter Interface + * + * \subsection subsec_filter_intro Introduction + * + * The HDF5 Filter interface (H5Z) provides a flexible pipeline mechanism for processing + * dataset data during I/O operations. Filters can perform data compression, error checking, + * data transformation, and other custom operations on dataset chunks. + * + * Filters operate on chunked datasets only (see \ref subsec_dataset_filters for details on + * dataset chunking) and are applied independently to each chunk. Multiple filters can be + * chained together in a pipeline, where the output of one filter becomes the input to the next. + * + * \subsection subsec_filter_builtin Built-in Filters + * + * HDF5 includes several standard filters: + * + * \li \Bold{DEFLATE (gzip)}: General-purpose compression using the gzip algorithm + * (#H5Z_FILTER_DEFLATE). Provides good compression ratios with moderate CPU usage. + * + * \li \Bold{SZIP}: Compression algorithm designed for scientific data (#H5Z_FILTER_SZIP). + * Offers better performance than DEFLATE for certain data patterns. + * + * \li \Bold{SHUFFLE}: Rearranges byte order to improve compression (#H5Z_FILTER_SHUFFLE). + * Typically used before compression filters. + * + * \li \Bold{FLETCHER32}: Computes and verifies checksums for error detection + * (#H5Z_FILTER_FLETCHER32). Ensures data integrity. + * + * \li \Bold{NBIT}: Lossless compression for datasets with unused bits (#H5Z_FILTER_NBIT). + * + * \li \Bold{SCALEOFFSET}: Lossy compression using scaling and offset (#H5Z_FILTER_SCALEOFFSET). + * + * \subsection subsec_filter_usage Using Filters + * + * Filters are configured through dataset creation property lists. Enable chunking first + * using #H5Pset_chunk, then add compression with functions like #H5Pset_deflate. + * + * \subsection subsec_filter_pipeline Filter Pipelines + * + * Multiple filters can be combined in a pipeline. Filters are applied in the order + * they are added during write operations and in reverse order during read operations. + * Common pipelines combine #H5Pset_shuffle, #H5Pset_deflate, and #H5Pset_fletcher32. + * + * \subsection subsec_filter_custom Custom Filters + * + * Applications can create and register custom filters: + * + * \li Define a filter function with signature matching #H5Z_func_t + * \li Create a #H5Z_class_t structure describing the filter + * \li Register the filter using #H5Zregister + * \li Apply the filter using #H5Pset_filter with the filter ID + * + * Custom filters enable domain-specific data transformations, specialized compression + * algorithms, encryption, and other custom processing. + * + * \subsection subsec_filter_query Querying Filters + * + * The H5Z interface provides functions to query available filters: + * + * \li #H5Zfilter_avail checks if a filter is available + * \li #H5Zget_filter_info retrieves information about a filter's capabilities + * \li #H5Zunregister removes a filter from the pipeline + * + * \subsection subsec_filter_plugins Filter Plugins + * + * HDF5 supports dynamic loading of filter plugins, allowing filters to be added + * without recompiling applications. See \ref sec_filter_plugins for details on + * creating and using filter plugins. + * + * \subsection subsec_filter_summary Summary + * + * The H5Z filter interface provides: + * \li Built-in compression and error checking filters + * \li Flexible filter pipeline mechanism + * \li Support for custom user-defined filters + * \li Dynamic filter plugin loading + * \li Per-chunk processing for optimal performance + * + * Filters are essential for reducing storage requirements and ensuring data integrity + * in HDF5 files while maintaining compatibility and performance. * *
* Navigate back: \ref index "Main" / \ref UG diff --git a/src/H5module.h b/src/H5module.h index f62aea407ce..7f89841b6a7 100644 --- a/src/H5module.h +++ b/src/H5module.h @@ -409,7 +409,33 @@ * @see @ref sec_plist. * * \subsubsection subsubsec_data_model_abstract_link Link - * This section is under construction. + * + * Links are the fundamental mechanism for organizing objects in the HDF5 file hierarchy. A link is a + * path from one object (typically a group) to another object, creating relationships that enable + * navigation and provide meaningful structure to the data. + * + * HDF5 supports three types of links: + * + * \li \Bold{Hard Links}: Direct references to objects within the file. An object exists as long as at + * least one hard link points to it. Hard links implement reference counting - when the last hard + * link to an object is deleted, the object's storage is freed. Hard links cannot cross file boundaries. + * + * \li \Bold{Soft Links (Symbolic Links)}: Path-based references stored as strings. Soft links provide + * flexible indirection but can "dangle" if the target object is deleted or renamed. Soft links can + * reference objects that do not yet exist, enabling forward references. + * + * \li \Bold{External Links}: References to objects in other HDF5 files. External links store both the + * target filename and the path to the object within that file, enabling multi-file data organizations. + * + * Links have names and are stored within groups. The combination of links and groups creates a hierarchical + * namespace similar to a filesystem directory structure. Multiple hard links can point to the same object, + * allowing objects to appear in multiple locations within the hierarchy. + * + * The HDF5 Link interface (H5L) provides functions for creating, querying, and manipulating links. + * Additionally, HDF5 supports user-defined link types, allowing applications to implement custom link + * behaviors and traversal semantics. + * + * @see @ref sec_link for detailed information on links and the H5L interface. * * \subsection subsec_data_model_storage The HDF5 Storage Model * \subsubsection subsubsec_data_model_storage_spec The Abstract Storage Model: the HDF5 Format Specification