diff --git a/docs/source/backends/adios2.rst b/docs/source/backends/adios2.rst index 34450bd2c0..d639f01897 100644 --- a/docs/source/backends/adios2.rst +++ b/docs/source/backends/adios2.rst @@ -10,12 +10,43 @@ For further information, check out the :ref:`installation guide `, :ref:`build dependencies ` and the :ref:`build options `. -I/O Method ----------- +I/O Method and Engine Selection +------------------------------- ADIOS2 has several engines for alternative file formats and other kinds of backends, yet natively writes to ``.bp`` files. -The openPMD API uses the BP4 engine as the default file engine and the SST engine for streaming support. -We currently leverage the default ADIOS2 transport parameters, i.e. ``POSIX`` on Unix systems and ``FStream`` on Windows. +The openPMD API uses the File meta engine as the default file engine and the SST engine for streaming support. + +The ADIOS2 engine can be selected in different ways: + +1. Automatic detection via the selected file ending +2. Explicit selection of an engine by specifying the environment variable ``OPENPMD_ADIOS2_ENGINE`` (case-independent). + This overrides the automatically detected engine. +3. Explicit selection of an engine by specifying the JSON/TOML key ``adios2.engine.type`` as a case-independent string. + This overrides both previous options. + +Automatic engine detection supports the following extensions: + +.. list-table:: + :header-rows: 1 + + * - Extension + - Selected ADIOS2 Engine + * - ``.bp`` + - ``"file"`` + * - ``.bp4`` + - ``"bp4"`` + * - ``.bp5`` + - ``"bp5"`` + * - ``.sst`` + - ``"sst"`` + * - ``.ssc`` + - ``"ssc"`` + +Specifying any of these extensions will automatically activate the ADIOS2 backend. +The ADIOS2 backend will create file system paths exactly as they were specified and not change file extensions. +Exceptions to this are the BP3 and SST engines which require their endings ``.bp`` and ``.sst`` respectively. + +For file engines, we currently leverage the default ADIOS2 transport parameters, i.e. ``POSIX`` on Unix systems and ``FStream`` on Windows. Steps ----- diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index af79d72d3d..d4a199be53 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -132,12 +132,16 @@ class ADIOS2IOHandlerImpl AbstractIOHandler *, MPI_Comm, json::TracingJSON config, - std::string engineType); + std::string engineType, + std::string specifiedExtension); #endif // openPMD_HAVE_MPI explicit ADIOS2IOHandlerImpl( - AbstractIOHandler *, json::TracingJSON config, std::string engineType); + AbstractIOHandler *, + json::TracingJSON config, + std::string engineType, + std::string specifiedExtension); ~ADIOS2IOHandlerImpl() override; @@ -231,6 +235,11 @@ class ADIOS2IOHandlerImpl * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine */ std::string m_engineType; + /* + * The filename extension specified by the user. + */ + std::string m_userSpecifiedExtension; + ADIOS2Schema::schema_t m_schema = ADIOS2Schema::schema_0000_00_00; enum class UseSpan : char @@ -1320,7 +1329,8 @@ class ADIOS2IOHandler : public AbstractIOHandler Access, MPI_Comm, json::TracingJSON options, - std::string engineType); + std::string engineType, + std::string specifiedExtension); #endif @@ -1328,7 +1338,8 @@ class ADIOS2IOHandler : public AbstractIOHandler std::string path, Access, json::TracingJSON options, - std::string engineType); + std::string engineType, + std::string specifiedExtension); std::string backendName() const override { diff --git a/include/openPMD/IO/AbstractIOHandlerHelper.hpp b/include/openPMD/IO/AbstractIOHandlerHelper.hpp index e6775a4757..0d1fe4c8da 100644 --- a/include/openPMD/IO/AbstractIOHandlerHelper.hpp +++ b/include/openPMD/IO/AbstractIOHandlerHelper.hpp @@ -35,6 +35,8 @@ namespace openPMD * @param access Access mode describing desired operations and permissions of the desired handler. * @param format Format describing the IO backend of the desired handler. + * @param originalExtension The filename extension as it was originally + * specified by the user. * @param comm MPI communicator used for IO. * @param options JSON-formatted option string, to be interpreted by * the backend. @@ -47,6 +49,7 @@ std::shared_ptr createIOHandler( std::string path, Access access, Format format, + std::string originalExtension, MPI_Comm comm, JSON options); #endif @@ -58,6 +61,8 @@ std::shared_ptr createIOHandler( * @param access Access describing desired operations and permissions * of the desired handler. * @param format Format describing the IO backend of the desired handler. + * @param originalExtension The filename extension as it was originally + * specified by the user. * @param options JSON-formatted option string, to be interpreted by * the backend. * @tparam JSON Substitute for nlohmann::json. Templated to avoid @@ -66,9 +71,16 @@ std::shared_ptr createIOHandler( */ template std::shared_ptr createIOHandler( - std::string path, Access access, Format format, JSON options = JSON()); + std::string path, + Access access, + Format format, + std::string originalExtension, + JSON options = JSON()); // version without configuration to use in AuxiliaryTest -std::shared_ptr -createIOHandler(std::string path, Access access, Format format); +std::shared_ptr createIOHandler( + std::string path, + Access access, + Format format, + std::string originalExtension); } // namespace openPMD diff --git a/include/openPMD/IO/Format.hpp b/include/openPMD/IO/Format.hpp index 993f7bae90..697e73f788 100644 --- a/include/openPMD/IO/Format.hpp +++ b/include/openPMD/IO/Format.hpp @@ -30,7 +30,9 @@ enum class Format { HDF5, ADIOS1, - ADIOS2, + ADIOS2_BP, + ADIOS2_BP4, + ADIOS2_BP5, ADIOS2_SST, ADIOS2_SSC, JSON, diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 39da51057b..e828bcd7f7 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -108,6 +108,11 @@ namespace internal * Filename after the expansion pattern without filename extension. */ std::string m_filenamePostfix; + /** + * Filename extension as specified by the user. + * (Not necessarily the backend's default suffix) + */ + std::string m_filenameExtension; /** * The padding in file-based iteration encoding. * 0 if no padding is given (%T pattern). diff --git a/src/Format.cpp b/src/Format.cpp index b92c0c8cf5..7f30e1a793 100644 --- a/src/Format.cpp +++ b/src/Format.cpp @@ -45,7 +45,7 @@ Format determineFormat(std::string const &filename) ); if (bp_backend == "ADIOS2") - return Format::ADIOS2; + return Format::ADIOS2_BP; else if (bp_backend == "ADIOS1") return Format::ADIOS1; else @@ -54,6 +54,10 @@ Format determineFormat(std::string const &filename) "neither ADIOS1 nor ADIOS2: " + bp_backend); } + if (auxiliary::ends_with(filename, ".bp4")) + return Format::ADIOS2_BP4; + if (auxiliary::ends_with(filename, ".bp5")) + return Format::ADIOS2_BP5; if (auxiliary::ends_with(filename, ".sst")) return Format::ADIOS2_SST; if (auxiliary::ends_with(filename, ".ssc")) @@ -72,8 +76,12 @@ std::string suffix(Format f) case Format::HDF5: return ".h5"; case Format::ADIOS1: - case Format::ADIOS2: + case Format::ADIOS2_BP: return ".bp"; + case Format::ADIOS2_BP4: + return ".bp4"; + case Format::ADIOS2_BP5: + return ".bp5"; case Format::ADIOS2_SST: return ".sst"; case Format::ADIOS2_SSC: diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index f0e7976f16..047373cdff 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -69,10 +69,12 @@ ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( AbstractIOHandler *handler, MPI_Comm communicator, json::TracingJSON cfg, - std::string engineType) + std::string engineType, + std::string specifiedExtension) : AbstractIOHandlerImplCommon(handler) , m_ADIOS{communicator} , m_engineType(std::move(engineType)) + , m_userSpecifiedExtension{std::move(specifiedExtension)} { init(std::move(cfg)); } @@ -80,10 +82,14 @@ ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( #endif // openPMD_HAVE_MPI ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( - AbstractIOHandler *handler, json::TracingJSON cfg, std::string engineType) + AbstractIOHandler *handler, + json::TracingJSON cfg, + std::string engineType, + std::string specifiedExtension) : AbstractIOHandlerImplCommon(handler) , m_ADIOS{} , m_engineType(std::move(engineType)) + , m_userSpecifiedExtension(std::move(specifiedExtension)) { init(std::move(cfg)); } @@ -118,6 +124,14 @@ ADIOS2IOHandlerImpl::~ADIOS2IOHandlerImpl() void ADIOS2IOHandlerImpl::init(json::TracingJSON cfg) { + // allow overriding through environment variable + m_engineType = + auxiliary::getEnvString("OPENPMD_ADIOS2_ENGINE", m_engineType); + std::transform( + m_engineType.begin(), + m_engineType.end(), + m_engineType.begin(), + [](unsigned char c) { return std::tolower(c); }); if (cfg.json().contains("adios2")) { m_config = cfg["adios2"]; @@ -146,6 +160,7 @@ void ADIOS2IOHandlerImpl::init(json::TracingJSON cfg) json::asLowerCaseStringDynamic(engineTypeConfig); if (maybeEngine.has_value()) { + // override engine type by JSON/TOML configuration m_engineType = std::move(maybeEngine.value()); } else @@ -228,28 +243,85 @@ ADIOS2IOHandlerImpl::getOperators() return getOperators(m_config); } +using AcceptedEndingsForEngine = std::map; + std::string ADIOS2IOHandlerImpl::fileSuffix() const { // SST engine adds its suffix unconditionally // so we don't add it - static std::map endings{ - {"sst", ""}, - {"staging", ""}, - {"bp4", ".bp"}, - {"bp5", ".bp"}, - {"bp3", ".bp"}, - {"file", ".bp"}, - {"hdf5", ".h5"}, - {"nullcore", ".nullcore"}, - {"ssc", ".ssc"}}; - auto it = endings.find(m_engineType); - if (it != endings.end()) - { - return it->second; + static std::map const endings{ + {"sst", {{"", ""}, {".sst", ""}}}, + {"staging", {{"", ""}, {".sst", ""}}}, + {"filestream", {{".bp", ".bp"}, {".bp4", ".bp4"}, {".bp5", ".bp5"}}}, + {"bp4", {{".bp4", ".bp4"}, {".bp", ".bp"}}}, + {"bp5", {{".bp5", ".bp5"}, {".bp", ".bp"}}}, + {"bp3", {{".bp", ".bp"}}}, + {"file", {{".bp", ".bp"}, {".bp4", ".bp4"}, {".bp5", ".bp5"}}}, + {"hdf5", {{".h5", ".h5"}}}, + {"nullcore", {{".nullcore", ".nullcore"}, {".bp", ".bp"}}}, + {"ssc", {{".ssc", ".ssc"}}}}; + + if (auto engine = endings.find(m_engineType); engine != endings.end()) + { + auto const &acceptedEndings = engine->second; + if (auto ending = acceptedEndings.find(m_userSpecifiedExtension); + ending != acceptedEndings.end()) + { + if ((m_engineType == "file" || m_engineType == "filestream") && + (m_userSpecifiedExtension == ".bp3" || + m_userSpecifiedExtension == ".bp4" || + m_userSpecifiedExtension == ".bp5")) + { + std::cerr + << "[ADIOS2] Explicit ending '" << m_userSpecifiedExtension + << "' was specified in combination with generic file " + "engine '" + << m_engineType + << "'. ADIOS2 will pick a default file ending " + "independent of specified suffix. (E.g. 'simData.bp5' " + "might actually be written as a BP4 dataset.)" + << std::endl; + } + return ending->second; + } + else if (m_userSpecifiedExtension.empty()) + { + std::cerr << "[ADIOS2] No file ending specified. Will not add one." + << std::endl; + if (m_engineType == "bp3") + { + std::cerr + << "Note that the ADIOS2 BP3 engine will add its " + "ending '.bp' if not specified (e.g. 'simData.bp3' " + "will appear on disk as 'simData.bp3.bp')." + << std::endl; + } + return ""; + } + else + { + std::cerr << "[ADIOS2] Specified ending '" + << m_userSpecifiedExtension + << "' does not match the selected engine '" + << m_engineType + << "'. Will use the specified ending anyway." + << std::endl; + if (m_engineType == "bp3") + { + std::cerr + << "Note that the ADIOS2 BP3 engine will add its " + "ending '.bp' if not specified (e.g. 'simData.bp3' " + "will appear on disk as 'simData.bp3.bp')." + << std::endl; + } + return m_userSpecifiedExtension; + } } else { - return ".adios2"; + throw error::WrongAPIUsage( + "[ADIOS2] Specified engine '" + m_engineType + + "' is not supported by ADIOS2 backend."); } } @@ -386,12 +458,7 @@ void ADIOS2IOHandlerImpl::createFile( if (!writable->written) { - std::string name = parameters.name; - std::string suffix(fileSuffix()); - if (!auxiliary::ends_with(name, suffix)) - { - name += suffix; - } + std::string name = parameters.name + fileSuffix(); auto res_pair = getPossiblyExisting(name); InvalidatableFile shared_name = InvalidatableFile(name); @@ -561,12 +628,7 @@ void ADIOS2IOHandlerImpl::openFile( m_handler->directory); } - std::string name = parameters.name; - std::string suffix(fileSuffix()); - if (!auxiliary::ends_with(name, suffix)) - { - name += suffix; - } + std::string name = parameters.name + fileSuffix(); auto file = std::get(getPossiblyExisting(name)); @@ -2299,15 +2361,6 @@ namespace detail // set engine type bool isStreaming = false; { - // allow overriding through environment variable - m_engineType = - auxiliary::getEnvString("OPENPMD_ADIOS2_ENGINE", m_engineType); - std::transform( - m_engineType.begin(), - m_engineType.end(), - m_engineType.begin(), - [](unsigned char c) { return std::tolower(c); }); - impl.m_engineType = this->m_engineType; m_IO.SetEngine(m_engineType); auto it = streamingEngines.find(m_engineType); if (it != streamingEngines.end()) @@ -2366,8 +2419,8 @@ namespace detail { throw std::runtime_error( "[ADIOS2IOHandler] Unknown engine type. Please choose " - "one out of " - "[sst, staging, bp4, bp3, hdf5, file, null]"); + "one out of [sst, staging, bp4, bp3, hdf5, file, " + "filestream, null]"); // not listing unsupported engines } } @@ -3058,9 +3111,15 @@ ADIOS2IOHandler::ADIOS2IOHandler( openPMD::Access at, MPI_Comm comm, json::TracingJSON options, - std::string engineType) + std::string engineType, + std::string specifiedExtension) : AbstractIOHandler(std::move(path), at, comm) - , m_impl{this, comm, std::move(options), std::move(engineType)} + , m_impl{ + this, + comm, + std::move(options), + std::move(engineType), + std::move(specifiedExtension)} {} #endif @@ -3069,9 +3128,14 @@ ADIOS2IOHandler::ADIOS2IOHandler( std::string path, Access at, json::TracingJSON options, - std::string engineType) + std::string engineType, + std::string specifiedExtension) : AbstractIOHandler(std::move(path), at) - , m_impl{this, std::move(options), std::move(engineType)} + , m_impl{ + this, + std::move(options), + std::move(engineType), + std::move(specifiedExtension)} {} std::future @@ -3084,14 +3148,19 @@ ADIOS2IOHandler::flush(internal::ParsedFlushParams &flushParams) #if openPMD_HAVE_MPI ADIOS2IOHandler::ADIOS2IOHandler( - std::string path, Access at, MPI_Comm comm, json::TracingJSON, std::string) + std::string path, + Access at, + MPI_Comm comm, + json::TracingJSON, + std::string, + std::string) : AbstractIOHandler(std::move(path), at, comm) {} #endif // openPMD_HAVE_MPI ADIOS2IOHandler::ADIOS2IOHandler( - std::string path, Access at, json::TracingJSON, std::string) + std::string path, Access at, json::TracingJSON, std::string, std::string) : AbstractIOHandler(std::move(path), at) {} diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 22fa6af60f..0a0a32f53f 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -63,6 +63,8 @@ std::shared_ptr createIOHandler( std::string path, Access access, Format format, + std::string originalExtension, + MPI_Comm comm, json::TracingJSON options) { @@ -75,15 +77,51 @@ std::shared_ptr createIOHandler( case Format::ADIOS1: return constructIOHandler( "ADIOS1", path, access, std::move(options), comm); - case Format::ADIOS2: + case Format::ADIOS2_BP: + return constructIOHandler( + "ADIOS2", + path, + access, + comm, + std::move(options), + "file", + std::move(originalExtension)); + case Format::ADIOS2_BP4: + return constructIOHandler( + "ADIOS2", + path, + access, + comm, + std::move(options), + "bp4", + std::move(originalExtension)); + case Format::ADIOS2_BP5: return constructIOHandler( - "ADIOS2", path, access, comm, std::move(options), "bp4"); + "ADIOS2", + path, + access, + comm, + std::move(options), + "bp5", + std::move(originalExtension)); case Format::ADIOS2_SST: return constructIOHandler( - "ADIOS2", path, access, comm, std::move(options), "sst"); + "ADIOS2", + path, + access, + comm, + std::move(options), + "sst", + std::move(originalExtension)); case Format::ADIOS2_SSC: return constructIOHandler( - "ADIOS2", path, access, comm, std::move(options), "ssc"); + "ADIOS2", + path, + access, + comm, + std::move(options), + "ssc", + std::move(originalExtension)); default: throw std::runtime_error( "Unknown file format! Did you specify a file ending?"); @@ -93,7 +131,11 @@ std::shared_ptr createIOHandler( template <> std::shared_ptr createIOHandler( - std::string path, Access access, Format format, json::TracingJSON options) + std::string path, + Access access, + Format format, + std::string originalExtension, + json::TracingJSON options) { (void)options; switch (format) @@ -104,15 +146,46 @@ std::shared_ptr createIOHandler( case Format::ADIOS1: return constructIOHandler( "ADIOS1", path, access, std::move(options)); - case Format::ADIOS2: + case Format::ADIOS2_BP: + return constructIOHandler( + "ADIOS2", + path, + access, + std::move(options), + "file", + std::move(originalExtension)); + case Format::ADIOS2_BP4: + return constructIOHandler( + "ADIOS2", + path, + access, + std::move(options), + "bp4", + std::move(originalExtension)); + case Format::ADIOS2_BP5: return constructIOHandler( - "ADIOS2", path, access, std::move(options), "bp4"); + "ADIOS2", + path, + access, + std::move(options), + "bp5", + std::move(originalExtension)); case Format::ADIOS2_SST: return constructIOHandler( - "ADIOS2", path, access, std::move(options), "sst"); + "ADIOS2", + path, + access, + std::move(options), + "sst", + std::move(originalExtension)); case Format::ADIOS2_SSC: return constructIOHandler( - "ADIOS2", path, access, std::move(options), "ssc"); + "ADIOS2", + path, + access, + std::move(options), + "ssc", + std::move(originalExtension)); case Format::JSON: return constructIOHandler( "JSON", path, access); @@ -122,13 +195,17 @@ std::shared_ptr createIOHandler( } } -std::shared_ptr -createIOHandler(std::string path, Access access, Format format) +std::shared_ptr createIOHandler( + std::string path, + Access access, + Format format, + std::string originalExtension) { return createIOHandler( std::move(path), access, format, + std::move(originalExtension), json::TracingJSON(json::ParsedConfig{})); } } // namespace openPMD diff --git a/src/Series.cpp b/src/Series.cpp index a68d1270fa..814b0e2489 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -44,6 +44,18 @@ namespace openPMD { namespace { + struct CleanedFilename + { + std::string body; + std::string extension; + + std::tuple decompose() && + { + return std::tuple{ + std::move(body), std::move(extension)}; + } + }; + /** Remove the filename extension of a given storage format. * * @param filename String containing the filename, possibly with @@ -51,7 +63,8 @@ namespace * @param f File format to remove filename extension for. * @return String containing the filename without filename extension. */ - std::string cleanFilename(std::string const &filename, Format f); + CleanedFilename cleanFilename( + std::string const &filename, std::string const &filenameExtension); /** Compound return type for regex matching of filenames */ struct Match @@ -77,7 +90,7 @@ namespace * zero, any amount of padding is matched. * @param postfix String containing tail (i.e. after %T) of desired * filename without filename extension. - * @param f File format to check backend applicability for. + * @param filenameExtension Filename extension to match against * @return Functor returning tuple of bool and int. * bool is True if file could be of type f and matches the * iterationEncoding. False otherwise. int is the amount of padding present @@ -87,7 +100,7 @@ namespace std::string const &prefix, int padding, std::string const &postfix, - Format f); + std::string const &extension); } // namespace struct Series::ParsedInput @@ -98,6 +111,7 @@ struct Series::ParsedInput IterationEncoding iterationEncoding; std::string filenamePrefix; std::string filenamePostfix; + std::string filenameExtension; int filenamePadding = -1; }; // ParsedInput @@ -443,9 +457,10 @@ std::unique_ptr Series::parseInput(std::string filepath) "Can not determine iterationFormat from filename " + input->name); input->filenamePostfix = - cleanFilename(input->filenamePostfix, input->format); + cleanFilename(input->filenamePostfix, suffix(input->format)).body; - input->name = cleanFilename(input->name, input->format); + std::tie(input->name, input->filenameExtension) = + cleanFilename(input->name, suffix(input->format)).decompose(); return input; } @@ -536,6 +551,7 @@ void Series::init( series.m_filenamePrefix = input->filenamePrefix; series.m_filenamePostfix = input->filenamePostfix; series.m_filenamePadding = input->filenamePadding; + series.m_filenameExtension = input->filenameExtension; if (series.m_iterationEncoding == IterationEncoding::fileBased && !series.m_filenamePrefix.empty() && @@ -599,7 +615,7 @@ Given file pattern: ')END" series.m_filenamePrefix, series.m_filenamePadding, series.m_filenamePostfix, - series.m_format), + series.m_filenameExtension), IOHandler()->directory); switch (padding) { @@ -922,7 +938,7 @@ void Series::readFileBased() series.m_filenamePrefix, series.m_filenamePadding, series.m_filenamePostfix, - series.m_format); + series.m_filenameExtension); int padding = autoDetectPadding( std::move(isPartOfSeries), @@ -930,7 +946,11 @@ void Series::readFileBased() // foreach found file with `filename` and `index`: [&series](uint64_t index, std::string const &filename) { Iteration &i = series.iterations[index]; - i.deferParseAccess({std::to_string(index), index, true, filename}); + i.deferParseAccess( + {std::to_string(index), + index, + true, + cleanFilename(filename, series.m_filenameExtension).body}); }); if (series.iterations.empty()) @@ -1652,7 +1672,7 @@ void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) std::map const backendDescriptors{ {"hdf5", Format::HDF5}, {"adios1", Format::ADIOS1}, - {"adios2", Format::ADIOS2}, + {"adios2", Format::ADIOS2_BP}, {"json", Format::JSON}}; std::string backend; getJsonOptionLowerCase(options, "backend", backend); @@ -1661,7 +1681,20 @@ void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) auto it = backendDescriptors.find(backend); if (it != backendDescriptors.end()) { - if (input.format != Format::DUMMY && + if (backend == "adios2" && + (input.format == Format::ADIOS2_BP4 || + input.format == Format::ADIOS2_BP5 || + input.format == Format::ADIOS2_SST || + input.format == Format::ADIOS2_SSC || + input.format == Format::ADIOS2_BP)) + { + // backend = "adios2" should work as a catch-all for all + // different engines, using BP is just the default + // If the file ending was more explicit, keep it. + // -> Nothing to do + } + else if ( + input.format != Format::DUMMY && suffix(input.format) != suffix(it->second)) { std::cerr << "[Warning] Supplied filename extension '" @@ -1669,8 +1702,12 @@ void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) << "' contradicts the backend specified via the " "'backend' key. Will go on with backend " << it->first << "." << std::endl; + input.format = it->second; + } + else + { + input.format = it->second; } - input.format = it->second; } else { @@ -1768,8 +1805,13 @@ Series::Series( json::parseOptions(options, comm, /* considerFiles = */ true); auto input = parseInput(filepath); parseJsonOptions(optionsJson, *input); - auto handler = - createIOHandler(input->path, at, input->format, comm, optionsJson); + auto handler = createIOHandler( + input->path, + at, + input->format, + input->filenameExtension, + comm, + optionsJson); init(handler, std::move(input)); json::warnGlobalUnusedOptions(optionsJson); } @@ -1785,7 +1827,8 @@ Series::Series( json::parseOptions(options, /* considerFiles = */ true); auto input = parseInput(filepath); parseJsonOptions(optionsJson, *input); - auto handler = createIOHandler(input->path, at, input->format, optionsJson); + auto handler = createIOHandler( + input->path, at, input->format, input->filenameExtension, optionsJson); init(handler, std::move(input)); json::warnGlobalUnusedOptions(optionsJson); } @@ -1814,19 +1857,18 @@ WriteIterations Series::writeIterations() namespace { - std::string cleanFilename(std::string const &filename, Format f) + CleanedFilename cleanFilename( + std::string const &filename, std::string const &filenameExtension) { - switch (f) + std::string postfix = + auxiliary::replace_last(filename, filenameExtension, ""); + if (postfix == filename) { - case Format::HDF5: - case Format::ADIOS1: - case Format::ADIOS2: - case Format::ADIOS2_SST: - case Format::ADIOS2_SSC: - case Format::JSON: - return auxiliary::replace_last(filename, suffix(f), ""); - default: - return filename; + return {postfix, ""}; + } + else + { + return {postfix, filenameExtension}; } } @@ -1851,14 +1893,8 @@ namespace std::string const &prefix, int padding, std::string const &postfix, - Format f) + std::string const &filenameSuffix) { - std::string filenameSuffix = suffix(f); - if (filenameSuffix.empty()) - { - return [](std::string const &) -> Match { return {false, 0, 0}; }; - } - std::string nameReg = "^" + prefix; if (padding != 0) { diff --git a/src/config.cpp b/src/config.cpp index d37d9cc88e..b9e27be752 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -45,6 +45,13 @@ std::vector openPMD::getFileExtensions() #if openPMD_HAVE_ADIOS1 || openPMD_HAVE_ADIOS2 fext.emplace_back("bp"); #endif +#if openPMD_HAVE_ADIOS2 + // BP4 is always available in ADIOS2 + fext.emplace_back("bp4"); +#endif +#ifdef ADIOS2_HAVE_BP5 + fext.emplace_back("bp5"); +#endif #ifdef ADIOS2_HAVE_SST fext.emplace_back("sst"); #endif diff --git a/test/AuxiliaryTest.cpp b/test/AuxiliaryTest.cpp index 4a67507cbd..3af7b3741f 100644 --- a/test/AuxiliaryTest.cpp +++ b/test/AuxiliaryTest.cpp @@ -34,7 +34,7 @@ struct TestHelper : public Attributable TestHelper() { writable().IOHandler = - createIOHandler(".", Access::CREATE, Format::JSON); + createIOHandler(".", Access::CREATE, Format::JSON, ".json"); } }; } // namespace openPMD::test @@ -146,7 +146,8 @@ TEST_CASE("container_default_test", "[auxiliary]") { #if openPMD_USE_INVASIVE_TESTS Container c = Container(); - c.writable().IOHandler = createIOHandler(".", Access::CREATE, Format::JSON); + c.writable().IOHandler = + createIOHandler(".", Access::CREATE, Format::JSON, ".json"); REQUIRE(c.empty()); REQUIRE(c.erase("nonExistentKey") == false); @@ -183,7 +184,8 @@ TEST_CASE("container_retrieve_test", "[auxiliary]") #if openPMD_USE_INVASIVE_TESTS using structure = openPMD::test::structure; Container c = Container(); - c.writable().IOHandler = createIOHandler(".", Access::CREATE, Format::JSON); + c.writable().IOHandler = + createIOHandler(".", Access::CREATE, Format::JSON, ".json"); structure s; std::string text = @@ -255,7 +257,8 @@ TEST_CASE("container_access_test", "[auxiliary]") #if openPMD_USE_INVASIVE_TESTS using Widget = openPMD::test::Widget; Container c = Container(); - c.writable().IOHandler = createIOHandler(".", Access::CREATE, Format::JSON); + c.writable().IOHandler = + createIOHandler(".", Access::CREATE, Format::JSON, ".json"); c["1firstWidget"] = Widget(0); REQUIRE(c.size() == 1); diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index f737141bf4..fa2a1cad42 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -1074,12 +1074,24 @@ TEST_CASE("backend_via_json", "[core]") * BP4 engine should be selected even if ending .sst is given */ Series series( - "../samples/optionsViaJsonOverwritesAutomaticDetection.sst", + "../samples/optionsViaJsonOverwritesAutomaticDetectionFile.sst", + Access::CREATE, + R"({"adios2": {"engine": {"type": "file"}}})"); + } + REQUIRE(auxiliary::directory_exists( + "../samples/optionsViaJsonOverwritesAutomaticDetectionFile.sst")); + + { + /* + * BP4 engine should be selected even if ending .sst is given + */ + Series series( + "../samples/optionsViaJsonOverwritesAutomaticDetectionBp4.sst", Access::CREATE, R"({"adios2": {"engine": {"type": "bp4"}}})"); } REQUIRE(auxiliary::directory_exists( - "../samples/optionsViaJsonOverwritesAutomaticDetection.bp")); + "../samples/optionsViaJsonOverwritesAutomaticDetectionBp4.sst")); #if openPMD_HAVE_ADIOS1 setenv("OPENPMD_BP_BACKEND", "ADIOS1", 1); diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 4f9c2acb3c..0583ec5cf5 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1066,7 +1066,10 @@ void adios2_streaming(bool variableBasedLayout) if (rank == 0) { // write - Series writeSeries("../samples/adios2_stream.sst", Access::CREATE); + Series writeSeries( + "../samples/adios2_stream.sst", + Access::CREATE, + "adios2.engine.type = \"sst\""); if (variableBasedLayout) { writeSeries.setIterationEncoding(IterationEncoding::variableBased); @@ -1109,7 +1112,8 @@ void adios2_streaming(bool variableBasedLayout) Series readSeries( "../samples/adios2_stream.sst", Access::READ_ONLY, - "defer_iteration_parsing = true"); // inline TOML + // inline TOML + R"(defer_iteration_parsing = true)"); size_t last_iteration_index = 0; for (auto iteration : readSeries.readIterations()) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index c3543fcb7b..eac05b0214 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -55,7 +55,10 @@ std::vector testedBackends() { auto variants = getVariants(); std::map extensions{ - {"json", "json"}, {"adios1", "bp1"}, {"adios2", "bp"}, {"hdf5", "h5"}}; + {"json", "json"}, + {"adios1", "adios1.bp"}, + {"adios2", "bp"}, + {"hdf5", "h5"}}; std::vector res; for (auto const &pair : variants) { @@ -77,7 +80,9 @@ std::vector testedFileExtensions() auto allExtensions = getFileExtensions(); auto newEnd = std::remove_if( allExtensions.begin(), allExtensions.end(), [](std::string const &ext) { - return ext == "sst" || ext == "ssc"; + // sst and ssc need a receiver for testing + // bp4 is already tested via bp + return ext == "sst" || ext == "ssc" || ext == "bp4"; }); return {allExtensions.begin(), newEnd}; } @@ -279,6 +284,10 @@ TEST_CASE("write_and_read_many_iterations", "[serial]") auxiliary::remove_directory("../samples/many_iterations"); for (auto const &t : testedFileExtensions()) { + if (t == "bp4" || t == "bp5") + { + continue; + } write_and_read_many_iterations(t, intermittentFlushes); intermittentFlushes = !intermittentFlushes; } @@ -1484,7 +1493,9 @@ inline void dtype_test(const std::string &backend) TEST_CASE("dtype_test", "[serial]") { for (auto const &t : testedFileExtensions()) + { dtype_test(t); + } } inline void write_test(const std::string &backend) @@ -1604,7 +1615,8 @@ void test_complex(const std::string &backend) "../samples/serial_write_complex." + backend, Access::CREATE); o.setAttribute("lifeIsComplex", std::complex(4.56, 7.89)); o.setAttribute("butComplexFloats", std::complex(42.3, -99.3)); - if (backend != "bp") + if (o.backend() != "ADIOS2" && o.backend() != "ADIOS1" && + o.backend() != "MPI_ADIOS1") o.setAttribute( "longDoublesYouSay", std::complex(5.5, -4.55)); @@ -1625,7 +1637,8 @@ void test_complex(const std::string &backend) Cdbl.storeChunk(cdoubles, {0}); std::vector > cldoubles(3); - if (backend != "bp") + if (o.backend() != "ADIOS2" && o.backend() != "ADIOS1" && + o.backend() != "MPI_ADIOS1") { auto Cldbl = o.iterations[0].meshes["Cldbl"][RecordComponent::SCALAR]; @@ -1649,7 +1662,8 @@ void test_complex(const std::string &backend) REQUIRE( i.getAttribute("butComplexFloats").get >() == std::complex(42.3, -99.3)); - if (backend != "bp") + if (i.backend() != "ADIOS2" && i.backend() != "ADIOS1" && + i.backend() != "MPI_ADIOS1") { REQUIRE( i.getAttribute("longDoublesYouSay") @@ -1668,7 +1682,8 @@ void test_complex(const std::string &backend) REQUIRE(rcflt.get()[1] == std::complex(-3., 4.)); REQUIRE(rcdbl.get()[2] == std::complex(6, -5.)); - if (backend != "bp") + if (i.backend() != "ADIOS2" && i.backend() != "ADIOS1" && + i.backend() != "MPI_ADIOS1") { auto rcldbl = i.iterations[0] .meshes["Cldbl"][RecordComponent::SCALAR] @@ -2176,8 +2191,8 @@ inline void bool_test(const std::string &backend) Series o = Series("../samples/serial_bool." + backend, Access::CREATE); REQUIRE_THROWS_AS(o.setAuthor(""), std::runtime_error); - o.setAttribute("Bool attribute (true)", true); - o.setAttribute("Bool attribute (false)", false); + o.setAttribute("Bool attribute true", true); + o.setAttribute("Bool attribute false", false); } { Series o = @@ -2185,13 +2200,12 @@ inline void bool_test(const std::string &backend) auto attrs = o.attributes(); REQUIRE( - std::count(attrs.begin(), attrs.end(), "Bool attribute (true)") == - 1); + std::count(attrs.begin(), attrs.end(), "Bool attribute true") == 1); REQUIRE( - std::count(attrs.begin(), attrs.end(), "Bool attribute (false)") == + std::count(attrs.begin(), attrs.end(), "Bool attribute false") == 1); - REQUIRE(o.getAttribute("Bool attribute (true)").get() == true); - REQUIRE(o.getAttribute("Bool attribute (false)").get() == false); + REQUIRE(o.getAttribute("Bool attribute true").get() == true); + REQUIRE(o.getAttribute("Bool attribute false").get() == false); } { Series list{"../samples/serial_bool." + backend, Access::READ_ONLY}; @@ -4221,6 +4235,309 @@ BufferChunkSize = 2147483646 # 2^31 - 2 } #endif +TEST_CASE("adios2_engines_and_file_endings") +{ + size_t filenameCounter = 0; + auto groupbased_test_explicit_backend = + [&filenameCounter]( + std::string const &ext, + bool directory, + std::string const &requiredEngine, + std::string const &filesystemExt, + std::string const &jsonCfg = "{}") mutable { + // Env. var. OPENPMD_BP_BACKEND does not matter for this test as + // we always override it in the JSON config + auto basename = "../samples/file_endings/groupbased" + + std::to_string(filenameCounter++); + auto name = basename + ext; + std::cout << "Writing to file '" << name << "', should be engine " + << requiredEngine << " (explicit backend)." << std::endl; + auto filesystemname = + filesystemExt.empty() ? name : basename + filesystemExt; + { + Series write( + name, + Access::CREATE, + json::merge("backend = \"adios2\"", jsonCfg)); + } + if (directory) + { + REQUIRE(auxiliary::directory_exists(filesystemname)); + } + else + { + REQUIRE(auxiliary::file_exists(filesystemname)); + } + { + Series read( + name, + Access::READ_ONLY, + "backend = \"adios2\"\nadios2.engine.type = \"" + + requiredEngine + "\""); + } + }; + + groupbased_test_explicit_backend(".bp", true, "file", ""); + groupbased_test_explicit_backend(".bp4", true, "bp4", ""); + groupbased_test_explicit_backend( + ".bp", true, "bp4", "", "adios2.engine.type = \"bp4\""); + groupbased_test_explicit_backend( + ".bp4", true, "bp4", "", "adios2.engine.type = \"bp4\""); + groupbased_test_explicit_backend( + ".bp5", true, "bp4", "", "adios2.engine.type = \"bp4\""); + groupbased_test_explicit_backend( + ".bp", false, "bp3", "", "adios2.engine.type = \"bp3\""); + groupbased_test_explicit_backend( + ".sst", false, "bp3", ".sst.bp", "adios2.engine.type = \"bp3\""); + groupbased_test_explicit_backend( + "", false, "bp3", ".bp", "adios2.engine.type = \"bp3\""); + groupbased_test_explicit_backend( + "", true, "bp4", "", "adios2.engine.type = \"bp4\""); + +#ifdef ADIOS2_HAVE_BP5 + // BP5 tests + groupbased_test_explicit_backend(".bp5", true, "bp5", ""); + groupbased_test_explicit_backend( + ".bp", true, "bp5", "", "adios2.engine.type = \"bp5\""); + groupbased_test_explicit_backend( + ".bp4", true, "bp5", "", "adios2.engine.type = \"bp5\""); + groupbased_test_explicit_backend( + ".bp5", true, "bp5", "", "adios2.engine.type = \"bp5\""); + groupbased_test_explicit_backend( + "", true, "bp5", "", "adios2.engine.type = \"bp5\""); +#endif + + auto groupbased_test_no_explicit_backend = + [&filenameCounter]( + std::string const &ext, + bool directory, + std::string const &requiredEngine, + std::string const &filesystemExt, + std::string const &jsonCfg = "{}") mutable { + auto basename = "../samples/file_endings/groupbased" + + std::to_string(filenameCounter++); + auto name = basename + ext; + std::cout << "Writing to file '" << name << "', should be engine " + << requiredEngine << " (no explicit backend)." + << std::endl; + auto filesystemname = + filesystemExt.empty() ? name : basename + filesystemExt; + { + Series write(name, Access::CREATE, jsonCfg); + } + bool isThisADIOS1 = + auxiliary::getEnvString("OPENPMD_BP_BACKEND", "") == "ADIOS1" && + ext == ".bp"; + if (directory && !isThisADIOS1) + { + REQUIRE(auxiliary::directory_exists(filesystemname)); + } + else + { + REQUIRE(auxiliary::file_exists(filesystemname)); + } + { + Series read( + name, + Access::READ_ONLY, + isThisADIOS1 + ? "backend = \"adios1\"" + : "backend = \"adios2\"\nadios2.engine.type = \"" + + requiredEngine + "\""); + } + }; + + groupbased_test_no_explicit_backend(".bp", true, "file", ""); + groupbased_test_no_explicit_backend(".bp4", true, "bp4", ""); + groupbased_test_no_explicit_backend( + ".bp", true, "bp4", "", "adios2.engine.type = \"bp4\""); + groupbased_test_no_explicit_backend( + ".bp4", true, "bp4", "", "adios2.engine.type = \"bp4\""); + groupbased_test_no_explicit_backend( + ".bp5", true, "bp4", "", "adios2.engine.type = \"bp4\""); + groupbased_test_no_explicit_backend( + ".bp", false, "bp3", "", "adios2.engine.type = \"bp3\""); + groupbased_test_no_explicit_backend( + ".sst", false, "bp3", ".sst.bp", "adios2.engine.type = \"bp3\""); + REQUIRE_THROWS(groupbased_test_no_explicit_backend( + "", false, "bp3", ".bp", "adios2.engine.type = \"bp3\"")); + REQUIRE_THROWS(groupbased_test_no_explicit_backend( + "", true, "bp4", "", "adios2.engine.type = \"bp4\"")); + +#ifdef ADIOS2_HAVE_BP5 + // BP5 tests + groupbased_test_no_explicit_backend(".bp5", true, "bp5", ""); + groupbased_test_no_explicit_backend( + ".bp", true, "bp5", "", "adios2.engine.type = \"bp5\""); + groupbased_test_no_explicit_backend( + ".bp4", true, "bp5", "", "adios2.engine.type = \"bp5\""); + groupbased_test_no_explicit_backend( + ".bp5", true, "bp5", "", "adios2.engine.type = \"bp5\""); + REQUIRE_THROWS(groupbased_test_no_explicit_backend( + "", true, "bp5", "", "adios2.engine.type = \"bp5\"")); +#endif + + filenameCounter = 0; + auto filebased_test_explicit_backend = + [&filenameCounter]( + std::string const &ext, + bool directory, + std::string const &requiredEngine, + std::string const &filesystemExt, + std::string const &jsonCfg = "{}") mutable { + auto basename = "../samples/file_endings/filebased" + + std::to_string(filenameCounter++); + auto name = basename + "_%T" + ext; + std::cout << "Writing to file '" << name << "', should be engine " + << requiredEngine << " (explicit backend)." << std::endl; + auto filesystemname = + basename + "_0" + (filesystemExt.empty() ? ext : filesystemExt); + { + Series write( + name, + Access::CREATE, + json::merge("backend = \"adios2\"", jsonCfg)); + write.writeIterations()[0]; + } + if (directory) + { + REQUIRE(auxiliary::directory_exists(filesystemname)); + } + else + { + REQUIRE(auxiliary::file_exists(filesystemname)); + } + { + if (requiredEngine == "bp3" && + !auxiliary::ends_with(name, ".bp")) + { + /* + * File-based parsing procedures are not aware that BP3 + * engine adds its ending and won't find the iterations if + * we don't give it a little nudge. + */ + name += ".bp"; + } + Series read( + name, + Access::READ_ONLY, + "backend = \"adios2\"\nadios2.engine.type = \"" + + requiredEngine + "\""); + } + }; + + filebased_test_explicit_backend(".bp", true, "file", ""); + filebased_test_explicit_backend(".bp4", true, "bp4", ""); + filebased_test_explicit_backend( + ".bp", true, "bp4", "", "adios2.engine.type = \"bp4\""); + filebased_test_explicit_backend( + ".bp4", true, "bp4", "", "adios2.engine.type = \"bp4\""); + filebased_test_explicit_backend( + ".bp5", true, "bp4", "", "adios2.engine.type = \"bp4\""); + filebased_test_explicit_backend( + ".bp", false, "bp3", "", "adios2.engine.type = \"bp3\""); + filebased_test_explicit_backend( + ".sst", false, "bp3", ".sst.bp", "adios2.engine.type = \"bp3\""); + filebased_test_explicit_backend( + "", false, "bp3", ".bp", "adios2.engine.type = \"bp3\""); + filebased_test_explicit_backend( + "", true, "bp4", "", "adios2.engine.type = \"bp4\""); + +#ifdef ADIOS2_HAVE_BP5 + // BP5 tests + filebased_test_explicit_backend(".bp5", true, "bp5", ""); + filebased_test_explicit_backend( + ".bp", true, "bp5", "", "adios2.engine.type = \"bp5\""); + filebased_test_explicit_backend( + ".bp4", true, "bp5", "", "adios2.engine.type = \"bp5\""); + filebased_test_explicit_backend( + ".bp5", true, "bp5", "", "adios2.engine.type = \"bp5\""); + filebased_test_explicit_backend( + "", true, "bp5", "", "adios2.engine.type = \"bp5\""); +#endif + + auto filebased_test_no_explicit_backend = + [&filenameCounter]( + std::string const &ext, + bool directory, + std::string const &requiredEngine, + std::string const &filesystemExt, + std::string const &jsonCfg = "{}") mutable { + auto basename = "../samples/file_endings/filebased" + + std::to_string(filenameCounter++); + auto name = basename + "_%T" + ext; + std::cout << "Writing to file '" << name << "', should be engine " + << requiredEngine << " (no explicit backend)." + << std::endl; + auto filesystemname = + basename + "_0" + (filesystemExt.empty() ? ext : filesystemExt); + { + Series write(name, Access::CREATE, jsonCfg); + write.writeIterations()[0]; + } + bool isThisADIOS1 = + auxiliary::getEnvString("OPENPMD_BP_BACKEND", "") == "ADIOS1" && + ext == ".bp"; + if (directory && !isThisADIOS1) + { + REQUIRE(auxiliary::directory_exists(filesystemname)); + } + else + { + REQUIRE(auxiliary::file_exists(filesystemname)); + } + { + if (requiredEngine == "bp3" && + !auxiliary::ends_with(name, ".bp")) + { + /* + * File-based parsing procedures are not aware that BP3 + * engine adds its ending and won't find the iterations if + * we don't give it a little nudge. + */ + name += ".bp"; + } + Series read( + name, + Access::READ_ONLY, + isThisADIOS1 + ? "backend = \"adios1\"" + : "backend = \"adios2\"\nadios2.engine.type = \"" + + requiredEngine + "\""); + } + }; + + filebased_test_no_explicit_backend(".bp", true, "file", ""); + filebased_test_no_explicit_backend(".bp4", true, "bp4", ""); + filebased_test_no_explicit_backend( + ".bp", true, "bp4", "", "adios2.engine.type = \"bp4\""); + filebased_test_no_explicit_backend( + ".bp4", true, "bp4", "", "adios2.engine.type = \"bp4\""); + filebased_test_no_explicit_backend( + ".bp5", true, "bp4", "", "adios2.engine.type = \"bp4\""); + filebased_test_no_explicit_backend( + ".bp", false, "bp3", "", "adios2.engine.type = \"bp3\""); + filebased_test_no_explicit_backend( + ".sst", false, "bp3", ".sst.bp", "adios2.engine.type = \"bp3\""); + REQUIRE_THROWS(filebased_test_no_explicit_backend( + "", false, "bp3", ".bp", "adios2.engine.type = \"bp3\"")); + REQUIRE_THROWS(filebased_test_no_explicit_backend( + "", true, "bp4", "", "adios2.engine.type = \"bp4\"")); + +#ifdef ADIOS2_HAVE_BP5 + // BP5 tests + filebased_test_no_explicit_backend(".bp5", true, "bp5", ""); + filebased_test_no_explicit_backend( + ".bp", true, "bp5", "", "adios2.engine.type = \"bp5\""); + filebased_test_no_explicit_backend( + ".bp4", true, "bp5", "", "adios2.engine.type = \"bp5\""); + filebased_test_no_explicit_backend( + ".bp5", true, "bp5", "", "adios2.engine.type = \"bp5\""); + REQUIRE_THROWS(filebased_test_no_explicit_backend( + "", true, "bp5", "", "adios2.engine.type = \"bp5\"")); +#endif +} + TEST_CASE("serial_adios2_backend_config", "[serial][adios2]") { if (auxiliary::getEnvString("OPENPMD_BP_BACKEND", "NOT_SET") == "ADIOS1") @@ -5867,7 +6184,6 @@ void no_explicit_flush(std::string filename) "adios2": { "schema": 20210209, "engine": { - "type": "bp4", "usesteps": true } } @@ -6103,6 +6419,54 @@ void append_mode( */ helper::listSeries(read); } +#if 100000000 * ADIOS2_VERSION_MAJOR + 1000000 * ADIOS2_VERSION_MINOR + \ + 10000 * ADIOS2_VERSION_PATCH + 100 * ADIOS2_VERSION_TWEAK >= \ + 208002700 + // AppendAfterSteps has a bug before that version + if (extension == "bp5") + { + { + Series write( + filename, + Access::APPEND, + json::merge( + jsonConfig, + R"({"adios2":{"engine":{"parameters":{"AppendAfterSteps":-3}}}})")); + if (variableBased) + { + write.setIterationEncoding(IterationEncoding::variableBased); + } + if (write.backend() == "ADIOS1") + { + REQUIRE_THROWS_WITH( + write.flush(), + Catch::Equals( + "Operation unsupported in ADIOS1: Appending to " + "existing " + "file on disk (use Access::CREATE to overwrite)")); + // destructor will be noisy now + return; + } + + writeSomeIterations( + write.writeIterations(), std::vector{4, 5}); + write.flush(); + } + { + Series read(filename, Access::READ_ONLY); + // in variable-based encodings, iterations are not parsed ahead of + // time but as they go + unsigned counter = 0; + for (auto const &iteration : read.readIterations()) + { + REQUIRE(iteration.iterationIndex == counter); + ++counter; + } + REQUIRE(counter == 6); + helper::listSeries(read); + } + } +#endif } TEST_CASE("append_mode", "[serial]") diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 0876e2c14a..2503e8c619 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -167,7 +167,7 @@ def attributeRoundTrip(self, file_ending): series.set_attribute("longdouble", np.longdouble(1.23456789)) series.set_attribute("csingle", np.complex64(1.+2.j)) series.set_attribute("cdouble", np.complex128(3.+4.j)) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: series.set_attribute("clongdouble", np.clongdouble(5.+6.j)) # array of ... series.set_attribute("arr_int16", (np.int16(23), np.int16(26), )) @@ -202,7 +202,7 @@ def attributeRoundTrip(self, file_ending): # series.set_attribute("l_cdouble", # [np.complex_(6.7+6.8j), # np.complex_(7.1+7.2j)]) - # if file_ending != "bp": + # if file_ending not in ["bp", "bp4", "bp5"]: # series.set_attribute("l_clongdouble", # [np.clongfloat(7.8e9-6.5e9j), # np.clongfloat(8.2e3-9.1e3j)]) @@ -231,7 +231,7 @@ def attributeRoundTrip(self, file_ending): series.set_attribute("nparr_cdouble", np.array([4.5 + 1.1j, 6.7 - 2.2j], dtype=np.complex128)) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: series.set_attribute("nparr_clongdouble", np.array([8.9 + 7.8j, 7.6 + 9.2j], dtype=np.clongdouble)) @@ -286,7 +286,7 @@ def attributeRoundTrip(self, file_ending): np.complex64(1.+2.j)) self.assertAlmostEqual(series.get_attribute("cdouble"), 3.+4.j) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: self.assertAlmostEqual(series.get_attribute("clongdouble"), 5.+6.j) # array of ... (will be returned as list) @@ -315,7 +315,7 @@ def attributeRoundTrip(self, file_ending): # self.assertListEqual(series.get_attribute("l_cdouble"), # [np.complex128(6.7 + 6.8j), # np.double(7.1 + 7.2j)]) - # if file_ending != "bp": + # if file_ending not in ["bp", "bp4", "bp5"]: # self.assertListEqual(series.get_attribute("l_clongdouble"), # [np.clongdouble(7.8e9 - 6.5e9j), # np.clongdouble(8.2e3 - 9.1e3j)]) @@ -342,7 +342,8 @@ def attributeRoundTrip(self, file_ending): np.testing.assert_almost_equal( series.get_attribute("nparr_cdouble"), [4.5 + 1.1j, 6.7 - 2.2j]) - if file_ending != "bp": # not in ADIOS 1.13.1 nor ADIOS 2.7.0 + # not in ADIOS 1.13.1 nor ADIOS 2.7.0 + if file_ending not in ["bp", "bp4", "bp5"]: np.testing.assert_almost_equal( series.get_attribute("nparr_clongdouble"), [8.9 + 7.8j, 7.6 + 9.2j]) @@ -453,7 +454,7 @@ def makeConstantRoundTrip(self, file_ending): DS(np.dtype("complex128"), extent)) ms["complex128"][SCALAR].make_constant( np.complex128(1.234567 + 2.345678j)) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: ms["clongdouble"][SCALAR].reset_dataset( DS(np.dtype("clongdouble"), extent)) ms["clongdouble"][SCALAR].make_constant( @@ -534,7 +535,7 @@ def makeConstantRoundTrip(self, file_ending): == np.dtype('complex64')) self.assertTrue(ms["complex128"][SCALAR].load_chunk(o, e).dtype == np.dtype('complex128')) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: self.assertTrue(ms["clongdouble"][SCALAR].load_chunk(o, e) .dtype == np.dtype('clongdouble')) @@ -561,7 +562,7 @@ def makeConstantRoundTrip(self, file_ending): np.complex64(1.234 + 2.345j)) self.assertEqual(ms["complex128"][SCALAR].load_chunk(o, e), np.complex128(1.234567 + 2.345678j)) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: self.assertEqual(ms["clongdouble"][SCALAR].load_chunk(o, e), np.clongdouble(1.23456789 + 2.34567890j)) @@ -600,7 +601,7 @@ def makeDataRoundTrip(self, file_ending): ms["complex128"][SCALAR].store_chunk( np.ones(extent, dtype=np.complex128) * np.complex128(1.234567 + 2.345678j)) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: ms["clongdouble"][SCALAR].reset_dataset( DS(np.dtype("clongdouble"), extent)) ms["clongdouble"][SCALAR].store_chunk( @@ -630,12 +631,12 @@ def makeDataRoundTrip(self, file_ending): dc64 = ms["complex64"][SCALAR].load_chunk(o, e) dc128 = ms["complex128"][SCALAR].load_chunk(o, e) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: dc256 = ms["clongdouble"][SCALAR].load_chunk(o, e) self.assertTrue(dc64.dtype == np.dtype('complex64')) self.assertTrue(dc128.dtype == np.dtype('complex128')) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: self.assertTrue( dc256.dtype == np.dtype('clongdouble')) @@ -645,7 +646,7 @@ def makeDataRoundTrip(self, file_ending): np.complex64(1.234 + 2.345j)) self.assertEqual(dc128, np.complex128(1.234567 + 2.345678j)) - if file_ending != "bp": + if file_ending not in ["bp", "bp4", "bp5"]: self.assertEqual(dc256, np.clongdouble(1.23456789 + 2.34567890j))