From 104b3cfd8017b68bf5b3da0a09779ae2bfc18140 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Thu, 22 May 2025 15:11:11 +0000 Subject: [PATCH 01/22] Add target_include_directories to clp_s::search::ast library --- components/core/.clang-format | 6 +++--- components/core/src/clp_s/search/ast/CMakeLists.txt | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/components/core/.clang-format b/components/core/.clang-format index 626220791f..d58cdccfe6 100644 --- a/components/core/.clang-format +++ b/components/core/.clang-format @@ -4,9 +4,9 @@ IncludeCategories: # NOTE: A header is grouped by first matching regex # Library headers. Update when adding new libraries. # NOTE: clang-format retains leading white-space on a line in violation of the YAML spec. - - Regex: "<(absl|antlr4|archive|boost|bsoncxx|catch2|curl|date|fmt|log_surgeon|lzma|mongocxx\ -|msgpack|mysql|nlohmann|openssl|outcome|regex_utils|simdjson|spdlog|sqlite3|string_utils|yaml-cpp\ -|ystdlib|zstd)" + - Regex: "<(absl|antlr4|archive|boost|bsoncxx|catch2|clp_s|curl|date|fmt|log_surgeon|lzma\ +|mongocxx|msgpack|mysql|nlohmann|openssl|outcome|regex_utils|simdjson|spdlog|sqlite3|string_utils\ +|yaml-cpp|ystdlib|zstd)" Priority: 3 # C system headers - Regex: "^<.+\\.h>" diff --git a/components/core/src/clp_s/search/ast/CMakeLists.txt b/components/core/src/clp_s/search/ast/CMakeLists.txt index e46455f998..bbc19dabfe 100644 --- a/components/core/src/clp_s/search/ast/CMakeLists.txt +++ b/components/core/src/clp_s/search/ast/CMakeLists.txt @@ -63,3 +63,4 @@ target_link_libraries( PRIVATE simdjson::simdjson ) +target_include_directories(clp_s_search_ast PUBLIC ../../../) From 9791a51bdeb9547bedcf359778636b15f1028bb0 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Thu, 22 May 2025 20:43:08 +0000 Subject: [PATCH 02/22] Move FileType enum into InputConfig to remove transitive dependencies on CommandLineArguments --- components/core/src/clp_s/CommandLineArguments.hpp | 5 ----- components/core/src/clp_s/InputConfig.hpp | 8 ++++++++ components/core/src/clp_s/JsonParser.hpp | 2 +- components/core/src/clp_s/clp-s.cpp | 2 +- components/core/tests/clp_s_test_utils.cpp | 7 +++---- components/core/tests/clp_s_test_utils.hpp | 4 ++-- components/core/tests/test-clp_s-end_to_end.cpp | 2 +- components/core/tests/test-clp_s-range_index.cpp | 4 ++-- components/core/tests/test-clp_s-search.cpp | 2 +- 9 files changed, 19 insertions(+), 17 deletions(-) diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index a5c8295098..5faa9f4b99 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -36,11 +36,6 @@ class CommandLineArguments { Stdout, }; - enum class FileType : uint8_t { - Json = 0, - KeyValueIr - }; - // Constructors explicit CommandLineArguments(std::string const& program_name) : m_program_name(program_name) {} diff --git a/components/core/src/clp_s/InputConfig.hpp b/components/core/src/clp_s/InputConfig.hpp index 1d0eae35dd..658774c75b 100644 --- a/components/core/src/clp_s/InputConfig.hpp +++ b/components/core/src/clp_s/InputConfig.hpp @@ -13,6 +13,14 @@ constexpr char cAwsAccessKeyIdEnvVar[] = "AWS_ACCESS_KEY_ID"; constexpr char cAwsSecretAccessKeyEnvVar[] = "AWS_SECRET_ACCESS_KEY"; constexpr char cAwsSessionTokenEnvVar[] = "AWS_SESSION_TOKEN"; +/** + * Enum class defining various file types. +*/ +enum class FileType : uint8_t { + Json = 0, + KeyValueIr +}; + /** * Enum class defining the source of a resource. */ diff --git a/components/core/src/clp_s/JsonParser.hpp b/components/core/src/clp_s/JsonParser.hpp index aff26ff93b..3ff230f2f4 100644 --- a/components/core/src/clp_s/JsonParser.hpp +++ b/components/core/src/clp_s/JsonParser.hpp @@ -39,7 +39,7 @@ using namespace simdjson; namespace clp_s { struct JsonParserOption { std::vector input_paths; - CommandLineArguments::FileType input_file_type{CommandLineArguments::FileType::Json}; + FileType input_file_type{FileType::Json}; std::string timestamp_key; std::string archives_dir; size_t target_encoded_size{}; diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 651638bd11..a40d082978 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -107,7 +107,7 @@ bool compress(CommandLineArguments const& command_line_arguments) { option.record_log_order = command_line_arguments.get_record_log_order(); clp_s::JsonParser parser(option); - if (CommandLineArguments::FileType::KeyValueIr == option.input_file_type) { + if (clp_s::FileType::KeyValueIr == option.input_file_type) { if (false == parser.parse_from_ir()) { SPDLOG_ERROR("Encountered error while parsing input"); return false; diff --git a/components/core/tests/clp_s_test_utils.cpp b/components/core/tests/clp_s_test_utils.cpp index 243c5e55ce..ff5f45c2f5 100644 --- a/components/core/tests/clp_s_test_utils.cpp +++ b/components/core/tests/clp_s_test_utils.cpp @@ -5,7 +5,6 @@ #include -#include "../src/clp_s/CommandLineArguments.hpp" #include "../src/clp_s/InputConfig.hpp" #include "../src/clp_s/JsonParser.hpp" @@ -14,7 +13,7 @@ void compress_archive( std::string const& archive_directory, bool single_file_archive, bool structurize_arrays, - clp_s::CommandLineArguments::FileType file_type + clp_s::FileType file_type ) { constexpr auto cDefaultTargetEncodedSize{8ULL * 1024 * 1024 * 1024}; // 8 GiB constexpr auto cDefaultMaxDocumentSize{512ULL * 1024 * 1024}; // 512 MiB @@ -40,9 +39,9 @@ void compress_archive( parser_option.input_file_type = file_type; clp_s::JsonParser parser{parser_option}; - if (clp_s::CommandLineArguments::FileType::Json == file_type) { + if (clp_s::FileType::Json == file_type) { REQUIRE(parser.parse()); - } else if (clp_s::CommandLineArguments::FileType::KeyValueIr == file_type) { + } else if (clp_s::FileType::KeyValueIr == file_type) { REQUIRE(parser.parse_from_ir()); } else { // This branch should be unreachable. diff --git a/components/core/tests/clp_s_test_utils.hpp b/components/core/tests/clp_s_test_utils.hpp index f121c76548..ad3b8b30d5 100644 --- a/components/core/tests/clp_s_test_utils.hpp +++ b/components/core/tests/clp_s_test_utils.hpp @@ -2,7 +2,7 @@ #define CLP_S_TEST_UTILS_HPP #include -#include "../src/clp_s/CommandLineArguments.hpp" +#include "../src/clp_s/InputConfig.hpp" /** * Compresses a file into an archive directory according to a given set of configuration options. @@ -20,6 +20,6 @@ void compress_archive( std::string const& archive_directory, bool single_file_archive, bool structurize_arrays, - clp_s::CommandLineArguments::FileType file_type + clp_s::FileType file_type ); #endif // CLP_S_TEST_UTILS_HPP diff --git a/components/core/tests/test-clp_s-end_to_end.cpp b/components/core/tests/test-clp_s-end_to_end.cpp index 5947b305e1..82ae0bb11d 100644 --- a/components/core/tests/test-clp_s-end_to_end.cpp +++ b/components/core/tests/test-clp_s-end_to_end.cpp @@ -107,7 +107,7 @@ TEST_CASE("clp-s-compress-extract-no-floats", "[clp-s][end-to-end]") { std::string{cTestEndToEndArchiveDirectory}, single_file_archive, structurize_arrays, - clp_s::CommandLineArguments::FileType::Json + clp_s::FileType::Json )); auto extracted_json_path = extract(); diff --git a/components/core/tests/test-clp_s-range_index.cpp b/components/core/tests/test-clp_s-range_index.cpp index 447a189c2b..ab89474d2e 100644 --- a/components/core/tests/test-clp_s-range_index.cpp +++ b/components/core/tests/test-clp_s-range_index.cpp @@ -172,11 +172,11 @@ TEST_CASE("clp-s-range-index", "[clp-s][range-index]") { }; auto input_file{get_test_input_local_path()}; - auto input_file_type{clp_s::CommandLineArguments::FileType::Json}; + auto input_file_type{clp_s::FileType::Json}; if (from_ir) { generate_ir(); input_file = get_ir_test_input_relative_path(); - input_file_type = clp_s::CommandLineArguments::FileType::KeyValueIr; + input_file_type = clp_s::FileType::KeyValueIr; } REQUIRE_NOTHROW(compress_archive( input_file, diff --git a/components/core/tests/test-clp_s-search.cpp b/components/core/tests/test-clp_s-search.cpp index e92501c219..76a283f290 100644 --- a/components/core/tests/test-clp_s-search.cpp +++ b/components/core/tests/test-clp_s-search.cpp @@ -160,7 +160,7 @@ TEST_CASE("clp-s-search", "[clp-s][search]") { std::string{cTestSearchArchiveDirectory}, single_file_archive, structurize_arrays, - clp_s::CommandLineArguments::FileType::Json + clp_s::FileType::Json )); for (auto const& [query, expected_results] : queries_and_results) { From 681079468d3c177c502cb6409916d97244199468 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Fri, 23 May 2025 16:00:01 +0000 Subject: [PATCH 03/22] TMP --- components/core/src/clp_s/CMakeLists.txt | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index ad1f5f6d85..3a221e8751 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -82,6 +82,23 @@ set( ../clp/WriterInterface.hpp ) +set( + CLP_S_IO_SOURCES + +) + +set( + CLP_S_ARCHIVE_WRITER_SOURCES +) + +set( + CLP_S_ARCHIVE_READER_SOURCES +) + +set( + CLP_S_READER_UTILS_SOURCES +) + set( CLP_S_SOURCES archive_constants.hpp @@ -172,7 +189,7 @@ set( search/EvaluateTimestampIndex.hpp search/Output.cpp search/Output.hpp - search/OutputHandler.cpp + search/OutputHandler.cpp # TODO maybe don't include search/OutputHandler.hpp search/Projection.cpp search/Projection.hpp @@ -227,6 +244,7 @@ target_link_libraries( date::date spdlog::spdlog ) +target_include_directories(clp_s_TimestampPattern PUBLIC ../) add_executable( clp-s From 8f5e65a6d06db29f927698851c8e77e9939bf424 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Fri, 23 May 2025 19:22:38 +0000 Subject: [PATCH 04/22] Remove unnecessary include in JsonParser.hpp --- components/core/src/clp_s/JsonParser.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/core/src/clp_s/JsonParser.hpp b/components/core/src/clp_s/JsonParser.hpp index 3ff230f2f4..04072426ae 100644 --- a/components/core/src/clp_s/JsonParser.hpp +++ b/components/core/src/clp_s/JsonParser.hpp @@ -27,7 +27,6 @@ #include "ParsedMessage.hpp" #include "ReaderUtils.hpp" #include "Schema.hpp" -#include "SchemaMap.hpp" #include "SchemaTree.hpp" #include "SchemaWriter.hpp" #include "TimestampDictionaryWriter.hpp" From 05daa40a8acb301576830ccc909ce5f79afde49e Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Fri, 23 May 2025 19:58:17 +0000 Subject: [PATCH 05/22] Move try_create_reader from ReaderUtils to InputConfig to remove transient dependencies to many archive read-side components --- .../core/src/clp_s/ArchiveReaderAdaptor.cpp | 3 +- components/core/src/clp_s/InputConfig.cpp | 91 +++++++++++++++++++ components/core/src/clp_s/InputConfig.hpp | 14 ++- components/core/src/clp_s/JsonParser.cpp | 5 +- components/core/src/clp_s/JsonParser.hpp | 2 - components/core/src/clp_s/ReaderUtils.cpp | 90 ------------------ components/core/src/clp_s/ReaderUtils.hpp | 13 +-- components/core/src/clp_s/indexer/indexer.cpp | 1 - components/core/src/clp_s/kv_ir_search.cpp | 3 +- 9 files changed, 111 insertions(+), 111 deletions(-) diff --git a/components/core/src/clp_s/ArchiveReaderAdaptor.cpp b/components/core/src/clp_s/ArchiveReaderAdaptor.cpp index ba8dc262d0..e70f3f865d 100644 --- a/components/core/src/clp_s/ArchiveReaderAdaptor.cpp +++ b/components/core/src/clp_s/ArchiveReaderAdaptor.cpp @@ -18,7 +18,6 @@ #include "archive_constants.hpp" #include "InputConfig.hpp" #include "RangeIndexWriter.hpp" -#include "ReaderUtils.hpp" #include "SingleFileArchiveDefs.hpp" namespace clp_s { @@ -261,7 +260,7 @@ std::shared_ptr ArchiveReaderAdaptor::try_create_reader_at return nullptr; } } else { - return ReaderUtils::try_create_reader(m_archive_path, m_network_auth); + return try_create_reader(m_archive_path, m_network_auth); } } diff --git a/components/core/src/clp_s/InputConfig.cpp b/components/core/src/clp_s/InputConfig.cpp index b21efdd186..61fe5415a2 100644 --- a/components/core/src/clp_s/InputConfig.cpp +++ b/components/core/src/clp_s/InputConfig.cpp @@ -1,10 +1,19 @@ #include "InputConfig.hpp" +#include #include #include +#include #include #include +#include + +#include "../clp/aws/AwsAuthenticationSigner.hpp" +#include "../clp/FileReader.hpp" +#include "../clp/NetworkReader.hpp" +#include "../clp/ReaderInterface.hpp" +#include "../clp/spdlog_with_specializations.hpp" #include "Utils.hpp" namespace clp_s { @@ -84,4 +93,86 @@ auto get_archive_id_from_path(Path const& archive_path, std::string& archive_id) } return true; } + +namespace { +auto try_create_file_reader(std::string_view const file_path) + -> std::shared_ptr { + try { + return std::make_shared(std::string{file_path}); + } catch (clp::FileReader::OperationFailed const& e) { + SPDLOG_ERROR("Failed to open file for reading - {} - {}", file_path, e.what()); + return nullptr; + } +} + +auto try_sign_url(std::string& url) -> bool { + auto const aws_access_key = std::getenv(cAwsAccessKeyIdEnvVar); + auto const aws_secret_access_key = std::getenv(cAwsSecretAccessKeyEnvVar); + if (nullptr == aws_access_key || nullptr == aws_secret_access_key) { + SPDLOG_ERROR( + "{} and {} environment variables not available for presigned url authentication.", + cAwsAccessKeyIdEnvVar, + cAwsSecretAccessKeyEnvVar + ); + return false; + } + std::optional optional_aws_session_token{std::nullopt}; + auto const aws_session_token = std::getenv(cAwsSessionTokenEnvVar); + if (nullptr != aws_session_token) { + optional_aws_session_token = std::string{aws_session_token}; + } + + clp::aws::AwsAuthenticationSigner signer{ + aws_access_key, + aws_secret_access_key, + optional_aws_session_token + }; + + try { + clp::aws::S3Url s3_url{url}; + if (auto const rc = signer.generate_presigned_url(s3_url, url); + clp::ErrorCode::ErrorCode_Success != rc) + { + return false; + } + } catch (std::exception const& e) { + return false; + } + return true; +} + +auto try_create_network_reader(std::string_view const url, NetworkAuthOption const& auth) + -> std::shared_ptr { + std::string request_url{url}; + switch (auth.method) { + case AuthMethod::S3PresignedUrlV4: + if (false == try_sign_url(request_url)) { + return nullptr; + } + break; + case AuthMethod::None: + break; + default: + return nullptr; + } + + try { + return std::make_shared(request_url); + } catch (clp::NetworkReader::OperationFailed const& e) { + SPDLOG_ERROR("Failed to open url for reading - {}", e.what()); + return nullptr; + } +} +} // namespace + +auto try_create_reader(Path const& path, NetworkAuthOption const& network_auth) + -> std::shared_ptr { + if (InputSource::Filesystem == path.source) { + return try_create_file_reader(path.path); + } else if (InputSource::Network == path.source) { + return try_create_network_reader(path.path, network_auth); + } else { + return nullptr; + } +} } // namespace clp_s diff --git a/components/core/src/clp_s/InputConfig.hpp b/components/core/src/clp_s/InputConfig.hpp index 658774c75b..e8df89ca4e 100644 --- a/components/core/src/clp_s/InputConfig.hpp +++ b/components/core/src/clp_s/InputConfig.hpp @@ -2,11 +2,14 @@ #define CLP_S_INPUTCONFIG_HPP #include +#include #include #include #include #include +#include "../clp/ReaderInterface.hpp" + namespace clp_s { // Constants used for input configuration constexpr char cAwsAccessKeyIdEnvVar[] = "AWS_ACCESS_KEY_ID"; @@ -15,7 +18,7 @@ constexpr char cAwsSessionTokenEnvVar[] = "AWS_SESSION_TOKEN"; /** * Enum class defining various file types. -*/ + */ enum class FileType : uint8_t { Json = 0, KeyValueIr @@ -109,6 +112,15 @@ get_input_archives_for_raw_path(std::string_view const path, std::vector& */ [[nodiscard]] auto get_archive_id_from_path(Path const& archive_path, std::string& archive_id) -> bool; + +/** + * Tries to open a clp::ReaderInterface using the given Path and NetworkAuthOption. + * @param path + * @param network_auth + * @return the opened clp::ReaderInterface or nullptr on error + */ +[[nodiscard]] auto try_create_reader(Path const& path, NetworkAuthOption const& network_auth) + -> std::shared_ptr; } // namespace clp_s #endif // CLP_S_INPUTCONFIG_HPP diff --git a/components/core/src/clp_s/JsonParser.cpp b/components/core/src/clp_s/JsonParser.cpp index 603ad58e6e..04d74ed256 100644 --- a/components/core/src/clp_s/JsonParser.cpp +++ b/components/core/src/clp_s/JsonParser.cpp @@ -28,6 +28,7 @@ #include "../clp/time_types.hpp" #include "archive_constants.hpp" #include "ErrorCode.hpp" +#include "InputConfig.hpp" #include "JsonFileIterator.hpp" #include "JsonParser.hpp" #include "search/ast/ColumnDescriptor.hpp" @@ -484,7 +485,7 @@ void JsonParser::parse_line(ondemand::value line, int32_t parent_node_id, std::s bool JsonParser::parse() { auto archive_creator_id = boost::uuids::to_string(m_generator()); for (auto const& path : m_input_paths) { - auto reader{ReaderUtils::try_create_reader(path, m_network_auth)}; + auto reader{try_create_reader(path, m_network_auth)}; if (nullptr == reader) { m_archive_writer->close(); return false; @@ -947,7 +948,7 @@ auto JsonParser::parse_from_ir() -> bool { constexpr size_t cDecompressorReadBufferCapacity{64 * 1024}; // 64 KB auto archive_creator_id = boost::uuids::to_string(m_generator()); for (auto const& path : m_input_paths) { - auto reader{ReaderUtils::try_create_reader(path, m_network_auth)}; + auto reader{try_create_reader(path, m_network_auth)}; if (nullptr == reader) { m_archive_writer->close(); return false; diff --git a/components/core/src/clp_s/JsonParser.hpp b/components/core/src/clp_s/JsonParser.hpp index 04072426ae..51cf397e95 100644 --- a/components/core/src/clp_s/JsonParser.hpp +++ b/components/core/src/clp_s/JsonParser.hpp @@ -19,13 +19,11 @@ #include "../clp/ffi/Value.hpp" #include "../clp/ReaderInterface.hpp" #include "ArchiveWriter.hpp" -#include "CommandLineArguments.hpp" #include "DictionaryWriter.hpp" #include "FileReader.hpp" #include "FileWriter.hpp" #include "InputConfig.hpp" #include "ParsedMessage.hpp" -#include "ReaderUtils.hpp" #include "Schema.hpp" #include "SchemaTree.hpp" #include "SchemaWriter.hpp" diff --git a/components/core/src/clp_s/ReaderUtils.cpp b/components/core/src/clp_s/ReaderUtils.cpp index b0f3e31820..a751671fbd 100644 --- a/components/core/src/clp_s/ReaderUtils.cpp +++ b/components/core/src/clp_s/ReaderUtils.cpp @@ -1,18 +1,9 @@ #include "ReaderUtils.hpp" -#include #include #include -#include - -#include "../clp/aws/AwsAuthenticationSigner.hpp" -#include "../clp/FileReader.hpp" -#include "../clp/NetworkReader.hpp" -#include "../clp/ReaderInterface.hpp" -#include "../clp/spdlog_with_specializations.hpp" #include "archive_constants.hpp" -#include "Utils.hpp" namespace clp_s { std::shared_ptr ReaderUtils::read_schema_tree(ArchiveReaderAdaptor& adaptor) { @@ -143,85 +134,4 @@ std::shared_ptr ReaderUtils::read_schemas(ArchiveReaderA return schemas_pointer; } - -namespace { -std::shared_ptr try_create_file_reader(std::string_view const file_path) { - try { - return std::make_shared(std::string{file_path}); - } catch (clp::FileReader::OperationFailed const& e) { - SPDLOG_ERROR("Failed to open file for reading - {} - {}", file_path, e.what()); - return nullptr; - } -} - -bool try_sign_url(std::string& url) { - auto const aws_access_key = std::getenv(cAwsAccessKeyIdEnvVar); - auto const aws_secret_access_key = std::getenv(cAwsSecretAccessKeyEnvVar); - if (nullptr == aws_access_key || nullptr == aws_secret_access_key) { - SPDLOG_ERROR( - "{} and {} environment variables not available for presigned url authentication.", - cAwsAccessKeyIdEnvVar, - cAwsSecretAccessKeyEnvVar - ); - return false; - } - std::optional optional_aws_session_token{std::nullopt}; - auto const aws_session_token = std::getenv(cAwsSessionTokenEnvVar); - if (nullptr != aws_session_token) { - optional_aws_session_token = std::string{aws_session_token}; - } - - clp::aws::AwsAuthenticationSigner signer{ - aws_access_key, - aws_secret_access_key, - optional_aws_session_token - }; - - try { - clp::aws::S3Url s3_url{url}; - if (auto const rc = signer.generate_presigned_url(s3_url, url); - clp::ErrorCode::ErrorCode_Success != rc) - { - return false; - } - } catch (std::exception const& e) { - return false; - } - return true; -} - -std::shared_ptr -try_create_network_reader(std::string_view const url, NetworkAuthOption const& auth) { - std::string request_url{url}; - switch (auth.method) { - case AuthMethod::S3PresignedUrlV4: - if (false == try_sign_url(request_url)) { - return nullptr; - } - break; - case AuthMethod::None: - break; - default: - return nullptr; - } - - try { - return std::make_shared(request_url); - } catch (clp::NetworkReader::OperationFailed const& e) { - SPDLOG_ERROR("Failed to open url for reading - {}", e.what()); - return nullptr; - } -} -} // namespace - -std::shared_ptr -ReaderUtils::try_create_reader(Path const& path, NetworkAuthOption const& network_auth) { - if (InputSource::Filesystem == path.source) { - return try_create_file_reader(path.path); - } else if (InputSource::Network == path.source) { - return try_create_network_reader(path.path, network_auth); - } else { - return nullptr; - } -} } // namespace clp_s diff --git a/components/core/src/clp_s/ReaderUtils.hpp b/components/core/src/clp_s/ReaderUtils.hpp index bebf110acd..c86c3b9c47 100644 --- a/components/core/src/clp_s/ReaderUtils.hpp +++ b/components/core/src/clp_s/ReaderUtils.hpp @@ -1,12 +1,12 @@ #ifndef CLP_S_READERUTILS_HPP #define CLP_S_READERUTILS_HPP +#include +#include #include -#include "../clp/ReaderInterface.hpp" #include "ArchiveReaderAdaptor.hpp" #include "DictionaryReader.hpp" -#include "InputConfig.hpp" #include "Schema.hpp" #include "SchemaReader.hpp" #include "SchemaTree.hpp" @@ -67,15 +67,6 @@ class ReaderUtils { ArchiveReaderAdaptor& adaptor ); - /** - * Tries to open a clp::ReaderInterface using the given Path and NetworkAuthOption. - * @param path - * @param network_auth - * @return the opened clp::ReaderInterface or nullptr on error - */ - static std::shared_ptr - try_create_reader(Path const& path, NetworkAuthOption const& network_auth); - private: /** * Appends a column to the given schema reader diff --git a/components/core/src/clp_s/indexer/indexer.cpp b/components/core/src/clp_s/indexer/indexer.cpp index 76797d507d..4883e32a1f 100644 --- a/components/core/src/clp_s/indexer/indexer.cpp +++ b/components/core/src/clp_s/indexer/indexer.cpp @@ -4,7 +4,6 @@ #include #include -#include "../ReaderUtils.hpp" #include "CommandLineArguments.hpp" #include "IndexManager.hpp" diff --git a/components/core/src/clp_s/kv_ir_search.cpp b/components/core/src/clp_s/kv_ir_search.cpp index 9449fad41d..cd5d61b96f 100644 --- a/components/core/src/clp_s/kv_ir_search.cpp +++ b/components/core/src/clp_s/kv_ir_search.cpp @@ -27,7 +27,6 @@ #include "../clp/TraceableException.hpp" #include "CommandLineArguments.hpp" #include "InputConfig.hpp" -#include "ReaderUtils.hpp" #include "search/ast/Expression.hpp" // This include has a circular dependency with the `.inc` file. @@ -250,7 +249,7 @@ auto search_kv_ir_stream( } auto const raw_reader{ - ReaderUtils::try_create_reader(stream_path, command_line_arguments.get_network_auth()) + try_create_reader(stream_path, command_line_arguments.get_network_auth()) }; if (nullptr == raw_reader) { return KvIrSearchError{KvIrSearchErrorEnum::StreamReaderCreationFailure}; From 115020bdcb723d26099a43b25d4c6d08504d6fbd Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 26 May 2025 15:14:59 +0000 Subject: [PATCH 06/22] Remove unnecessary simdjson includes --- components/core/src/clp_s/ColumnWriter.hpp | 4 ---- components/core/src/clp_s/VariableEncoder.hpp | 4 ---- 2 files changed, 8 deletions(-) diff --git a/components/core/src/clp_s/ColumnWriter.hpp b/components/core/src/clp_s/ColumnWriter.hpp index 7282cf7ea7..64b79f960e 100644 --- a/components/core/src/clp_s/ColumnWriter.hpp +++ b/components/core/src/clp_s/ColumnWriter.hpp @@ -4,8 +4,6 @@ #include #include -#include - #include "DictionaryWriter.hpp" #include "FileWriter.hpp" #include "ParsedMessage.hpp" @@ -13,8 +11,6 @@ #include "VariableEncoder.hpp" #include "ZstdCompressor.hpp" -using namespace simdjson; - namespace clp_s { class BaseColumnWriter { public: diff --git a/components/core/src/clp_s/VariableEncoder.hpp b/components/core/src/clp_s/VariableEncoder.hpp index d604b7c0d2..90c6edd61c 100644 --- a/components/core/src/clp_s/VariableEncoder.hpp +++ b/components/core/src/clp_s/VariableEncoder.hpp @@ -5,13 +5,9 @@ #include -#include - #include "DictionaryEntry.hpp" #include "DictionaryWriter.hpp" -using namespace simdjson; - namespace clp_s { class VariableEncoder { public: From 78c090eef5b67de59dcd978a5437197f67b0cd23 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 26 May 2025 15:15:22 +0000 Subject: [PATCH 07/22] Remove unnecessary includes from JsonConstructor --- components/core/src/clp_s/JsonConstructor.cpp | 2 -- components/core/src/clp_s/JsonConstructor.hpp | 5 ----- 2 files changed, 7 deletions(-) diff --git a/components/core/src/clp_s/JsonConstructor.cpp b/components/core/src/clp_s/JsonConstructor.cpp index b4995f4cd4..8873722eb9 100644 --- a/components/core/src/clp_s/JsonConstructor.cpp +++ b/components/core/src/clp_s/JsonConstructor.cpp @@ -13,8 +13,6 @@ #include "archive_constants.hpp" #include "ErrorCode.hpp" -#include "ReaderUtils.hpp" -#include "SchemaTree.hpp" #include "TraceableException.hpp" namespace clp_s { diff --git a/components/core/src/clp_s/JsonConstructor.hpp b/components/core/src/clp_s/JsonConstructor.hpp index 349dcf47d6..607d5e8349 100644 --- a/components/core/src/clp_s/JsonConstructor.hpp +++ b/components/core/src/clp_s/JsonConstructor.hpp @@ -7,13 +7,8 @@ #include #include "ArchiveReader.hpp" -#include "ColumnReader.hpp" -#include "DictionaryReader.hpp" #include "ErrorCode.hpp" -#include "FileWriter.hpp" #include "InputConfig.hpp" -#include "SchemaReader.hpp" -#include "SchemaTree.hpp" #include "TraceableException.hpp" namespace clp_s { From 116cfebcdcda0883ecb2b1dbe43b190fc5ae4f42 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 26 May 2025 15:16:37 +0000 Subject: [PATCH 08/22] Rename sql and kql libraries to clp_s::search::sql and clp_s::search::kql --- components/core/CMakeLists.txt | 4 ++-- components/core/src/clp_s/search/kql/CMakeLists.txt | 9 +++++---- components/core/src/clp_s/search/sql/CMakeLists.txt | 9 +++++---- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 98a33b16a0..1f8d5a18e8 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -685,10 +685,11 @@ target_link_libraries(unitTest Catch2::Catch2 ${CURL_LIBRARIES} clp_s::search::ast + clp_s::search::kql + clp_s::search::sql clp_s::TimestampPattern date::date fmt::fmt - kql log_surgeon::log_surgeon LibArchive::LibArchive MariaDBClient::MariaDBClient @@ -696,7 +697,6 @@ target_link_libraries(unitTest nlohmann_json::nlohmann_json simdjson::simdjson spdlog::spdlog - sql OpenSSL::Crypto ${sqlite_LIBRARY_DEPENDENCIES} ${STD_FS_LIBS} diff --git a/components/core/src/clp_s/search/kql/CMakeLists.txt b/components/core/src/clp_s/search/kql/CMakeLists.txt index 4b150c130e..2a661d7a61 100644 --- a/components/core/src/clp_s/search/kql/CMakeLists.txt +++ b/components/core/src/clp_s/search/kql/CMakeLists.txt @@ -6,16 +6,17 @@ ANTLR_TARGET( ) add_library( - kql + clp_s_search_kql ../antlr_common/ErrorListener.hpp ${ANTLR_KqlParser_CXX_OUTPUTS} kql.cpp kql.hpp ) -target_compile_features(kql PRIVATE cxx_std_20) -target_include_directories(kql PRIVATE ${ANTLR_KqlParser_OUTPUT_DIR}) +add_library(clp_s::search::kql ALIAS clp_s_search_kql) +target_compile_features(clp_s_search_kql PRIVATE cxx_std_20) +target_include_directories(clp_s_search_kql PRIVATE ${ANTLR_KqlParser_OUTPUT_DIR}) target_link_libraries( - kql + clp_s_search_kql PRIVATE antlr4_static clp::string_utils diff --git a/components/core/src/clp_s/search/sql/CMakeLists.txt b/components/core/src/clp_s/search/sql/CMakeLists.txt index 0ab9493c16..672a221b9e 100644 --- a/components/core/src/clp_s/search/sql/CMakeLists.txt +++ b/components/core/src/clp_s/search/sql/CMakeLists.txt @@ -6,16 +6,17 @@ ANTLR_TARGET( ) add_library( - sql + clp_s_search_sql ../antlr_common/ErrorListener.hpp ${ANTLR_SqlParser_CXX_OUTPUTS} sql.cpp sql.hpp ) -target_compile_features(sql PRIVATE cxx_std_20) -target_include_directories(sql PRIVATE ${ANTLR_SqlParser_OUTPUT_DIR}) +add_library(clp_s::search::sql ALIAS clp_s_search_sql) +target_compile_features(clp_s_search_sql PRIVATE cxx_std_20) +target_include_directories(clp_s_search_sql PRIVATE ${ANTLR_SqlParser_OUTPUT_DIR}) target_link_libraries( - sql + clp_s_search_sql PRIVATE antlr4_static clp_s::search::ast From 38f7027ae6c978fa7dff9c58a0bcecc02f41f620 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 26 May 2025 15:17:26 +0000 Subject: [PATCH 09/22] Split clp-s build into many smaller libraries. --- components/core/src/clp_s/CMakeLists.txt | 329 +++++++++++++++++------ 1 file changed, 247 insertions(+), 82 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 3a221e8751..4608e784a2 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -4,7 +4,7 @@ add_subdirectory(search/kql) add_subdirectory(search/sql) set( - CLP_SOURCES + CLP_S_CLP_SOURCES ../clp/aws/AwsAuthenticationSigner.cpp ../clp/aws/AwsAuthenticationSigner.hpp ../clp/BoundedReader.cpp @@ -82,65 +82,127 @@ set( ../clp/WriterInterface.hpp ) -set( - CLP_S_IO_SOURCES - +# This library is intended as a temporary stand-in until clp has been packaged into libraries. +add_library( + clp_s_clp_dependencies + ${CLP_S_CLP_SOURCES} +) +add_library(clp_s::clp_dependencies ALIAS clp_s_clp_dependencies) +target_compile_features(clp_s_clp_dependencies PRIVATE cxx_std_20) +target_include_directories( + clp_s_clp_dependencies + PUBLIC + "${CLP_OUTCOME_INCLUDE_DIRECTORY}" +) +target_link_libraries( + clp_s_clp_dependencies + PUBLIC + clp::string_utils + ystdlib::containers + PRIVATE + Boost::regex + ${CURL_LIBRARIES} + fmt::fmt + spdlog::spdlog + nlohmann_json::nlohmann_json + OpenSSL::Crypto + ystdlib::error_handling + ZStd::ZStd ) set( - CLP_S_ARCHIVE_WRITER_SOURCES + CLP_S_REDUCER_SOURCES + ../reducer/BufferedSocketWriter.cpp + ../reducer/BufferedSocketWriter.hpp + ../reducer/ConstRecordIterator.hpp + ../reducer/CountOperator.cpp + ../reducer/CountOperator.hpp + ../reducer/DeserializedRecordGroup.cpp + ../reducer/DeserializedRecordGroup.hpp + ../reducer/GroupTags.hpp + ../reducer/network_utils.cpp + ../reducer/network_utils.hpp + ../reducer/Operator.cpp + ../reducer/Operator.hpp + ../reducer/Pipeline.cpp + ../reducer/Pipeline.hpp + ../reducer/Record.hpp + ../reducer/RecordGroup.hpp + ../reducer/RecordGroupIterator.hpp + ../reducer/RecordTypedKeyIterator.hpp + ../reducer/types.hpp ) -set( - CLP_S_ARCHIVE_READER_SOURCES +# This library is intended as a temporary stand-in until the reducer has been packaged into +# libraries. +add_library( + clp_s_reducer_dependencies + ${CLP_S_REDUCER_SOURCES} +) +add_library(clp_s::reducer_dependencies ALIAS clp_s_reducer_dependencies) +target_compile_features(clp_s_reducer_dependencies PRIVATE cxx_std_20) +target_link_libraries( + clp_s_reducer_dependencies + PUBLIC + nlohmann_json::nlohmann_json + PRIVATE + clp_s::clp_dependencies ) set( - CLP_S_READER_UTILS_SOURCES + CLP_S_IO_SOURCES + Compressor.hpp + Decompressor.hpp + ErrorCode.hpp + FileReader.cpp + FileReader.hpp + FileWriter.cpp + FileWriter.hpp + InputConfig.cpp + InputConfig.hpp + TraceableException.hpp + Utils.cpp + Utils.hpp + ZstdCompressor.cpp + ZstdCompressor.hpp + ZstdDecompressor.cpp + ZstdDecompressor.hpp ) +add_library( + clp_s_IO + ${CLP_S_IO_SOURCES} +) +add_library(clp_s::IO ALIAS clp_s_IO) +target_compile_features(clp_s_IO PRIVATE cxx_std_20) +target_link_libraries( + clp_s_IO + PRIVATE + Boost::iostreams Boost::url + clp_s::clp_dependencies + fmt::fmt + spdlog::spdlog + ZStd::ZStd +) +target_include_directories(clp_s_IO PUBLIC ../) + set( - CLP_S_SOURCES + CLP_S_ARCHIVE_WRITER_SOURCES archive_constants.hpp - ArchiveReader.cpp - ArchiveReader.hpp - ArchiveReaderAdaptor.cpp - ArchiveReaderAdaptor.hpp ArchiveWriter.cpp ArchiveWriter.hpp - BufferViewReader.hpp - ColumnReader.cpp - ColumnReader.hpp ColumnWriter.cpp ColumnWriter.hpp - CommandLineArguments.cpp - CommandLineArguments.hpp - Compressor.hpp - Decompressor.hpp Defs.hpp DictionaryEntry.cpp DictionaryEntry.hpp - DictionaryReader.hpp DictionaryWriter.cpp DictionaryWriter.hpp ErrorCode.hpp - FileReader.cpp - FileReader.hpp - FileWriter.cpp - FileWriter.hpp - InputConfig.cpp - InputConfig.hpp - JsonConstructor.cpp - JsonConstructor.hpp JsonFileIterator.cpp JsonFileIterator.hpp JsonParser.cpp JsonParser.hpp - JsonSerializer.hpp - kv_ir_search.cpp - kv_ir_search.hpp - PackedStreamReader.cpp - PackedStreamReader.hpp ParsedMessage.hpp RangeIndexWriter.cpp RangeIndexWriter.hpp @@ -150,14 +212,10 @@ set( Schema.hpp SchemaMap.cpp SchemaMap.hpp - SchemaReader.cpp - SchemaReader.hpp SchemaTree.cpp SchemaTree.hpp SchemaWriter.cpp SchemaWriter.hpp - TimestampDictionaryReader.cpp - TimestampDictionaryReader.hpp TimestampDictionaryWriter.cpp TimestampDictionaryWriter.hpp TimestampEntry.cpp @@ -165,18 +223,132 @@ set( TraceableException.hpp Utils.cpp Utils.hpp - VariableDecoder.cpp - VariableDecoder.hpp VariableEncoder.cpp VariableEncoder.hpp - ZstdCompressor.cpp - ZstdCompressor.hpp - ZstdDecompressor.cpp - ZstdDecompressor.hpp +) + +add_library( + clp_s_ArchiveWriter + ${CLP_S_ARCHIVE_WRITER_SOURCES} +) +add_library(clp_s::ArchiveWriter ALIAS clp_s_ArchiveWriter) +target_compile_features(clp_s_ArchiveWriter PRIVATE cxx_std_20) +target_link_libraries( + clp_s_ArchiveWriter + PUBLIC + absl::flat_hash_map + simdjson::simdjson + PRIVATE + Boost::url + clp_s::clp_dependencies + clp_s::IO + clp_s::TimestampPattern + ${CURL_LIBRARIES} + fmt::fmt + msgpack-cxx + nlohmann_json::nlohmann_json + spdlog::spdlog +) +target_include_directories(clp_s_ArchiveWriter PUBLIC ../) + +set( + CLP_S_ARCHIVE_READER_SOURCES + archive_constants.hpp + ArchiveReader.cpp + ArchiveReader.hpp + ArchiveReaderAdaptor.cpp + ArchiveReaderAdaptor.hpp + BufferViewReader.hpp + ColumnReader.cpp + ColumnReader.hpp + Defs.hpp + DictionaryEntry.cpp + DictionaryEntry.hpp + DictionaryReader.hpp + ErrorCode.hpp + JsonSerializer.hpp + PackedStreamReader.cpp + PackedStreamReader.hpp + ReaderUtils.cpp + ReaderUtils.hpp + Schema.cpp + Schema.hpp + SchemaReader.cpp + SchemaReader.hpp + SchemaTree.cpp + SchemaTree.hpp + TimestampDictionaryReader.cpp + TimestampDictionaryReader.hpp + TimestampEntry.cpp + TimestampEntry.hpp + TraceableException.hpp + Utils.cpp + Utils.hpp + VariableDecoder.cpp + VariableDecoder.hpp +) + +add_library( + clp_s_ArchiveReader + ${CLP_S_ARCHIVE_READER_SOURCES} +) +add_library(clp_s::ArchiveReader ALIAS clp_s_ArchiveReader) +target_compile_features(clp_s_ArchiveReader PRIVATE cxx_std_20) +target_link_libraries( + clp_s_ArchiveReader + PUBLIC + absl::flat_hash_map + nlohmann_json::nlohmann_json + PRIVATE + absl::flat_hash_map + Boost::url + clp_s::clp_dependencies + clp_s::IO + clp_s::TimestampPattern + ${CURL_LIBRARIES} + fmt::fmt + msgpack-cxx + spdlog::spdlog +) +target_include_directories(clp_s_ArchiveReader PUBLIC ../) + +set( + CLP_S_JSON_CONSTRUCTOR_SOURCES + ErrorCode.hpp + JsonConstructor.cpp + JsonConstructor.hpp + TraceableException.hpp +) + +add_library( + clp_s_JsonConstructor + ${CLP_S_JSON_CONSTRUCTOR_SOURCES} +) +add_library(clp_s::JsonConstructor ALIAS clp_s_JsonConstructor) +target_compile_features(clp_s_JsonConstructor PRIVATE cxx_std_20) +target_link_libraries( + clp_s_JsonConstructor + PUBLIC + clp_s::ArchiveReader + PRIVATE + fmt::fmt + ${MONGOCXX_TARGET} +) +target_include_directories(clp_s_JsonConstructor PUBLIC ../) + +set( + CLP_S_SOURCES + CommandLineArguments.cpp + CommandLineArguments.hpp + ErrorCode.hpp + kv_ir_search.cpp + kv_ir_search.hpp + TraceableException.hpp ) set( CLP_S_SEARCH_SOURCES + Defs.hpp search/AddTimestampConditions.cpp search/AddTimestampConditions.hpp search/clp_search/EncodedVariableInterpreter.cpp @@ -197,29 +369,30 @@ set( search/QueryRunner.hpp search/SchemaMatch.cpp search/SchemaMatch.hpp + VariableEncoder.cpp + VariableEncoder.hpp ) -set( - REDUCER_SOURCES - ../reducer/BufferedSocketWriter.cpp - ../reducer/BufferedSocketWriter.hpp - ../reducer/ConstRecordIterator.hpp - ../reducer/CountOperator.cpp - ../reducer/CountOperator.hpp - ../reducer/DeserializedRecordGroup.cpp - ../reducer/DeserializedRecordGroup.hpp - ../reducer/GroupTags.hpp - ../reducer/network_utils.cpp - ../reducer/network_utils.hpp - ../reducer/Operator.cpp - ../reducer/Operator.hpp - ../reducer/Pipeline.cpp - ../reducer/Pipeline.hpp - ../reducer/Record.hpp - ../reducer/RecordGroup.hpp - ../reducer/RecordGroupIterator.hpp - ../reducer/RecordTypedKeyIterator.hpp - ../reducer/types.hpp +add_library( + clp_s_search + ${CLP_S_SEARCH_SOURCES} +) +add_library(clp_s::search ALIAS clp_s_search) +target_compile_features(clp_s_search PRIVATE cxx_std_20) +target_link_libraries( + clp_s_search + PUBLIC + absl::flat_hash_map + clp_s::ArchiveReader + clp_s::search::ast + PRIVATE + spdlog::spdlog + simdjson::simdjson + + clp_s::clp_dependencies + clp_s::reducer_dependencies + ${MONGOCXX_TARGET} + msgpack-cxx ) set( @@ -249,33 +422,25 @@ target_include_directories(clp_s_TimestampPattern PUBLIC ../) add_executable( clp-s clp-s.cpp - ${CLP_SOURCES} ${CLP_S_SOURCES} - ${CLP_S_SEARCH_SOURCES} - ${REDUCER_SOURCES} ) target_compile_features(clp-s PRIVATE cxx_std_20) target_link_libraries( clp-s PRIVATE - absl::flat_hash_map - Boost::iostreams Boost::program_options Boost::regex Boost::url - ${CURL_LIBRARIES} - clp::string_utils + Boost::program_options + clp_s::ArchiveReader + clp_s::ArchiveWriter + clp_s::clp_dependencies + clp_s::IO + clp_s::JsonConstructor + clp_s::reducer_dependencies + clp_s::search clp_s::search::ast - clp_s::TimestampPattern - kql - MariaDBClient::MariaDBClient + clp_s::search::kql ${MONGOCXX_TARGET} - msgpack-cxx - nlohmann_json::nlohmann_json - OpenSSL::Crypto - simdjson::simdjson spdlog::spdlog - yaml-cpp - ystdlib::containers ystdlib::error_handling - ZStd::ZStd ) target_include_directories(clp-s PRIVATE From fe104efbd73d1d6ad424005f619ba516cd0f861c Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 26 May 2025 16:03:09 +0000 Subject: [PATCH 10/22] Decouple OutputHandler and OutputHandler implementations --- components/core/CMakeLists.txt | 3 +- components/core/src/clp_s/CMakeLists.txt | 30 +- ...utputHandler.cpp => OutputHandlerImpl.cpp} | 29 +- .../core/src/clp_s/OutputHandlerImpl.hpp | 289 +++++++++++++++++ components/core/src/clp_s/clp-s.cpp | 11 +- .../core/src/clp_s/search/OutputHandler.hpp | 297 +----------------- components/core/tests/test-clp_s-search.cpp | 10 +- taskfiles/lint.yaml | 4 +- 8 files changed, 344 insertions(+), 329 deletions(-) rename components/core/src/clp_s/{search/OutputHandler.cpp => OutputHandlerImpl.cpp} (89%) create mode 100644 components/core/src/clp_s/OutputHandlerImpl.hpp diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 1f8d5a18e8..51f29659cb 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -315,6 +315,8 @@ set(SOURCE_FILES_clp_s_unitTest src/clp_s/JsonFileIterator.hpp src/clp_s/JsonParser.cpp src/clp_s/JsonParser.hpp + src/clp_s/OutputHandlerImpl.cpp + src/clp_s/OutputHandlerImpl.hpp src/clp_s/PackedStreamReader.cpp src/clp_s/PackedStreamReader.hpp src/clp_s/RangeIndexWriter.cpp @@ -343,7 +345,6 @@ set(SOURCE_FILES_clp_s_unitTest src/clp_s/search/EvaluateTimestampIndex.hpp src/clp_s/search/Output.cpp src/clp_s/search/Output.hpp - src/clp_s/search/OutputHandler.cpp src/clp_s/search/OutputHandler.hpp src/clp_s/search/Projection.cpp src/clp_s/search/Projection.hpp diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 4608e784a2..f2a514fadb 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -336,16 +336,6 @@ target_link_libraries( ) target_include_directories(clp_s_JsonConstructor PUBLIC ../) -set( - CLP_S_SOURCES - CommandLineArguments.cpp - CommandLineArguments.hpp - ErrorCode.hpp - kv_ir_search.cpp - kv_ir_search.hpp - TraceableException.hpp -) - set( CLP_S_SEARCH_SOURCES Defs.hpp @@ -361,7 +351,6 @@ set( search/EvaluateTimestampIndex.hpp search/Output.cpp search/Output.hpp - search/OutputHandler.cpp # TODO maybe don't include search/OutputHandler.hpp search/Projection.cpp search/Projection.hpp @@ -388,11 +377,6 @@ target_link_libraries( PRIVATE spdlog::spdlog simdjson::simdjson - - clp_s::clp_dependencies - clp_s::reducer_dependencies - ${MONGOCXX_TARGET} - msgpack-cxx ) set( @@ -419,6 +403,18 @@ target_link_libraries( ) target_include_directories(clp_s_TimestampPattern PUBLIC ../) +set( + CLP_S_SOURCES + CommandLineArguments.cpp + CommandLineArguments.hpp + ErrorCode.hpp + kv_ir_search.cpp + kv_ir_search.hpp + OutputHandlerImpl.cpp + OutputHandlerImpl.hpp + TraceableException.hpp +) + add_executable( clp-s clp-s.cpp @@ -441,6 +437,8 @@ target_link_libraries( ${MONGOCXX_TARGET} spdlog::spdlog ystdlib::error_handling + + msgpack-cxx ) target_include_directories(clp-s PRIVATE diff --git a/components/core/src/clp_s/search/OutputHandler.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp similarity index 89% rename from components/core/src/clp_s/search/OutputHandler.cpp rename to components/core/src/clp_s/OutputHandlerImpl.cpp index 00f8d5be1b..59bb718827 100644 --- a/components/core/src/clp_s/search/OutputHandler.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -1,27 +1,34 @@ -#include "OutputHandler.hpp" +#include "OutputHandlerImpl.hpp" #include #include #include +#include +#include +#include +#include +#include +#include #include -#include "../../clp/networking/socket_utils.hpp" -#include "../../reducer/CountOperator.hpp" -#include "../../reducer/network_utils.hpp" -#include "../../reducer/Record.hpp" -#include "../archive_constants.hpp" +#include "../clp/networking/socket_utils.hpp" +#include "../reducer/CountOperator.hpp" +#include "../reducer/network_utils.hpp" +#include "../reducer/Record.hpp" +#include "archive_constants.hpp" +#include "search/OutputHandler.hpp" using std::string; using std::string_view; -namespace clp_s::search { +namespace clp_s { NetworkOutputHandler::NetworkOutputHandler( string const& host, int port, bool should_output_timestamp ) - : OutputHandler(should_output_timestamp, true) { + : ::clp_s::search::OutputHandler(should_output_timestamp, true) { m_socket_fd = clp::networking::connect_to_server(host, std::to_string(port)); if (-1 == m_socket_fd) { SPDLOG_ERROR("Failed to connect to the server, errno={}", errno); @@ -53,7 +60,7 @@ ResultsCacheOutputHandler::ResultsCacheOutputHandler( uint64_t max_num_results, bool should_output_timestamp ) - : OutputHandler(should_output_timestamp, true), + : ::clp_s::search::OutputHandler(should_output_timestamp, true), m_batch_size(batch_size), m_max_num_results(max_num_results) { try { @@ -143,7 +150,7 @@ void ResultsCacheOutputHandler::write( } CountOutputHandler::CountOutputHandler(int reducer_socket_fd) - : OutputHandler(false, false), + : ::clp_s::search::OutputHandler(false, false), m_reducer_socket_fd(reducer_socket_fd), m_pipeline(reducer::PipelineInputMode::InterStage) { m_pipeline.add_pipeline_stage(std::make_shared()); @@ -176,4 +183,4 @@ ErrorCode CountByTimeOutputHandler::finish() { } return ErrorCode::ErrorCodeSuccess; } -} // namespace clp_s::search +} // namespace clp_s diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp new file mode 100644 index 0000000000..8441c61dca --- /dev/null +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -0,0 +1,289 @@ +#ifndef CLP_S_OUTPUTHANDLERIMPL_HPP +#define CLP_S_OUTPUTHANDLERIMPL_HPP + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include "../reducer/Pipeline.hpp" +#include "../reducer/RecordGroupIterator.hpp" +#include "Defs.hpp" +#include "search/OutputHandler.hpp" +#include "TraceableException.hpp" + +namespace clp_s { +/** + * Output handler that writes to standard output. + */ +class StandardOutputHandler : public ::clp_s::search::OutputHandler { +public: + // Constructors + explicit StandardOutputHandler(bool should_output_metadata = false) + : ::clp_s::search::OutputHandler(should_output_metadata, true) {} + + // Methods inherited from OutputHandler + void write( + std::string_view message, + epochtime_t timestamp, + std::string_view archive_id, + int64_t log_event_idx + ) override { + std::cout << archive_id << ": " << log_event_idx << ": " << timestamp << " " << message; + } + + void write(std::string_view message) override { std::cout << message; } +}; + +/** + * Output handler that writes to a network destination. + */ +class NetworkOutputHandler : public ::clp_s::search::OutputHandler { +public: + // Types + class OperationFailed : public TraceableException { + public: + // Constructors + OperationFailed(ErrorCode error_code, char const* const filename, int line_number) + : TraceableException(error_code, filename, line_number) {} + }; + + // Constructors + explicit NetworkOutputHandler( + std::string const& host, + int port, + bool should_output_metadata = false + ); + + // Destructor + ~NetworkOutputHandler() override { + if (-1 != m_socket_fd) { + close(m_socket_fd); + } + } + + // Methods inherited from OutputHandler + void write( + std::string_view message, + epochtime_t timestamp, + std::string_view archive_id, + int64_t log_event_idx + ) override; + + void write(std::string_view message) override { write(message, 0, {}, 0); } + +private: + std::string m_host; + std::string m_port; + int m_socket_fd; +}; + +/** + * Output handler that writes to a MongoDB collection. + */ +class ResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { +public: + // Types + struct QueryResult { + // Constructors + QueryResult( + std::string_view original_path, + std::string_view message, + epochtime_t timestamp, + std::string_view archive_id, + int64_t log_event_idx + ) + : original_path(original_path), + message(message), + timestamp(timestamp), + archive_id(archive_id), + log_event_idx(log_event_idx) {} + + std::string original_path; + std::string message; + epochtime_t timestamp; + std::string archive_id; + int64_t log_event_idx; + }; + + struct QueryResultGreaterTimestampComparator { + bool operator()( + std::unique_ptr const& r1, + std::unique_ptr const& r2 + ) const { + return r1->timestamp > r2->timestamp; + } + }; + + class OperationFailed : public TraceableException { + public: + // Constructors + OperationFailed(ErrorCode error_code, char const* const filename, int line_number) + : TraceableException(error_code, filename, line_number) {} + }; + + // Constructor + ResultsCacheOutputHandler( + std::string const& uri, + std::string const& collection, + uint64_t batch_size, + uint64_t max_num_results, + bool should_output_metadata = true + ); + + // Methods inherited from OutputHandler + /** + * Flushes the output handler after each table that gets searched. + * @return ErrorCodeSuccess on success + * @return ErrorCodeFailureDbBulkWrite on failure to write results to the results cache + */ + ErrorCode flush() override; + + void write( + std::string_view message, + epochtime_t timestamp, + std::string_view archive_id, + int64_t log_event_idx + ) override; + + void write(std::string_view message) override { write(message, 0, {}, 0); } + +private: + mongocxx::client m_client; + mongocxx::collection m_collection; + std::vector m_results; + uint64_t m_batch_size; + uint64_t m_max_num_results; + std::priority_queue< + std::unique_ptr, + std::vector>, + QueryResultGreaterTimestampComparator> + m_latest_results; +}; + +/** + * Output handler that performs a count aggregation and sends the results to a reducer. + */ +class CountOutputHandler : public ::clp_s::search::OutputHandler { +public: + // Constructors + CountOutputHandler(int reducer_socket_fd); + + // Methods inherited from OutputHandler + void write( + std::string_view message, + epochtime_t timestamp, + std::string_view archive_id, + int64_t log_event_idx + ) override {} + + void write(std::string_view message) override; + + /** + * Flushes the count. + * @return ErrorCodeSuccess on success + * @return ErrorCodeFailureNetwork on network error + */ + ErrorCode finish() override; + +private: + int m_reducer_socket_fd; + reducer::Pipeline m_pipeline; +}; + +/** + * Output handler that performs a count aggregation bucketed by time and sends the results to a + * reducer. + */ +class CountByTimeOutputHandler : public ::clp_s::search::OutputHandler { +public: + // Constructors + CountByTimeOutputHandler(int reducer_socket_fd, int64_t count_by_time_bucket_size) + : search::OutputHandler{true, false}, + m_reducer_socket_fd{reducer_socket_fd}, + m_count_by_time_bucket_size{count_by_time_bucket_size} {} + + // Methods inherited from OutputHandler + void write( + std::string_view message, + epochtime_t timestamp, + std::string_view archive_id, + int64_t log_event_idx + ) override { + int64_t bucket = (timestamp / m_count_by_time_bucket_size) * m_count_by_time_bucket_size; + m_bucket_counts[bucket] += 1; + } + + void write(std::string_view message) override {} + + /** + * Flushes the counts. + * @return ErrorCodeSuccess on success + * @return ErrorCodeFailureNetwork on network error + */ + ErrorCode finish() override; + +private: + int m_reducer_socket_fd; + std::map m_bucket_counts; + int64_t m_count_by_time_bucket_size; +}; + +/** + * Output handler that records all results in a provided vector. + */ +class VectorOutputHandler : public ::clp_s::search::OutputHandler { +public: + // Types + struct QueryResult { + // Constructors + QueryResult( + std::string_view message, + epochtime_t timestamp, + std::string_view archive_id, + int64_t log_event_idx + ) + : message{message}, + timestamp{timestamp}, + archive_id{archive_id}, + log_event_idx{log_event_idx} {} + + std::string message; + epochtime_t timestamp; + std::string archive_id; + int64_t log_event_idx; + }; + + // Constructors + VectorOutputHandler(std::vector& output) + : search::OutputHandler{true, true}, + m_output(output) {} + + // Methods inherited from OutputHandler + void write( + std::string_view message, + epochtime_t timestamp, + std::string_view archive_id, + int64_t log_event_idx + ) override { + m_output.emplace_back(message, timestamp, archive_id, log_event_idx); + } + + void write(std::string_view message) override { + m_output.emplace_back(message, epochtime_t{}, std::string_view{}, int64_t{}); + } + +private: + std::vector& m_output; +}; +} // namespace clp_s + +#endif // CLP_S_OUTPUTHANDLERIMPL_HPP diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index a40d082978..5fb071864d 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -22,6 +22,7 @@ #include "JsonConstructor.hpp" #include "JsonParser.hpp" #include "kv_ir_search.hpp" +#include "OutputHandlerImpl.hpp" #include "search/AddTimestampConditions.hpp" #include "search/ast/ConvertToExists.hpp" #include "search/ast/EmptyExpr.hpp" @@ -227,16 +228,16 @@ bool search_archive( try { switch (command_line_arguments.get_output_handler_type()) { case CommandLineArguments::OutputHandlerType::Network: - output_handler = std::make_unique( + output_handler = std::make_unique( command_line_arguments.get_network_dest_host(), command_line_arguments.get_network_dest_port() ); break; case CommandLineArguments::OutputHandlerType::Reducer: if (command_line_arguments.do_count_results_aggregation()) { - output_handler = std::make_unique(reducer_socket_fd); + output_handler = std::make_unique(reducer_socket_fd); } else if (command_line_arguments.do_count_by_time_aggregation()) { - output_handler = std::make_unique( + output_handler = std::make_unique( reducer_socket_fd, command_line_arguments.get_count_by_time_bucket_size() ); @@ -246,7 +247,7 @@ bool search_archive( } break; case CommandLineArguments::OutputHandlerType::ResultsCache: - output_handler = std::make_unique( + output_handler = std::make_unique( command_line_arguments.get_mongodb_uri(), command_line_arguments.get_mongodb_collection(), command_line_arguments.get_batch_size(), @@ -254,7 +255,7 @@ bool search_archive( ); break; case CommandLineArguments::OutputHandlerType::Stdout: - output_handler = std::make_unique(); + output_handler = std::make_unique(); break; default: SPDLOG_ERROR("Unhandled OutputHandlerType."); diff --git a/components/core/src/clp_s/search/OutputHandler.hpp b/components/core/src/clp_s/search/OutputHandler.hpp index 79cb2585f4..b421dd9322 100644 --- a/components/core/src/clp_s/search/OutputHandler.hpp +++ b/components/core/src/clp_s/search/OutputHandler.hpp @@ -1,32 +1,15 @@ #ifndef CLP_S_SEARCH_OUTPUTHANDLER_HPP #define CLP_S_SEARCH_OUTPUTHANDLER_HPP -#include -#include -#include - -#include -#include -#include #include #include -#include -#include -#include -#include -#include -#include -#include - -#include "../../reducer/Pipeline.hpp" -#include "../../reducer/RecordGroupIterator.hpp" #include "../Defs.hpp" -#include "../TraceableException.hpp" +#include "../ErrorCode.hpp" namespace clp_s::search { /** - * Abstract class for handling output from the search command. + * Abstract class for handling search output. */ class OutputHandler { public: @@ -63,285 +46,21 @@ class OutputHandler { * Flushes the output handler after each table that gets searched. * @return ErrorCodeSuccess on success or relevant error code on error */ - virtual ErrorCode flush() { return ErrorCode::ErrorCodeSuccess; } + [[nodiscard]] virtual auto flush() -> ErrorCode { return ErrorCode::ErrorCodeSuccess; } /** * Performs any final operations after all tables have been searched. * @return ErrorCodeSuccess on success or relevant error code on error */ - virtual ErrorCode finish() { return ErrorCode::ErrorCodeSuccess; } - - [[nodiscard]] bool should_output_metadata() const { return m_should_output_metadata; } - - [[nodiscard]] bool should_marshal_records() const { return m_should_marshal_records; } - -private: - bool m_should_output_metadata; - bool m_should_marshal_records; -}; - -/** - * Output handler that writes to standard output. - */ -class StandardOutputHandler : public OutputHandler { -public: - // Constructors - explicit StandardOutputHandler(bool should_output_metadata = false) - : OutputHandler(should_output_metadata, true) {} - - // Methods inherited from OutputHandler - void write( - std::string_view message, - epochtime_t timestamp, - std::string_view archive_id, - int64_t log_event_idx - ) override { - std::cout << archive_id << ": " << log_event_idx << ": " << timestamp << " " << message; - } - - void write(std::string_view message) override { std::cout << message; } -}; - -/** - * Output handler that writes to a network destination. - */ -class NetworkOutputHandler : public OutputHandler { -public: - // Types - class OperationFailed : public TraceableException { - public: - // Constructors - OperationFailed(ErrorCode error_code, char const* const filename, int line_number) - : TraceableException(error_code, filename, line_number) {} - }; - - // Constructors - explicit NetworkOutputHandler( - std::string const& host, - int port, - bool should_output_metadata = false - ); - - // Destructor - ~NetworkOutputHandler() override { - if (-1 != m_socket_fd) { - close(m_socket_fd); - } - } - - // Methods inherited from OutputHandler - void write( - std::string_view message, - epochtime_t timestamp, - std::string_view archive_id, - int64_t log_event_idx - ) override; - - void write(std::string_view message) override { write(message, 0, {}, 0); } - -private: - std::string m_host; - std::string m_port; - int m_socket_fd; -}; - -/** - * Output handler that writes to a MongoDB collection. - */ -class ResultsCacheOutputHandler : public OutputHandler { -public: - // Types - struct QueryResult { - // Constructors - QueryResult( - std::string_view original_path, - std::string_view message, - epochtime_t timestamp, - std::string_view archive_id, - int64_t log_event_idx - ) - : original_path(original_path), - message(message), - timestamp(timestamp), - archive_id(archive_id), - log_event_idx(log_event_idx) {} - - std::string original_path; - std::string message; - epochtime_t timestamp; - std::string archive_id; - int64_t log_event_idx; - }; - - struct QueryResultGreaterTimestampComparator { - bool operator()( - std::unique_ptr const& r1, - std::unique_ptr const& r2 - ) const { - return r1->timestamp > r2->timestamp; - } - }; - - class OperationFailed : public TraceableException { - public: - // Constructors - OperationFailed(ErrorCode error_code, char const* const filename, int line_number) - : TraceableException(error_code, filename, line_number) {} - }; - - // Constructor - ResultsCacheOutputHandler( - std::string const& uri, - std::string const& collection, - uint64_t batch_size, - uint64_t max_num_results, - bool should_output_metadata = true - ); - - // Methods inherited from OutputHandler - /** - * Flushes the output handler after each table that gets searched. - * @return ErrorCodeSuccess on success - * @return ErrorCodeFailureDbBulkWrite on failure to write results to the results cache - */ - ErrorCode flush() override; - - void write( - std::string_view message, - epochtime_t timestamp, - std::string_view archive_id, - int64_t log_event_idx - ) override; - - void write(std::string_view message) override { write(message, 0, {}, 0); } - -private: - mongocxx::client m_client; - mongocxx::collection m_collection; - std::vector m_results; - uint64_t m_batch_size; - uint64_t m_max_num_results; - std::priority_queue< - std::unique_ptr, - std::vector>, - QueryResultGreaterTimestampComparator> - m_latest_results; -}; - -/** - * Output handler that performs a count aggregation and sends the results to a reducer. - */ -class CountOutputHandler : public OutputHandler { -public: - // Constructors - CountOutputHandler(int reducer_socket_fd); - - // Methods inherited from OutputHandler - void write( - std::string_view message, - epochtime_t timestamp, - std::string_view archive_id, - int64_t log_event_idx - ) override {} - - void write(std::string_view message) override; - - /** - * Flushes the count. - * @return ErrorCodeSuccess on success - * @return ErrorCodeFailureNetwork on network error - */ - ErrorCode finish() override; + [[nodiscard]] virtual auto finish() -> ErrorCode { return ErrorCode::ErrorCodeSuccess; } -private: - int m_reducer_socket_fd; - reducer::Pipeline m_pipeline; -}; - -/** - * Output handler that performs a count aggregation bucketed by time and sends the results to a - * reducer. - */ -class CountByTimeOutputHandler : public OutputHandler { -public: - // Constructors - CountByTimeOutputHandler(int reducer_socket_fd, int64_t count_by_time_bucket_size) - : OutputHandler{true, false}, - m_reducer_socket_fd{reducer_socket_fd}, - m_count_by_time_bucket_size{count_by_time_bucket_size} {} - - // Methods inherited from OutputHandler - void write( - std::string_view message, - epochtime_t timestamp, - std::string_view archive_id, - int64_t log_event_idx - ) override { - int64_t bucket = (timestamp / m_count_by_time_bucket_size) * m_count_by_time_bucket_size; - m_bucket_counts[bucket] += 1; - } - - void write(std::string_view message) override {} - - /** - * Flushes the counts. - * @return ErrorCodeSuccess on success - * @return ErrorCodeFailureNetwork on network error - */ - ErrorCode finish() override; - -private: - int m_reducer_socket_fd; - std::map m_bucket_counts; - int64_t m_count_by_time_bucket_size; -}; - -/** - * Output handler that records all results in a provided vector. - */ -class VectorOutputHandler : public OutputHandler { -public: - // Types - struct QueryResult { - // Constructors - QueryResult( - std::string_view message, - epochtime_t timestamp, - std::string_view archive_id, - int64_t log_event_idx - ) - : message{message}, - timestamp{timestamp}, - archive_id{archive_id}, - log_event_idx{log_event_idx} {} - - std::string message; - epochtime_t timestamp; - std::string archive_id; - int64_t log_event_idx; - }; - - // Constructors - VectorOutputHandler(std::vector& output) - : OutputHandler{true, true}, - m_output(output) {} - - // Methods inherited from OutputHandler - void write( - std::string_view message, - epochtime_t timestamp, - std::string_view archive_id, - int64_t log_event_idx - ) override { - m_output.emplace_back(message, timestamp, archive_id, log_event_idx); - } + [[nodiscard]] auto should_output_metadata() const -> bool { return m_should_output_metadata; } - void write(std::string_view message) override { - m_output.emplace_back(message, epochtime_t{}, std::string_view{}, int64_t{}); - } + [[nodiscard]] auto should_marshal_records() const -> bool { return m_should_marshal_records; } private: - std::vector& m_output; + bool m_should_output_metadata{}; + bool m_should_marshal_records{}; }; } // namespace clp_s::search diff --git a/components/core/tests/test-clp_s-search.cpp b/components/core/tests/test-clp_s-search.cpp index 76a283f290..784b88a6d6 100644 --- a/components/core/tests/test-clp_s-search.cpp +++ b/components/core/tests/test-clp_s-search.cpp @@ -16,6 +16,7 @@ #include "../src/clp_s/ArchiveReader.hpp" #include "../src/clp_s/CommandLineArguments.hpp" #include "../src/clp_s/InputConfig.hpp" +#include "../src/clp_s/OutputHandlerImpl.hpp" #include "../src/clp_s/search/ast/ConvertToExists.hpp" #include "../src/clp_s/search/ast/EmptyExpr.hpp" #include "../src/clp_s/search/ast/Expression.hpp" @@ -24,7 +25,6 @@ #include "../src/clp_s/search/EvaluateTimestampIndex.hpp" #include "../src/clp_s/search/kql/kql.hpp" #include "../src/clp_s/search/Output.hpp" -#include "../src/clp_s/search/OutputHandler.hpp" #include "../src/clp_s/search/Projection.hpp" #include "../src/clp_s/search/SchemaMatch.hpp" #include "../src/clp_s/Utils.hpp" @@ -42,7 +42,7 @@ auto get_test_input_local_path() -> std::string; void search(std::string const& query, bool ignore_case, std::vector const& expected_results); void validate_results( - std::vector const& results, + std::vector const& results, std::vector const& expected_results ); @@ -57,7 +57,7 @@ auto get_test_input_local_path() -> std::string { } void validate_results( - std::vector const& results, + std::vector const& results, std::vector const& expected_results ) { std::set results_set; @@ -96,7 +96,7 @@ search(std::string const& query, bool ignore_case, std::vector const& e expr = convert_pass.run(expr); REQUIRE(nullptr != expr); - std::vector results; + std::vector results; for (auto const& entry : std::filesystem::directory_iterator(cTestSearchArchiveDirectory)) { auto archive_reader = std::make_shared(); auto archive_path = clp_s::Path{ @@ -118,7 +118,7 @@ search(std::string const& query, bool ignore_case, std::vector const& e archive_expr = match_pass->run(archive_expr); REQUIRE(nullptr != archive_expr); - auto output_handler = std::make_unique(results); + auto output_handler = std::make_unique(results); clp_s::search::Output output_pass( match_pass, archive_expr, diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 33a7752ddb..70d04e4920 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -429,6 +429,8 @@ tasks: - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/JsonParser.cpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/JsonParser.hpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/JsonSerializer.hpp" + - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/search/OutputHandlerImpl.cpp" + - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/search/OutputHandlerImpl.hpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/PackedStreamReader.cpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/PackedStreamReader.hpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/ParsedMessage.hpp" @@ -492,8 +494,6 @@ tasks: - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/search/kql/kql.hpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/search/Output.cpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/search/Output.hpp" - - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/search/OutputHandler.cpp" - - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/search/OutputHandler.hpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/search/Projection.cpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/search/Projection.hpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/search/QueryRunner.cpp" From ec078a531ddcd42bf363f225b31a80ba4c637eec Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 26 May 2025 16:17:00 +0000 Subject: [PATCH 11/22] Clean up CMakeLists files --- components/core/src/clp_s/CMakeLists.txt | 51 ++----------------- .../core/src/clp_s/search/CMakeLists.txt | 47 +++++++++++++++++ 2 files changed, 50 insertions(+), 48 deletions(-) create mode 100644 components/core/src/clp_s/search/CMakeLists.txt diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index f2a514fadb..568ee4dd07 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -1,7 +1,5 @@ add_subdirectory(indexer) -add_subdirectory(search/ast) -add_subdirectory(search/kql) -add_subdirectory(search/sql) +add_subdirectory(search) set( CLP_S_CLP_SOURCES @@ -237,11 +235,11 @@ target_link_libraries( clp_s_ArchiveWriter PUBLIC absl::flat_hash_map + clp_s::IO simdjson::simdjson PRIVATE Boost::url clp_s::clp_dependencies - clp_s::IO clp_s::TimestampPattern ${CURL_LIBRARIES} fmt::fmt @@ -298,12 +296,12 @@ target_link_libraries( clp_s_ArchiveReader PUBLIC absl::flat_hash_map + clp_s::IO nlohmann_json::nlohmann_json PRIVATE absl::flat_hash_map Boost::url clp_s::clp_dependencies - clp_s::IO clp_s::TimestampPattern ${CURL_LIBRARIES} fmt::fmt @@ -336,49 +334,6 @@ target_link_libraries( ) target_include_directories(clp_s_JsonConstructor PUBLIC ../) -set( - CLP_S_SEARCH_SOURCES - Defs.hpp - search/AddTimestampConditions.cpp - search/AddTimestampConditions.hpp - search/clp_search/EncodedVariableInterpreter.cpp - search/clp_search/EncodedVariableInterpreter.hpp - search/clp_search/Grep.cpp - search/clp_search/Grep.hpp - search/clp_search/Query.cpp - search/clp_search/Query.hpp - search/EvaluateTimestampIndex.cpp - search/EvaluateTimestampIndex.hpp - search/Output.cpp - search/Output.hpp - search/OutputHandler.hpp - search/Projection.cpp - search/Projection.hpp - search/QueryRunner.cpp - search/QueryRunner.hpp - search/SchemaMatch.cpp - search/SchemaMatch.hpp - VariableEncoder.cpp - VariableEncoder.hpp -) - -add_library( - clp_s_search - ${CLP_S_SEARCH_SOURCES} -) -add_library(clp_s::search ALIAS clp_s_search) -target_compile_features(clp_s_search PRIVATE cxx_std_20) -target_link_libraries( - clp_s_search - PUBLIC - absl::flat_hash_map - clp_s::ArchiveReader - clp_s::search::ast - PRIVATE - spdlog::spdlog - simdjson::simdjson -) - set( CLP_S_TIMESTAMP_PATTERN_SOURCES Defs.hpp diff --git a/components/core/src/clp_s/search/CMakeLists.txt b/components/core/src/clp_s/search/CMakeLists.txt new file mode 100644 index 0000000000..a8ed5f76b9 --- /dev/null +++ b/components/core/src/clp_s/search/CMakeLists.txt @@ -0,0 +1,47 @@ +add_subdirectory(ast) +add_subdirectory(kql) +add_subdirectory(sql) + +set( + CLP_S_SEARCH_SOURCES + ../Defs.hpp + ../VariableEncoder.cpp + ../VariableEncoder.hpp + AddTimestampConditions.cpp + AddTimestampConditions.hpp + clp_search/EncodedVariableInterpreter.cpp + clp_search/EncodedVariableInterpreter.hpp + clp_search/Grep.cpp + clp_search/Grep.hpp + clp_search/Query.cpp + clp_search/Query.hpp + EvaluateTimestampIndex.cpp + EvaluateTimestampIndex.hpp + Output.cpp + Output.hpp + OutputHandler.hpp + Projection.cpp + Projection.hpp + QueryRunner.cpp + QueryRunner.hpp + SchemaMatch.cpp + SchemaMatch.hpp +) + +add_library( + clp_s_search + ${CLP_S_SEARCH_SOURCES} +) +add_library(clp_s::search ALIAS clp_s_search) +target_compile_features(clp_s_search PRIVATE cxx_std_20) +target_link_libraries( + clp_s_search + PUBLIC + absl::flat_hash_map + clp_s::ArchiveReader + clp_s::search::ast + PRIVATE + spdlog::spdlog + simdjson::simdjson +) +target_include_directories(clp_s_search PUBLIC ../../) From 4e5ce27c8e0269f6d9a9c3d64ce491fac3781245 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 26 May 2025 16:34:58 +0000 Subject: [PATCH 12/22] Remove unnecessary simdjson include from Output.hpp --- components/core/src/clp_s/search/Output.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/components/core/src/clp_s/search/Output.hpp b/components/core/src/clp_s/search/Output.hpp index 7a621e6a4e..0e4119d03a 100644 --- a/components/core/src/clp_s/search/Output.hpp +++ b/components/core/src/clp_s/search/Output.hpp @@ -9,8 +9,6 @@ #include #include -#include - #include "../ArchiveReader.hpp" #include "../SchemaReader.hpp" #include "../Utils.hpp" @@ -21,7 +19,6 @@ #include "QueryRunner.hpp" #include "SchemaMatch.hpp" -using namespace simdjson; using namespace clp_s::search::clp_search; namespace clp_s::search { From a9436955bf0d0c6761c87c866df212963c40f354 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 26 May 2025 16:35:17 +0000 Subject: [PATCH 13/22] Minor cleanup --- components/core/src/clp_s/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 568ee4dd07..16cfb3d57c 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -390,10 +390,9 @@ target_link_libraries( clp_s::search::ast clp_s::search::kql ${MONGOCXX_TARGET} + msgpack-cxx spdlog::spdlog ystdlib::error_handling - - msgpack-cxx ) target_include_directories(clp-s PRIVATE From 33d65cb13300430875d3cc60b367cbeee593f8cf Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 26 May 2025 16:36:04 +0000 Subject: [PATCH 14/22] Modify clp_s::search PUBLIC link libraries to ensure simple test project that links against it can build properly --- components/core/src/clp_s/search/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp_s/search/CMakeLists.txt b/components/core/src/clp_s/search/CMakeLists.txt index a8ed5f76b9..24402d4fc5 100644 --- a/components/core/src/clp_s/search/CMakeLists.txt +++ b/components/core/src/clp_s/search/CMakeLists.txt @@ -40,8 +40,8 @@ target_link_libraries( absl::flat_hash_map clp_s::ArchiveReader clp_s::search::ast + simdjson::simdjson PRIVATE spdlog::spdlog - simdjson::simdjson ) target_include_directories(clp_s_search PUBLIC ../../) From 7a4e2d295587568232aac0a71e7904ebe2a9e0e5 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Mon, 26 May 2025 17:07:09 +0000 Subject: [PATCH 15/22] Fix spdlog header includes and linkage --- components/core/src/clp_s/ArchiveWriter.cpp | 1 + components/core/src/clp_s/CMakeLists.txt | 1 + components/core/src/clp_s/DictionaryWriter.cpp | 2 ++ components/core/src/clp_s/JsonConstructor.cpp | 1 + components/core/src/clp_s/VariableDecoder.cpp | 2 ++ components/core/src/clp_s/ZstdCompressor.cpp | 2 ++ components/core/src/clp_s/ZstdCompressor.hpp | 1 - components/core/src/clp_s/search/Output.cpp | 2 ++ 8 files changed, 11 insertions(+), 1 deletion(-) diff --git a/components/core/src/clp_s/ArchiveWriter.cpp b/components/core/src/clp_s/ArchiveWriter.cpp index 11429c8f9c..4821d98037 100644 --- a/components/core/src/clp_s/ArchiveWriter.cpp +++ b/components/core/src/clp_s/ArchiveWriter.cpp @@ -5,6 +5,7 @@ #include #include +#include #include "../clp/streaming_archive/Constants.hpp" #include "archive_constants.hpp" diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 16cfb3d57c..4a6eb24677 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -331,6 +331,7 @@ target_link_libraries( PRIVATE fmt::fmt ${MONGOCXX_TARGET} + spdlog::spdlog ) target_include_directories(clp_s_JsonConstructor PUBLIC ../) diff --git a/components/core/src/clp_s/DictionaryWriter.cpp b/components/core/src/clp_s/DictionaryWriter.cpp index 9e4693fcfe..7e4d48d521 100644 --- a/components/core/src/clp_s/DictionaryWriter.cpp +++ b/components/core/src/clp_s/DictionaryWriter.cpp @@ -2,6 +2,8 @@ #include "DictionaryWriter.hpp" +#include + namespace clp_s { bool VariableDictionaryWriter::add_entry(std::string const& value, uint64_t& id) { bool new_entry = false; diff --git a/components/core/src/clp_s/JsonConstructor.cpp b/components/core/src/clp_s/JsonConstructor.cpp index 8873722eb9..85739f0768 100644 --- a/components/core/src/clp_s/JsonConstructor.cpp +++ b/components/core/src/clp_s/JsonConstructor.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include "archive_constants.hpp" #include "ErrorCode.hpp" diff --git a/components/core/src/clp_s/VariableDecoder.cpp b/components/core/src/clp_s/VariableDecoder.cpp index a4c36ce7b3..92117df24e 100644 --- a/components/core/src/clp_s/VariableDecoder.cpp +++ b/components/core/src/clp_s/VariableDecoder.cpp @@ -2,6 +2,8 @@ #include "VariableDecoder.hpp" +#include + namespace clp_s { bool VariableDecoder::decode_variables_into_message( LogTypeDictionaryEntry const& logtype_dict_entry, diff --git a/components/core/src/clp_s/ZstdCompressor.cpp b/components/core/src/clp_s/ZstdCompressor.cpp index 1ac74154db..ba55173064 100644 --- a/components/core/src/clp_s/ZstdCompressor.cpp +++ b/components/core/src/clp_s/ZstdCompressor.cpp @@ -1,6 +1,8 @@ // Code from CLP #include "ZstdCompressor.hpp" +#include + namespace clp_s { ZstdCompressor::ZstdCompressor() : Compressor{CompressorType::ZSTD}, diff --git a/components/core/src/clp_s/ZstdCompressor.hpp b/components/core/src/clp_s/ZstdCompressor.hpp index 4104571c74..db14167dbf 100644 --- a/components/core/src/clp_s/ZstdCompressor.hpp +++ b/components/core/src/clp_s/ZstdCompressor.hpp @@ -6,7 +6,6 @@ #include #include -#include #include #include diff --git a/components/core/src/clp_s/search/Output.cpp b/components/core/src/clp_s/search/Output.cpp index 610f06890f..e0b32a26a4 100644 --- a/components/core/src/clp_s/search/Output.cpp +++ b/components/core/src/clp_s/search/Output.cpp @@ -3,6 +3,8 @@ #include #include +#include + #include "../../clp/type_utils.hpp" #include "../SchemaTree.hpp" #include "../Utils.hpp" From 382abc14d133bf479193839d038707934bd04759 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Tue, 27 May 2025 21:04:42 +0000 Subject: [PATCH 16/22] Correct path to OutputHandlerImpl in lint task exceptions --- taskfiles/lint.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 70d04e4920..b6dd7ccc9c 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -429,8 +429,8 @@ tasks: - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/JsonParser.cpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/JsonParser.hpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/JsonSerializer.hpp" - - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/search/OutputHandlerImpl.cpp" - - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/search/OutputHandlerImpl.hpp" + - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/OutputHandlerImpl.cpp" + - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/OutputHandlerImpl.hpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/PackedStreamReader.cpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/PackedStreamReader.hpp" - "{{.G_CORE_COMPONENT_DIR}}/src/clp_s/ParsedMessage.hpp" From f7b65140cb0a9cef491f3ffc6ec29d57e6bd8f59 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Wed, 28 May 2025 15:00:01 +0000 Subject: [PATCH 17/22] Remove unnecessary include directive in clp-s search test --- components/core/tests/test-clp_s-search.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/core/tests/test-clp_s-search.cpp b/components/core/tests/test-clp_s-search.cpp index 9151c9d840..481c17a009 100644 --- a/components/core/tests/test-clp_s-search.cpp +++ b/components/core/tests/test-clp_s-search.cpp @@ -15,7 +15,6 @@ #include "../src/clp_s/archive_constants.hpp" #include "../src/clp_s/ArchiveReader.hpp" -#include "../src/clp_s/CommandLineArguments.hpp" #include "../src/clp_s/InputConfig.hpp" #include "../src/clp_s/OutputHandlerImpl.hpp" #include "../src/clp_s/search/ast/ColumnDescriptor.hpp" From c49e95cb49f99a7f573147ce2c05c58c79d4c3eb Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Fri, 30 May 2025 19:10:21 +0000 Subject: [PATCH 18/22] Make all library targets lower case --- components/core/src/clp_s/CMakeLists.txt | 58 +++++++++---------- .../core/src/clp_s/search/CMakeLists.txt | 2 +- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 4a6eb24677..09297beb8d 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -168,13 +168,13 @@ set( ) add_library( - clp_s_IO + clp_s_io ${CLP_S_IO_SOURCES} ) -add_library(clp_s::IO ALIAS clp_s_IO) -target_compile_features(clp_s_IO PRIVATE cxx_std_20) +add_library(clp_s::io ALIAS clp_s_io) +target_compile_features(clp_s_io PRIVATE cxx_std_20) target_link_libraries( - clp_s_IO + clp_s_io PRIVATE Boost::iostreams Boost::url clp_s::clp_dependencies @@ -182,7 +182,7 @@ target_link_libraries( spdlog::spdlog ZStd::ZStd ) -target_include_directories(clp_s_IO PUBLIC ../) +target_include_directories(clp_s_io PUBLIC ../) set( CLP_S_ARCHIVE_WRITER_SOURCES @@ -226,16 +226,16 @@ set( ) add_library( - clp_s_ArchiveWriter + clp_s_archive_writer ${CLP_S_ARCHIVE_WRITER_SOURCES} ) -add_library(clp_s::ArchiveWriter ALIAS clp_s_ArchiveWriter) -target_compile_features(clp_s_ArchiveWriter PRIVATE cxx_std_20) +add_library(clp_s::archive_writer ALIAS clp_s_archive_writer) +target_compile_features(clp_s_archive_writer PRIVATE cxx_std_20) target_link_libraries( - clp_s_ArchiveWriter + clp_s_archive_writer PUBLIC absl::flat_hash_map - clp_s::IO + clp_s::io simdjson::simdjson PRIVATE Boost::url @@ -247,7 +247,7 @@ target_link_libraries( nlohmann_json::nlohmann_json spdlog::spdlog ) -target_include_directories(clp_s_ArchiveWriter PUBLIC ../) +target_include_directories(clp_s_archive_writer PUBLIC ../) set( CLP_S_ARCHIVE_READER_SOURCES @@ -287,16 +287,16 @@ set( ) add_library( - clp_s_ArchiveReader + clp_s_archive_reader ${CLP_S_ARCHIVE_READER_SOURCES} ) -add_library(clp_s::ArchiveReader ALIAS clp_s_ArchiveReader) -target_compile_features(clp_s_ArchiveReader PRIVATE cxx_std_20) +add_library(clp_s::archive_reader ALIAS clp_s_archive_reader) +target_compile_features(clp_s_archive_reader PRIVATE cxx_std_20) target_link_libraries( - clp_s_ArchiveReader + clp_s_archive_reader PUBLIC absl::flat_hash_map - clp_s::IO + clp_s::io nlohmann_json::nlohmann_json PRIVATE absl::flat_hash_map @@ -308,7 +308,7 @@ target_link_libraries( msgpack-cxx spdlog::spdlog ) -target_include_directories(clp_s_ArchiveReader PUBLIC ../) +target_include_directories(clp_s_archive_reader PUBLIC ../) set( CLP_S_JSON_CONSTRUCTOR_SOURCES @@ -319,21 +319,21 @@ set( ) add_library( - clp_s_JsonConstructor + clp_s_json_constructor ${CLP_S_JSON_CONSTRUCTOR_SOURCES} ) -add_library(clp_s::JsonConstructor ALIAS clp_s_JsonConstructor) -target_compile_features(clp_s_JsonConstructor PRIVATE cxx_std_20) +add_library(clp_s::json_constructor ALIAS clp_s_json_constructor) +target_compile_features(clp_s_json_constructor PRIVATE cxx_std_20) target_link_libraries( - clp_s_JsonConstructor + clp_s_json_constructor PUBLIC - clp_s::ArchiveReader + clp_s::archive_reader PRIVATE fmt::fmt ${MONGOCXX_TARGET} spdlog::spdlog ) -target_include_directories(clp_s_JsonConstructor PUBLIC ../) +target_include_directories(clp_s_json_constructor PUBLIC ../) set( CLP_S_TIMESTAMP_PATTERN_SOURCES @@ -360,7 +360,7 @@ target_link_libraries( target_include_directories(clp_s_TimestampPattern PUBLIC ../) set( - CLP_S_SOURCES + CLP_S_EXE_SOURCES CommandLineArguments.cpp CommandLineArguments.hpp ErrorCode.hpp @@ -374,18 +374,18 @@ set( add_executable( clp-s clp-s.cpp - ${CLP_S_SOURCES} + ${CLP_S_EXE_SOURCES} ) target_compile_features(clp-s PRIVATE cxx_std_20) target_link_libraries( clp-s PRIVATE Boost::program_options - clp_s::ArchiveReader - clp_s::ArchiveWriter + clp_s::archive_reader + clp_s::archive_writer clp_s::clp_dependencies - clp_s::IO - clp_s::JsonConstructor + clp_s::io + clp_s::json_constructor clp_s::reducer_dependencies clp_s::search clp_s::search::ast diff --git a/components/core/src/clp_s/search/CMakeLists.txt b/components/core/src/clp_s/search/CMakeLists.txt index 24402d4fc5..f5a337386d 100644 --- a/components/core/src/clp_s/search/CMakeLists.txt +++ b/components/core/src/clp_s/search/CMakeLists.txt @@ -38,7 +38,7 @@ target_link_libraries( clp_s_search PUBLIC absl::flat_hash_map - clp_s::ArchiveReader + clp_s::archive_reader clp_s::search::ast simdjson::simdjson PRIVATE From 8e39d8e71aa3db8716574be7387b1895568cf7b0 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Fri, 30 May 2025 19:20:21 +0000 Subject: [PATCH 19/22] Reorder target_include directives --- components/core/src/clp_s/CMakeLists.txt | 18 +++++++++--------- .../core/src/clp_s/search/CMakeLists.txt | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 09297beb8d..3764133c26 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -173,6 +173,7 @@ add_library( ) add_library(clp_s::io ALIAS clp_s_io) target_compile_features(clp_s_io PRIVATE cxx_std_20) +target_include_directories(clp_s_io PUBLIC ../) target_link_libraries( clp_s_io PRIVATE @@ -182,7 +183,6 @@ target_link_libraries( spdlog::spdlog ZStd::ZStd ) -target_include_directories(clp_s_io PUBLIC ../) set( CLP_S_ARCHIVE_WRITER_SOURCES @@ -231,6 +231,7 @@ add_library( ) add_library(clp_s::archive_writer ALIAS clp_s_archive_writer) target_compile_features(clp_s_archive_writer PRIVATE cxx_std_20) +target_include_directories(clp_s_archive_writer PUBLIC ../) target_link_libraries( clp_s_archive_writer PUBLIC @@ -247,7 +248,6 @@ target_link_libraries( nlohmann_json::nlohmann_json spdlog::spdlog ) -target_include_directories(clp_s_archive_writer PUBLIC ../) set( CLP_S_ARCHIVE_READER_SOURCES @@ -292,6 +292,7 @@ add_library( ) add_library(clp_s::archive_reader ALIAS clp_s_archive_reader) target_compile_features(clp_s_archive_reader PRIVATE cxx_std_20) +target_include_directories(clp_s_archive_reader PUBLIC ../) target_link_libraries( clp_s_archive_reader PUBLIC @@ -308,7 +309,6 @@ target_link_libraries( msgpack-cxx spdlog::spdlog ) -target_include_directories(clp_s_archive_reader PUBLIC ../) set( CLP_S_JSON_CONSTRUCTOR_SOURCES @@ -324,6 +324,7 @@ add_library( ) add_library(clp_s::json_constructor ALIAS clp_s_json_constructor) target_compile_features(clp_s_json_constructor PRIVATE cxx_std_20) +target_include_directories(clp_s_json_constructor PUBLIC ../) target_link_libraries( clp_s_json_constructor PUBLIC @@ -333,7 +334,6 @@ target_link_libraries( ${MONGOCXX_TARGET} spdlog::spdlog ) -target_include_directories(clp_s_json_constructor PUBLIC ../) set( CLP_S_TIMESTAMP_PATTERN_SOURCES @@ -350,6 +350,7 @@ add_library( ) add_library(clp_s::TimestampPattern ALIAS clp_s_TimestampPattern) target_compile_features(clp_s_TimestampPattern PRIVATE cxx_std_20) +target_include_directories(clp_s_TimestampPattern PUBLIC ../) target_link_libraries( clp_s_TimestampPattern PRIVATE @@ -357,7 +358,6 @@ target_link_libraries( date::date spdlog::spdlog ) -target_include_directories(clp_s_TimestampPattern PUBLIC ../) set( CLP_S_EXE_SOURCES @@ -377,6 +377,10 @@ add_executable( ${CLP_S_EXE_SOURCES} ) target_compile_features(clp-s PRIVATE cxx_std_20) +target_include_directories(clp-s + PRIVATE + "${CLP_OUTCOME_INCLUDE_DIRECTORY}" +) target_link_libraries( clp-s PRIVATE @@ -395,10 +399,6 @@ target_link_libraries( spdlog::spdlog ystdlib::error_handling ) -target_include_directories(clp-s - PRIVATE - "${CLP_OUTCOME_INCLUDE_DIRECTORY}" -) set_target_properties( clp-s PROPERTIES diff --git a/components/core/src/clp_s/search/CMakeLists.txt b/components/core/src/clp_s/search/CMakeLists.txt index f5a337386d..08e13ef54d 100644 --- a/components/core/src/clp_s/search/CMakeLists.txt +++ b/components/core/src/clp_s/search/CMakeLists.txt @@ -34,6 +34,7 @@ add_library( ) add_library(clp_s::search ALIAS clp_s_search) target_compile_features(clp_s_search PRIVATE cxx_std_20) +target_include_directories(clp_s_search PUBLIC ../../) target_link_libraries( clp_s_search PUBLIC @@ -44,4 +45,3 @@ target_link_libraries( PRIVATE spdlog::spdlog ) -target_include_directories(clp_s_search PUBLIC ../../) From 4346bd7f589bcb4d2c298839b844767be61f9b0f Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Fri, 30 May 2025 19:29:38 +0000 Subject: [PATCH 20/22] Address rabbit comment --- components/core/src/clp_s/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 3764133c26..6562352eb3 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -300,7 +300,6 @@ target_link_libraries( clp_s::io nlohmann_json::nlohmann_json PRIVATE - absl::flat_hash_map Boost::url clp_s::clp_dependencies clp_s::TimestampPattern From dbf72315d4fe9b0eabc85322eb7a29de4ad76ec1 Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Fri, 30 May 2025 19:51:03 +0000 Subject: [PATCH 21/22] Rename clp_s::TimestampPattern -> clp_s::timestamp_pattern library --- components/core/CMakeLists.txt | 2 +- components/core/src/clp_s/CMakeLists.txt | 14 +++++++------- .../core/src/clp_s/search/ast/CMakeLists.txt | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 51f29659cb..a7e80d4d6d 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -688,7 +688,7 @@ target_link_libraries(unitTest clp_s::search::ast clp_s::search::kql clp_s::search::sql - clp_s::TimestampPattern + clp_s::timestamp_pattern date::date fmt::fmt log_surgeon::log_surgeon diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 6562352eb3..f295700b48 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -241,7 +241,7 @@ target_link_libraries( PRIVATE Boost::url clp_s::clp_dependencies - clp_s::TimestampPattern + clp_s::timestamp_pattern ${CURL_LIBRARIES} fmt::fmt msgpack-cxx @@ -302,7 +302,7 @@ target_link_libraries( PRIVATE Boost::url clp_s::clp_dependencies - clp_s::TimestampPattern + clp_s::timestamp_pattern ${CURL_LIBRARIES} fmt::fmt msgpack-cxx @@ -344,14 +344,14 @@ set( ) add_library( - clp_s_TimestampPattern + clp_s_timestamp_pattern ${CLP_S_TIMESTAMP_PATTERN_SOURCES} ) -add_library(clp_s::TimestampPattern ALIAS clp_s_TimestampPattern) -target_compile_features(clp_s_TimestampPattern PRIVATE cxx_std_20) -target_include_directories(clp_s_TimestampPattern PUBLIC ../) +add_library(clp_s::timestamp_pattern ALIAS clp_s_timestamp_pattern) +target_compile_features(clp_s_timestamp_pattern PRIVATE cxx_std_20) +target_include_directories(clp_s_timestamp_pattern PUBLIC ../) target_link_libraries( - clp_s_TimestampPattern + clp_s_timestamp_pattern PRIVATE clp::string_utils date::date diff --git a/components/core/src/clp_s/search/ast/CMakeLists.txt b/components/core/src/clp_s/search/ast/CMakeLists.txt index bbc19dabfe..c510ad60b9 100644 --- a/components/core/src/clp_s/search/ast/CMakeLists.txt +++ b/components/core/src/clp_s/search/ast/CMakeLists.txt @@ -59,7 +59,7 @@ target_compile_features(clp_s_search_ast PRIVATE cxx_std_20) target_link_libraries( clp_s_search_ast PUBLIC - clp_s::TimestampPattern + clp_s::timestamp_pattern PRIVATE simdjson::simdjson ) From 416dbd2f5c9481daf8de6f721369f3f21016fe3b Mon Sep 17 00:00:00 2001 From: gibber9809 Date: Fri, 30 May 2025 20:00:16 +0000 Subject: [PATCH 22/22] Re-order target_include... directive in clp_s::search::ast CMakeLists --- components/core/src/clp_s/search/ast/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp_s/search/ast/CMakeLists.txt b/components/core/src/clp_s/search/ast/CMakeLists.txt index c510ad60b9..31afec5b01 100644 --- a/components/core/src/clp_s/search/ast/CMakeLists.txt +++ b/components/core/src/clp_s/search/ast/CMakeLists.txt @@ -56,6 +56,7 @@ add_library( ) add_library(clp_s::search::ast ALIAS clp_s_search_ast) target_compile_features(clp_s_search_ast PRIVATE cxx_std_20) +target_include_directories(clp_s_search_ast PUBLIC ../../../) target_link_libraries( clp_s_search_ast PUBLIC @@ -63,4 +64,3 @@ target_link_libraries( PRIVATE simdjson::simdjson ) -target_include_directories(clp_s_search_ast PUBLIC ../../../)