A bit of refactoring and corrections recommended by coderabitai

Author: AVMatthews

components/core/src/clp_s/CommandLineArguments.cpp

@@ -426,6 +426,11 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) {
                     po::value<size_t>(&m_max_document_size)->value_name("DOC_SIZE")->
                         default_value(m_max_document_size),
                     "Maximum allowed size (B) for a single document before ir generation fails."
+            )(
+                    "max-ir-buffer-size",
+                    po::value<size_t>(&m_max_ir_buffer_size)->value_name("BUFFER_SIZE")->
+                        default_value(m_max_ir_buffer_size),
+                    "Maximum allowed size (B) for a in memory IR buffer befroe being written to file."
             )(
                     "encoding-type",
                     po::value<int>(&m_encoding_type)->value_name("ENCODING_TYPE")->
@@ -1040,11 +1045,10 @@ void CommandLineArguments::print_search_usage() const {
 }
 
 void CommandLineArguments::print_json_to_ir_usage() const {
-    std::cerr << "Usage: " << m_program_name << " r [OPTIONS] IRS_DIR [FILE/DIR ...]" << std::endl;
+    std::cerr << "Usage: " << m_program_name << " r [OPTIONS] IRS_DIR [FILE/DIR ...]\n";
 }
 
 void CommandLineArguments::print_ir_compression_usage() const {
-    std::cerr << "Usage: " << m_program_name << " i [OPTIONS] ARCHIVES_DIR [FILE/DIR ...]"
-              << std::endl;
+    std::cerr << "Usage: " << m_program_name << " i [OPTIONS] ARCHIVES_DIR [FILE/DIR ...]\n";
 }
 }  // namespace clp_s

components/core/src/clp_s/CommandLineArguments.hpp

@@ -62,7 +62,9 @@ class CommandLineArguments {
 
     size_t get_max_document_size() const { return m_max_document_size; }
 
-    int get_encoding_type() const { return m_encoding_type; }
+    [[nodiscard]] auto get_max_ir_buffer_size() const -> size_t { return m_max_ir_buffer_size; }
+
+    [[nodiscard]] auto get_encoding_type() const -> int { return m_encoding_type; }
 
     [[nodiscard]] bool print_archive_stats() const { return m_print_archive_stats; }
 
@@ -182,6 +184,7 @@ class CommandLineArguments {
     bool m_ordered_decompression{false};
     size_t m_ordered_chunk_size{0};
     int m_encoding_type{8};
+    size_t m_max_ir_buffer_size{512ULL * 1024 * 1024};
     // Metadata db variables
     std::optional<clp::GlobalMetadataDBConfig> m_metadata_db_config;
 

components/core/src/clp_s/JsonParser.cpp

@@ -18,6 +18,7 @@
 #include "DictionaryWriter.hpp"
 #include "JsonFileIterator.hpp"
 #include "ParsedMessage.hpp"
+#include "SchemaTree.hpp"
 
 using namespace simdjson;
 

components/core/src/clp_s/JsonParser.hpp

@@ -21,6 +21,7 @@
 #include "ArchiveWriter.hpp"
 #include "ParsedMessage.hpp"
 #include "Schema.hpp"
+#include "SchemaTree.hpp"
 
 using clp::BufferReader;
 using clp::ffi::ir_stream::Deserializer;
@@ -44,6 +45,7 @@ struct JsonToIRParserOption {
     std::vector<std::string> file_paths;
     std::string irs_dir;
     size_t max_document_size;
+    size_t max_ir_buffer_size;
     int compression_level;
     int encoding;
 };
@@ -93,11 +95,11 @@ class JsonParser {
     void parse_line(ondemand::value line, int32_t parent_node_id, std::string const& key);
 
     /**
-     * Compresses the input files specified by the command line arguments into an archive.
+     * Determines the archive node type based on the IR node type and value.
      * @param ir_node_type schema node type from the IR stream
-     * @param node_has_value Boolean that say whether or not the node has value.
-     * @param node_value The ir schema node value if the node has value
-     * @return The clp-s archive Node Type that shoudl be used for the archive node
+     * @param node_has_value Boolean that says whether or not the node has value.
+     * @param node_value The IR schema node value if the node has value
+     * @return The clp-s archive Node Type that should be used for the archive node
      */
     static auto get_archive_node_type(
             clp::ffi::SchemaTreeNode::Type ir_node_type,
@@ -109,9 +111,9 @@ class JsonParser {
      * Get archive node id for ir node
      * @param ir_node_to_archive_node_unordered_map cache of node id conversions between
      * deserializer schema tree nodes and archive schema tree nodes
-     * @param irNodeID
-     * @param irType
-     * @param irTree
+     * @param ir_node_id ID of the IR node
+     * @param archive_node_type Type of the archive node
+     * @param ir_treeThe IR schema tree
      */
     auto get_archive_node_id(
             std::unordered_map<int32_t, std::vector<std::pair<NodeType, int32_t>>>&

components/core/src/clp_s/clp-s.cpp

@@ -48,8 +48,6 @@ using clp_s::CommandLineArguments;
 
 namespace {
 
-size_t max_ir_buffer_size = 1'000'000'000;
-
 /**
  * Compresses the input files specified by the command line arguments into an archive.
  * @param command_line_arguments
@@ -87,6 +85,17 @@ auto run_serializer(clp_s::JsonToIRParserOption const& option, std::string path)
  */
 auto generate_ir(CommandLineArguments const& command_line_arguments) -> bool;
 
+/**
+ * Fill in JsonParserOption instance based on command line user input
+ * @param command_line_arguments
+ * @param option
+ * @return Whether setup was succesful
+ */
+auto setup_compression_options(
+        CommandLineArguments const& command_line_arguments,
+        clp_s::JsonParserOption& option
+) -> bool;
+
 /**
  * Compresses the input IR files specified by the command line arguments into an archive.
  * @param command_line_arguments
@@ -238,7 +247,7 @@ auto run_serializer(clp_s::JsonToIRParserOption const& option, std::string path)
                     return false;
                 }
                 flush_and_clear_serializer_buffer(serializer, ir_buf);
-                if (ir_buf.size() >= max_ir_buffer_size) {
+                if (ir_buf.size() >= option.max_ir_buffer_size) {
                     total_size = total_size + ir_buf.size();
                     zc.write(reinterpret_cast<char*>(ir_buf.data()), ir_buf.size());
                     zc.flush();
@@ -278,11 +287,13 @@ auto generate_ir(CommandLineArguments const& command_line_arguments) -> bool {
     option.file_paths = command_line_arguments.get_file_paths();
     option.irs_dir = irs_dir.string();
     option.max_document_size = command_line_arguments.get_max_document_size();
+    option.max_ir_buffer_size = command_line_arguments.get_max_ir_buffer_size();
     option.compression_level = command_line_arguments.get_compression_level();
     option.encoding = command_line_arguments.get_encoding_type();
 
     if (false == clp_s::FileUtils::validate_path(option.file_paths)) {
-        exit(1);
+        SPDLOG_ERROR("Invalid file path(s) provided");
+        return false;
     }
 
     std::vector<std::string> all_file_paths;
@@ -304,9 +315,11 @@ auto generate_ir(CommandLineArguments const& command_line_arguments) -> bool {
     return true;
 }
 
-auto ir_compress(CommandLineArguments const& command_line_arguments) -> bool {
+auto setup_compression_options(
+        CommandLineArguments const& command_line_arguments,
+        clp_s::JsonParserOption& option
+) -> bool {
     auto archives_dir = std::filesystem::path(command_line_arguments.get_archives_dir());
-
     // Create output directory in case it doesn't exist
     try {
         std::filesystem::create_directory(archives_dir.string());
@@ -318,8 +331,6 @@ auto ir_compress(CommandLineArguments const& command_line_arguments) -> bool {
         );
         return false;
     }
-
-    clp_s::JsonParserOption option{};
     option.file_paths = command_line_arguments.get_file_paths();
     option.archives_dir = archives_dir.string();
     option.target_encoded_size = command_line_arguments.get_target_encoded_size();
@@ -340,6 +351,14 @@ auto ir_compress(CommandLineArguments const& command_line_arguments) -> bool {
                 db_config.get_metadata_table_prefix()
         );
     }
+    return true;
+}
+
+auto ir_compress(CommandLineArguments const& command_line_arguments) -> bool {
+    clp_s::JsonParserOption option{};
+    if (false == setup_compression_options(command_line_arguments, option)) {
+        return false;
+    }
 
     clp_s::JsonParser parser(option);
     if (false == parser.parse_from_ir()) {