Add register operations; Add DFA test.

SharafMohamed · SharafMohamed · commit c42a21482804 · 2025-02-25T17:20:03.000-05:00
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -72,13 +72,15 @@ set(SOURCE_FILES
     src/log_surgeon/finite_automata/Dfa.hpp
     src/log_surgeon/finite_automata/DfaState.hpp
     src/log_surgeon/finite_automata/DfaStatePair.hpp
+    src/log_surgeon/finite_automata/DfaTransition.hpp
     src/log_surgeon/finite_automata/Nfa.hpp
     src/log_surgeon/finite_automata/NfaSpontaneousTransition.hpp
     src/log_surgeon/finite_automata/NfaState.hpp
     src/log_surgeon/finite_automata/PrefixTree.cpp
     src/log_surgeon/finite_automata/PrefixTree.hpp
     src/log_surgeon/finite_automata/RegexAST.hpp
     src/log_surgeon/finite_automata/RegisterHandler.hpp
+    src/log_surgeon/finite_automata/RegisterOperation.hpp
     src/log_surgeon/finite_automata/StateType.hpp
     src/log_surgeon/finite_automata/TagOperation.hpp
     src/log_surgeon/finite_automata/UnicodeIntervalTree.hpp
diff --git a/src/log_surgeon/Lexer.hpp b/src/log_surgeon/Lexer.hpp
@@ -158,8 +158,8 @@ class Lexer {
      */
     [[nodiscard]] auto get_reg_id_from_tag_id(tag_id_t const tag_id
     ) const -> std::optional<reg_id_t> {
-        if (m_tag_to_reg_id.contains(tag_id)) {
-            return m_tag_to_reg_id.at(tag_id);
+        if (m_tag_to_final_reg_id.contains(tag_id)) {
+            return m_tag_to_final_reg_id.at(tag_id);
         }
         return std::nullopt;
     }
@@ -218,7 +218,7 @@ class Lexer {
     TypedDfaState const* m_prev_state{nullptr};
     std::unordered_map<rule_id_t, std::vector<capture_id_t>> m_rule_id_to_capture_ids;
     std::unordered_map<capture_id_t, std::pair<tag_id_t, tag_id_t>> m_capture_id_to_tag_id_pair;
-    std::unordered_map<tag_id_t, reg_id_t> m_tag_to_reg_id;
+    std::map<tag_id_t, reg_id_t> m_tag_to_final_reg_id;
 };
 
 namespace lexers {
diff --git a/src/log_surgeon/finite_automata/Dfa.hpp b/src/log_surgeon/finite_automata/Dfa.hpp
@@ -8,6 +8,10 @@
 #include <stack>
 #include <vector>
 
+#include <fmt/core.h>
+#include <fmt/format.h>
+
+#include <log_surgeon/Constants.hpp>
 #include <log_surgeon/finite_automata/DfaStatePair.hpp>
 #include <log_surgeon/finite_automata/Nfa.hpp>
 
@@ -17,6 +21,11 @@ class Dfa {
 public:
     explicit Dfa(Nfa<TypedNfaState> const& nfa);
 
+    /**
+     * @return A string representation of the DFA.
+     */
+    [[nodiscard]] auto serialize() const -> std::string;
+
     /**
      * Creates a new DFA state based on a set of NFA states and adds it to `m_states`.
      * @param nfa_state_set The set of NFA states represented by this DFA state.
@@ -38,6 +47,12 @@ class Dfa {
     [[nodiscard]] auto get_intersect(Dfa const* dfa_in) const -> std::set<uint32_t>;
 
 private:
+    /**
+     * @return A vector representing the traversal order of the DFA states using breadth-first
+     * search (BFS).
+     */
+    [[nodiscard]] auto get_bfs_traversal_order() const -> std::vector<TypedDfaState const*>;
+
     std::vector<std::unique_ptr<TypedDfaState>> m_states;
 };
 
@@ -85,7 +100,7 @@ Dfa<TypedDfaState, TypedNfaState>::Dfa(Nfa<TypedNfaState> const& nfa) {
         };
         for (auto const& kv : ascii_transitions_map) {
             auto* dest_state = next_dfa_state(kv.second);
-            dfa_state->add_byte_transition(kv.first, dest_state);
+            dfa_state->add_byte_transition(kv.first, {{}, dest_state});
         }
         // TODO: add this for the utf8 case
     }
@@ -125,6 +140,54 @@ auto Dfa<TypedDfaState, TypedNfaState>::get_intersect(Dfa const* dfa_in
     }
     return schema_types;
 }
+
+template <typename TypedDfaState, typename TypedNfaState>
+auto Dfa<TypedDfaState, TypedNfaState>::get_bfs_traversal_order(
+) const -> std::vector<TypedDfaState const*> {
+    std::queue<TypedDfaState const*> state_queue;
+    std::unordered_set<TypedDfaState const*> visited_states;
+    std::vector<TypedDfaState const*> visited_order;
+    visited_states.reserve(m_states.size());
+    visited_order.reserve(m_states.size());
+
+    auto add_to_queue_and_visited
+            = [&state_queue, &visited_states](TypedDfaState const* dest_state) {
+                  if (visited_states.insert(dest_state).second) {
+                      state_queue.push(dest_state);
+                  }
+              };
+
+    add_to_queue_and_visited(get_root());
+    while (false == state_queue.empty()) {
+        auto const* current_state = state_queue.front();
+        visited_order.push_back(current_state);
+        state_queue.pop();
+        // TODO: handle the utf8 case
+        for (uint32_t idx{0}; idx < cSizeOfByte; ++idx) {
+            auto const dest_state{current_state->get_dest_state(idx)};
+            if (nullptr != dest_state) {
+                add_to_queue_and_visited(dest_state);
+            }
+        }
+    }
+    return visited_order;
+}
+
+template <typename TypedDfaState, typename TypedNfaState>
+auto Dfa<TypedDfaState, TypedNfaState>::serialize() const -> std::string {
+    auto const traversal_order = get_bfs_traversal_order();
+
+    std::unordered_map<TypedDfaState const*, uint32_t> state_ids;
+    for (auto const* state : traversal_order) {
+        state_ids.emplace(state, state_ids.size());
+    }
+
+    std::vector<std::string> serialized_states;
+    for (auto const* state : traversal_order) {
+        serialized_states.emplace_back(state->serialize(state_ids));
+    }
+    return fmt::format("{}\n", fmt::join(serialized_states, "\n"));
+}
 }  // namespace log_surgeon::finite_automata
 
 #endif  // LOG_SURGEON_FINITE_AUTOMATA_DFA_HPP
diff --git a/src/log_surgeon/finite_automata/DfaState.hpp b/src/log_surgeon/finite_automata/DfaState.hpp
@@ -4,11 +4,18 @@
 #include <cassert>
 #include <cstdint>
 #include <memory>
+#include <string>
 #include <tuple>
 #include <type_traits>
+#include <unordered_map>
 #include <vector>
 
+#include <fmt/core.h>
+#include <fmt/format.h>
+
 #include <log_surgeon/Constants.hpp>
+#include <log_surgeon/finite_automata/DfaTransition.hpp>
+#include <log_surgeon/finite_automata/RegisterOperation.hpp>
 #include <log_surgeon/finite_automata/StateType.hpp>
 #include <log_surgeon/finite_automata/UnicodeIntervalTree.hpp>
 
@@ -24,7 +31,11 @@ class DfaState {
 public:
     using Tree = UnicodeIntervalTree<DfaState*>;
 
-    DfaState() { std::fill(std::begin(m_bytes_transition), std::end(m_bytes_transition), nullptr); }
+    DfaState() {
+        for (auto& transition : m_bytes_transition) {
+            transition = DfaTransition<state_type>{{}, nullptr};
+        }
+    }
 
     auto add_matching_variable_id(uint32_t const variable_id) -> void {
         m_matching_variable_ids.push_back(variable_id);
@@ -38,19 +49,32 @@ class DfaState {
         return false == m_matching_variable_ids.empty();
     }
 
-    auto add_byte_transition(uint8_t const& byte, DfaState* dest_state) -> void {
-        m_bytes_transition[byte] = dest_state;
+    auto
+    add_byte_transition(uint8_t const& byte, DfaTransition<state_type> dfa_transition) -> void {
+        m_bytes_transition[byte] = dfa_transition;
+    }
+
+    auto add_accepting_op(RegisterOperation const reg_op) -> void {
+        m_accepting_ops.push_back(reg_op);
     }
 
+    /**
+     * @param state_ids A map of states to their unique identifiers.
+     * @return A string representation of the DFA state.
+     */
+    [[nodiscard]] auto serialize(std::unordered_map<DfaState const*, uint32_t> const& state_ids
+    ) const -> std::string;
+
     /**
      * @param character The character (byte or utf8) to transition on.
-     * @return A pointer to the DFA state reached after transitioning on `character`.
+     * @return The destination DFA state reached after transitioning on `character`.
      */
     [[nodiscard]] auto get_dest_state(uint32_t character) const -> DfaState const*;
 
 private:
     std::vector<uint32_t> m_matching_variable_ids;
-    DfaState* m_bytes_transition[cSizeOfByte];
+    std::vector<RegisterOperation> m_accepting_ops;
+    DfaTransition<state_type> m_bytes_transition[cSizeOfByte];
     // NOTE: We don't need m_tree_transitions for the `state_type == StateType::Byte` case, so we
     // use an empty class (`std::tuple<>`) in that case.
     std::conditional_t<state_type == StateType::Utf8, Tree, std::tuple<>> m_tree_transitions;
@@ -59,20 +83,62 @@ class DfaState {
 template <StateType state_type>
 auto DfaState<state_type>::get_dest_state(uint32_t character) const -> DfaState const* {
     if constexpr (StateType::Byte == state_type) {
-        return m_bytes_transition[character];
+        return m_bytes_transition[character].get_dest_state();
     } else {
         if (character < cSizeOfByte) {
-            return m_bytes_transition[character];
+            return m_bytes_transition[character].get_dest_state();
         }
         std::unique_ptr<std::vector<typename Tree::Data>> result
                 = m_tree_transitions.find(Interval(character, character));
         assert(result->size() <= 1);
         if (false == result->empty()) {
-            return result->front().m_value;
+            return result->front().m_value.get_dest_state();
         }
         return nullptr;
     }
 }
+
+template <StateType state_type>
+auto DfaState<state_type>::serialize(std::unordered_map<DfaState const*, uint32_t> const& state_ids
+) const -> std::string {
+    auto const accepting_tags_string = is_accepting()
+                                               ? fmt::format(
+                                                         "accepting_tags={{{}}},",
+                                                         fmt::join(m_matching_variable_ids, ",")
+                                                 )
+                                               : "";
+
+    std::vector<std::string> accepting_op_strings;
+    for (auto const& accepting_op : m_accepting_ops) {
+        auto serialized_accepting_op{accepting_op.serialize()};
+        if (serialized_accepting_op.has_value()) {
+            accepting_op_strings.push_back(serialized_accepting_op.value());
+        }
+    }
+    auto const accepting_ops_string = is_accepting() ? fmt::format(
+                                                               "accepting_operations={{{}}},",
+                                                               fmt::join(accepting_op_strings, ",")
+                                                       )
+                                                     : "";
+
+    std::vector<std::string> transition_strings;
+    for (uint32_t idx{0}; idx < cSizeOfByte; ++idx) {
+        auto const byte_transition_string{m_bytes_transition[idx].serialize(state_ids)};
+        if (byte_transition_string.has_value()) {
+            transition_strings.push_back(
+                    fmt::format("{}{}", static_cast<char>(idx), byte_transition_string.value())
+            );
+        }
+    }
+
+    return fmt::format(
+            "{}:{}{}byte_transitions={{{}}}",
+            state_ids.at(this),
+            accepting_tags_string,
+            accepting_ops_string,
+            fmt::join(transition_strings, ",")
+    );
+}
 }  // namespace log_surgeon::finite_automata
 
 #endif  // LOG_SURGEON_FINITE_AUTOMATA_DFA_STATE
diff --git a/src/log_surgeon/finite_automata/DfaTransition.hpp b/src/log_surgeon/finite_automata/DfaTransition.hpp
@@ -0,0 +1,70 @@
+#ifndef LOG_SURGEON_FINITE_AUTOMATA_DFATRANSITION_HPP
+#define LOG_SURGEON_FINITE_AUTOMATA_DFATRANSITION_HPP
+
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include <fmt/core.h>
+#include <fmt/format.h>
+
+#include <log_surgeon/finite_automata/RegisterOperation.hpp>
+#include <log_surgeon/finite_automata/StateType.hpp>
+
+namespace log_surgeon::finite_automata {
+template <StateType state_type>
+class DfaState;
+
+template <StateType state_type>
+class DfaTransition {
+public:
+    DfaTransition() = default;
+
+    DfaTransition(std::vector<RegisterOperation> reg_ops, DfaState<state_type>* dest_state)
+            : m_reg_ops{std::move(reg_ops)},
+              m_dest_state{dest_state} {}
+
+    [[nodiscard]] auto get_reg_ops() const -> std::vector<RegisterOperation> { return m_reg_ops; }
+
+    [[nodiscard]] auto get_dest_state() const -> DfaState<state_type>* { return m_dest_state; }
+
+    /**
+     * @param state_ids A map of states to their unique identifiers.
+     * @return A string representation of the DFA transition on success.
+     * @return Forwards `RegisterOperation::serialize`'s return value (std::nullopt) on failure.
+     * @return std::nullopt if `m_dest_state` is not in `statd_ids`.
+     */
+    [[nodiscard]] auto serialize(
+            std::unordered_map<DfaState<state_type> const*, uint32_t> const& state_ids
+    ) const -> std::optional<std::string>;
+
+private:
+    std::vector<RegisterOperation> m_reg_ops;
+    DfaState<state_type>* m_dest_state{nullptr};
+};
+
+template <StateType state_type>
+auto DfaTransition<state_type>::serialize(
+        std::unordered_map<DfaState<state_type> const*, uint32_t> const& state_ids
+) const -> std::optional<std::string> {
+    if (false == state_ids.contains(m_dest_state)) {
+        return std::nullopt;
+    }
+
+    std::vector<std::string> transformed_ops;
+    for (auto const& reg_op : m_reg_ops) {
+        auto const optional_serialized_op{reg_op.serialize()};
+        if (false == optional_serialized_op.has_value()) {
+            return std::nullopt;
+        }
+        transformed_ops.push_back(optional_serialized_op.value());
+    }
+
+    return fmt::format("-({})->{}", fmt::join(transformed_ops, ","), state_ids.at(m_dest_state));
+}
+}  // namespace log_surgeon::finite_automata
+
+#endif  // LOG_SURGEON_FINITE_AUTOMATA_DFATRANSITION_HPP
diff --git a/src/log_surgeon/finite_automata/RegisterHandler.hpp b/src/log_surgeon/finite_automata/RegisterHandler.hpp
@@ -5,10 +5,9 @@
 #include <vector>
 
 #include <log_surgeon/finite_automata/PrefixTree.hpp>
+#include <log_surgeon/types.hpp>
 
 namespace log_surgeon::finite_automata {
-using register_id_t = uint32_t;
-
 /**
  * The register handler maintains a prefix tree that is sufficient to represent all registers.
  * The register handler also contains a vector of registers, and performs the set, copy, and append
@@ -27,37 +26,36 @@ class RegisterHandler {
         return added_registers;
     }
 
-    auto add_register() -> register_id_t {
+    auto add_register() -> reg_id_t {
         auto const prefix_tree_node_id{
                 m_prefix_tree.insert(PrefixTree::cRootId, PrefixTree::cDefaultPos)
         };
         m_registers.emplace_back(prefix_tree_node_id);
         return m_registers.size() - 1;
     }
 
-    auto add_register(PrefixTree::id_t const prefix_tree_parent_node_id) -> register_id_t {
+    auto add_register(PrefixTree::id_t const prefix_tree_parent_node_id) -> reg_id_t {
         auto const prefix_tree_node_id{
                 m_prefix_tree.insert(prefix_tree_parent_node_id, PrefixTree::cDefaultPos)
         };
         m_registers.emplace_back(prefix_tree_node_id);
         return m_registers.size() - 1;
     }
 
-    auto set_register(register_id_t const reg_id, PrefixTree::position_t const position) -> void {
+    auto set_register(reg_id_t const reg_id, PrefixTree::position_t const position) -> void {
         m_prefix_tree.set(m_registers.at(reg_id), position);
     }
 
-    auto copy_register(register_id_t const dest_reg_id, register_id_t const source_reg_id) -> void {
+    auto copy_register(reg_id_t const dest_reg_id, reg_id_t const source_reg_id) -> void {
         m_registers.at(dest_reg_id) = m_registers.at(source_reg_id);
     }
 
-    auto
-    append_position(register_id_t const reg_id, PrefixTree::position_t const position) -> void {
+    auto append_position(reg_id_t const reg_id, PrefixTree::position_t const position) -> void {
         auto const node_id{m_registers.at(reg_id)};
         m_registers.at(reg_id) = m_prefix_tree.insert(node_id, position);
     }
 
-    [[nodiscard]] auto get_reversed_positions(register_id_t const reg_id
+    [[nodiscard]] auto get_reversed_positions(reg_id_t const reg_id
     ) const -> std::vector<PrefixTree::position_t> {
         return m_prefix_tree.get_reversed_positions(m_registers.at(reg_id));
     }
diff --git a/src/log_surgeon/finite_automata/RegisterOperation.hpp b/src/log_surgeon/finite_automata/RegisterOperation.hpp
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
diff --git a/tests/test-dfa.cpp b/tests/test-dfa.cpp
