feat: Separate concept tags from captures and store capture-to-tag mapping in the lexer.

Co-authored-by: Lin Zhihao <[email protected]>

Author: SharafMohamed

CMakeLists.txt

@@ -68,6 +68,20 @@ set(SOURCE_FILES
     src/log_surgeon/Constants.hpp
     src/log_surgeon/FileReader.cpp
     src/log_surgeon/FileReader.hpp
+    src/log_surgeon/finite_automata/Capture.hpp
+    src/log_surgeon/finite_automata/Dfa.hpp
+    src/log_surgeon/finite_automata/DfaState.hpp
+    src/log_surgeon/finite_automata/DfaStatePair.hpp
+    src/log_surgeon/finite_automata/Nfa.hpp
+    src/log_surgeon/finite_automata/NfaState.hpp
+    src/log_surgeon/finite_automata/PrefixTree.cpp
+    src/log_surgeon/finite_automata/PrefixTree.hpp
+    src/log_surgeon/finite_automata/RegexAST.hpp
+    src/log_surgeon/finite_automata/RegisterHandler.hpp
+    src/log_surgeon/finite_automata/StateType.hpp
+    src/log_surgeon/finite_automata/TaggedTransition.hpp
+    src/log_surgeon/finite_automata/UnicodeIntervalTree.hpp
+    src/log_surgeon/finite_automata/UnicodeIntervalTree.tpp
     src/log_surgeon/Lalr1Parser.cpp
     src/log_surgeon/Lalr1Parser.hpp
     src/log_surgeon/Lalr1Parser.tpp
@@ -93,20 +107,8 @@ set(SOURCE_FILES
     src/log_surgeon/SchemaParser.hpp
     src/log_surgeon/Token.cpp
     src/log_surgeon/Token.hpp
-    src/log_surgeon/finite_automata/PrefixTree.cpp
-    src/log_surgeon/finite_automata/PrefixTree.hpp
-    src/log_surgeon/finite_automata/RegexAST.hpp
-    src/log_surgeon/finite_automata/Dfa.hpp
-    src/log_surgeon/finite_automata/DfaState.hpp
-    src/log_surgeon/finite_automata/DfaStatePair.hpp
-    src/log_surgeon/finite_automata/Nfa.hpp
-    src/log_surgeon/finite_automata/NfaState.hpp
-    src/log_surgeon/finite_automata/RegisterHandler.hpp
-    src/log_surgeon/finite_automata/StateType.hpp
-    src/log_surgeon/finite_automata/Tag.hpp
-    src/log_surgeon/finite_automata/TaggedTransition.hpp
-    src/log_surgeon/finite_automata/UnicodeIntervalTree.hpp
-    src/log_surgeon/finite_automata/UnicodeIntervalTree.tpp
+    src/log_surgeon/types.hpp
+    src/log_surgeon/UniqueIdGenerator.hpp
     )
 
 set(LCHIP_INSTALL_CONFIG_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/log_surgeon)

src/log_surgeon/Lexer.hpp

@@ -2,22 +2,22 @@
 #define LOG_SURGEON_LEXER_HPP
 
 #include <array>
-#include <bitset>
 #include <cstdint>
 #include <memory>
+#include <optional>
 #include <string>
 #include <unordered_map>
-#include <unordered_set>
+#include <utility>
 #include <vector>
 
 #include <log_surgeon/Constants.hpp>
 #include <log_surgeon/finite_automata/Dfa.hpp>
 #include <log_surgeon/finite_automata/DfaState.hpp>
-#include <log_surgeon/finite_automata/Nfa.hpp>
 #include <log_surgeon/finite_automata/RegexAST.hpp>
 #include <log_surgeon/LexicalRule.hpp>
 #include <log_surgeon/ParserInputBuffer.hpp>
 #include <log_surgeon/Token.hpp>
+#include <log_surgeon/types.hpp>
 
 namespace log_surgeon {
 template <typename TypedNfaState, typename TypedDfaState>
@@ -35,13 +35,11 @@ class Lexer {
 
     /**
      * Add lexical rule to the lexer's list of rules
-     * @param id
-     * @param regex
+     * @param rule_id
+     * @param rule
      */
-    auto add_rule(
-            uint32_t const& id,
-            std::unique_ptr<finite_automata::RegexAST<TypedNfaState>> rule
-    ) -> void;
+    auto add_rule(rule_id_t rule_id, std::unique_ptr<finite_automata::RegexAST<TypedNfaState>> rule)
+            -> void;
 
     /**
      * Return regex pattern for a rule name
@@ -51,7 +49,8 @@ class Lexer {
     auto get_rule(uint32_t variable_id) -> finite_automata::RegexAST<TypedNfaState>*;
 
     /**
-     * Generate DFA for lexer
+     * Generate DFA for lexer.
+     * @throw std::invalid_argument if `m_rules` contains multipe captures with the same name.
      */
     auto generate() -> void;
 
@@ -122,8 +121,75 @@ class Lexer {
         return m_dfa;
     }
 
-    std::unordered_map<std::string, uint32_t> m_symbol_id;
-    std::unordered_map<uint32_t, std::string> m_id_symbol;
+    /**
+     * @param rule_id ID associated with a rule.
+     * @return A vector of capture IDs corresponding to each rule that contain the variable on
+     * success.
+     * @return std::nullopt if the variable is never captured in any rule.
+     */
+    [[nodiscard]] auto get_capture_ids_from_rule_id(rule_id_t const rule_id
+    ) const -> std::optional<std::vector<capture_id_t>> {
+        if (m_rule_id_to_capture_ids.contains(rule_id)) {
+            return m_rule_id_to_capture_ids.at(rule_id);
+        }
+        return std::nullopt;
+    }
+
+    /**
+     * @param capture_id ID associated with a capture within a rule.
+     * @return The start and end tag of the capture on success.
+     * @return std::nullopt if no capture is associated with the given capture ID.
+     */
+    [[nodiscard]] auto get_tag_id_pair_from_capture_id(capture_id_t const capture_id
+    ) const -> std::optional<std::pair<tag_id_t, tag_id_t>> {
+        if (m_capture_id_to_tag_id_pair.contains(capture_id)) {
+            return m_capture_id_to_tag_id_pair.at(capture_id);
+        }
+        return std::nullopt;
+    }
+
+    /**
+     * @param tag_id ID associated with a tag.
+     * @return The final register ID tracking the value of the tag ID during DFA simulation on
+     * success.
+     * @return std::nullopt if no tag is associated with the given tag ID.
+     */
+    [[nodiscard]] auto get_reg_id_from_tag_id(tag_id_t const tag_id
+    ) const -> std::optional<reg_id_t> {
+        if (m_tag_to_reg_id.contains(tag_id)) {
+            return m_tag_to_reg_id.at(tag_id);
+        }
+        return std::nullopt;
+    }
+
+    /**
+     * @param capture_id ID associated with a capture within a rule.
+     * @return The start and end final register IDs tracking the position of the capture on success.
+     * @return std::nullopt if no capture is associated with the given capture ID.
+     */
+    [[nodiscard]] auto get_reg_ids_from_capture_id(capture_id_t const capture_id
+    ) const -> std::optional<std::pair<reg_id_t, reg_id_t>> {
+        auto const optional_tag_id_pair{get_tag_id_pair_from_capture_id(capture_id)};
+        if (false == optional_tag_id_pair.has_value()) {
+            return std::nullopt;
+        }
+        auto const [start_tag_id, end_tag_id]{optional_tag_id_pair.value()};
+
+        auto const optional_start_reg_id{get_reg_id_from_tag_id(start_tag_id)};
+        if (false == optional_start_reg_id.has_value()) {
+            return std::nullopt;
+        }
+
+        auto const optional_end_reg_id{get_reg_id_from_tag_id(end_tag_id)};
+        if (false == optional_end_reg_id.has_value()) {
+            return std::nullopt;
+        }
+
+        return {optional_start_reg_id.value(), optional_end_reg_id.value()};
+    }
+
+    std::unordered_map<std::string, rule_id_t> m_symbol_id;
+    std::unordered_map<rule_id_t, std::string> m_id_symbol;
 
 private:
     /**
@@ -148,6 +214,9 @@ class Lexer {
     std::unique_ptr<finite_automata::Dfa<TypedDfaState>> m_dfa;
     bool m_asked_for_more_data{false};
     TypedDfaState const* m_prev_state{nullptr};
+    std::unordered_map<rule_id_t, std::vector<capture_id_t>> m_rule_id_to_capture_ids;
+    std::unordered_map<capture_id_t, std::pair<tag_id_t, tag_id_t>> m_capture_id_to_tag_id_pair;
+    std::unordered_map<tag_id_t, reg_id_t> m_tag_to_reg_id;
 };
 
 namespace lexers {

src/log_surgeon/Lexer.tpp

@@ -4,11 +4,13 @@
 #include <cassert>
 #include <memory>
 #include <stack>
+#include <stdexcept>
 #include <string>
 #include <vector>
 
 #include <log_surgeon/Constants.hpp>
 #include <log_surgeon/finite_automata/RegexAST.hpp>
+#include <log_surgeon/types.hpp>
 
 /**
  * utf8 format (https://en.wikipedia.org/wiki/UTF-8)
@@ -358,17 +360,17 @@ void Lexer<TypedNfaState, TypedDfaState>::add_delimiters(std::vector<uint32_t> c
 
 template <typename TypedNfaState, typename TypedDfaState>
 void Lexer<TypedNfaState, TypedDfaState>::add_rule(
-        uint32_t const& id,
+        rule_id_t const rule_id,
         std::unique_ptr<finite_automata::RegexAST<TypedNfaState>> rule
 ) {
-    m_rules.emplace_back(id, std::move(rule));
+    m_rules.emplace_back(rule_id, std::move(rule));
 }
 
 template <typename TypedNfaState, typename TypedDfaState>
-auto Lexer<TypedNfaState, TypedDfaState>::get_rule(uint32_t const variable_id
+auto Lexer<TypedNfaState, TypedDfaState>::get_rule(rule_id_t const rule_id
 ) -> finite_automata::RegexAST<TypedNfaState>* {
     for (auto const& rule : m_rules) {
-        if (rule.get_variable_id() == variable_id) {
+        if (rule.get_variable_id() == rule_id) {
             return rule.get_regex();
         }
     }
@@ -377,8 +379,31 @@ auto Lexer<TypedNfaState, TypedDfaState>::get_rule(uint32_t const variable_id
 
 template <typename TypedNfaState, typename TypedDfaState>
 void Lexer<TypedNfaState, TypedDfaState>::generate() {
-    finite_automata::Nfa<TypedNfaState> nfa{std::move(m_rules)};
-    // TODO: DFA ignores tags. E.g., treats "capture:user=(?<user_id>\d+)" as "capture:user=\d+"
+    for (auto const& rule : m_rules) {
+        for (auto const* capture : rule.get_captures()) {
+            std::string const capture_name{capture->get_name()};
+            if (m_symbol_id.contains(capture_name)) {
+                throw std::invalid_argument("`m_rules` contains capture names that are not unique."
+                );
+            }
+            auto const capture_id{m_symbol_id.size()};
+            m_symbol_id.emplace(capture_name, capture_id);
+            m_id_symbol.emplace(capture_id, capture_name);
+
+            auto const rule_id{rule.get_variable_id()};
+            m_rule_id_to_capture_ids.try_emplace(rule_id);
+            m_rule_id_to_capture_ids.at(rule_id).push_back(capture_id);
+        }
+    }
+
+    finite_automata::Nfa<TypedNfaState> nfa{m_rules};
+    for (auto const& [capture, tag_id_pair] : nfa.get_capture_to_tag_id_pair()) {
+        std::string const capture_name{capture->get_name()};
+        auto const capture_id{m_symbol_id.at(capture_name)};
+        m_capture_id_to_tag_id_pair.emplace(capture_id, tag_id_pair);
+    }
+
+    // TODO: DFA ignores captures. E.g., treats "capture:user=(?<user_id>\d+)" as "capture:user=\d+"
     m_dfa = std::make_unique<finite_automata::Dfa<TypedDfaState>>(std::move(nfa));
     auto const* state = m_dfa->get_root();
     for (uint32_t i = 0; i < cSizeOfByte; i++) {

src/log_surgeon/LexicalRule.hpp

@@ -23,6 +23,10 @@ class LexicalRule {
      */
     auto add_to_nfa(finite_automata::Nfa<TypedNfaState>* nfa) const -> void;
 
+    [[nodiscard]] auto get_captures() const -> std::vector<finite_automata::Capture const*> const& {
+        return m_regex->get_subtree_positive_captures();
+    }
+
     [[nodiscard]] auto get_variable_id() const -> uint32_t { return m_variable_id; }
 
     [[nodiscard]] auto get_regex() const -> finite_automata::RegexAST<TypedNfaState>* {
@@ -40,7 +44,7 @@ void LexicalRule<TypedNfaState>::add_to_nfa(finite_automata::Nfa<TypedNfaState>*
     auto* end_state = nfa->new_state();
     end_state->set_accepting(true);
     end_state->set_matching_variable_id(m_variable_id);
-    m_regex->add_to_nfa_with_negative_tags(nfa, end_state);
+    m_regex->add_to_nfa_with_negative_captures(nfa, end_state);
 }
 }  // namespace log_surgeon
 

src/log_surgeon/SchemaParser.cpp

@@ -9,8 +9,8 @@
 
 #include <log_surgeon/Constants.hpp>
 #include <log_surgeon/FileReader.hpp>
+#include <log_surgeon/finite_automata/Capture.hpp>
 #include <log_surgeon/finite_automata/RegexAST.hpp>
-#include <log_surgeon/finite_automata/Tag.hpp>
 #include <log_surgeon/Lalr1Parser.hpp>
 #include <log_surgeon/Lexer.hpp>
 #include <log_surgeon/utils.hpp>
@@ -167,7 +167,7 @@ static auto regex_capture_rule(NonTerminal const* m) -> std::unique_ptr<ParserAS
     auto& r6 = m->non_terminal_cast(5)->get_parser_ast()->get<unique_ptr<RegexASTByte>>();
     return std::make_unique<ParserValueRegex>(make_unique<RegexASTCaptureByte>(
             std::move(r6),
-            std::make_unique<finite_automata::Tag>(r4->m_name)
+            std::make_unique<finite_automata::Capture>(r4->m_name)
     ));
 }
 
@@ -202,7 +202,7 @@ static auto regex_or_rule(NonTerminal* m) -> unique_ptr<ParserAST> {
 static auto regex_match_zero_or_more_rule(NonTerminal* m) -> unique_ptr<ParserAST> {
     auto& r1 = m->non_terminal_cast(0)->get_parser_ast()->get<unique_ptr<RegexASTByte>>();
 
-    // To handle negative tags we treat `R*` as `R+ | ∅`.
+    // To handle negative captures we treat `R*` as `R+ | ∅`.
     return make_unique<ParserValueRegex>(make_unique<RegexASTOrByte>(
             make_unique<RegexASTEmptyByte>(),
             make_unique<RegexASTMultiplicationByte>(std::move(r1), 1, 0)
@@ -248,7 +248,7 @@ static auto regex_match_range_rule(NonTerminal* m) -> unique_ptr<ParserAST> {
     auto& r1 = m->non_terminal_cast(0)->get_parser_ast()->get<unique_ptr<RegexASTByte>>();
 
     if (0 == min) {
-        // To handle negative tags we treat `R*` as `R+ | ∅`.
+        // To handle negative captures we treat `R*` as `R+ | ∅`.
         return make_unique<ParserValueRegex>(make_unique<RegexASTOrByte>(
                 make_unique<RegexASTEmptyByte>(),
                 make_unique<RegexASTMultiplicationByte>(std::move(r1), 1, max)

src/log_surgeon/UniqueIdGenerator.hpp

@@ -0,0 +1,16 @@
+#ifndef LOG_SURGEON_UNIQUEIDGENERATOR_HPP
+#define LOG_SURGEON_UNIQUEIDGENERATOR_HPP
+
+#include <cstdint>
+
+namespace log_surgeon {
+class UniqueIdGenerator {
+public:
+    [[nodiscard]] auto generate_id() -> uint32_t { return m_current_id++; }
+
+private:
+    uint32_t m_current_id{0};
+};
+}  // namespace log_surgeon
+
+#endif  // LOG_SURGEON_UNIQUEIDGENERATOR_HPP

src/log_surgeon/finite_automata/Tag.hpp …/log_surgeon/finite_automata/Capture.hppsrc/log_surgeon/finite_automata/Tag.hpp renamed to src/log_surgeon/finite_automata/Capture.hpp src/log_surgeon/finite_automata/Tag.hpp renamed to src/log_surgeon/finite_automata/Capture.hpp

@@ -1,14 +1,14 @@
-#ifndef LOG_SURGEON_FINITE_AUTOMATA_TAG
-#define LOG_SURGEON_FINITE_AUTOMATA_TAG
+#ifndef LOG_SURGEON_FINITE_AUTOMATA_CAPTURE
+#define LOG_SURGEON_FINITE_AUTOMATA_CAPTURE
 
 #include <string>
 #include <string_view>
 #include <utility>
 
 namespace log_surgeon::finite_automata {
-class Tag {
+class Capture {
 public:
-    explicit Tag(std::string name) : m_name{std::move(name)} {}
+    explicit Capture(std::string name) : m_name{std::move(name)} {}
 
     [[nodiscard]] auto get_name() const -> std::string_view { return m_name; }
 
@@ -17,4 +17,4 @@ class Tag {
 };
 }  // namespace log_surgeon::finite_automata
 
-#endif  // LOG_SURGEON_FINITE_AUTOMATA_TAG
+#endif  // LOG_SURGEON_FINITE_AUTOMATA_CAPTURE

src/log_surgeon/finite_automata/Dfa.hpp

@@ -2,6 +2,7 @@
 #define LOG_SURGEON_FINITE_AUTOMATA_DFA_HPP
 
 #include <cstdint>
+#include <map>
 #include <memory>
 #include <set>
 #include <vector>