Merge branch 'main' into test-reader-parser

SharafMohamed · SharafMohamed · commit e4e35406c919 · 2025-11-06T05:49:10.000-05:00
diff --git a/src/log_surgeon/BufferParser.cpp b/src/log_surgeon/BufferParser.cpp
@@ -32,7 +32,7 @@ BufferParser::parse_next_event(char* buf, size_t size, size_t& offset, bool fini
         if (0 != m_log_parser.get_log_event_view().m_log_output_buffer->pos()) {
             offset = m_log_parser.get_log_event_view()
                              .m_log_output_buffer->get_token(0)
-                             .m_start_pos;
+                             .get_start_pos();
         }
         reset();
         return error_code;
diff --git a/src/log_surgeon/Lalr1Parser.tpp b/src/log_surgeon/Lalr1Parser.tpp
@@ -20,7 +20,7 @@ namespace {
         MatchedSymbol& curr_symbol = symbols.top();
         std::visit(
                 Overloaded{
-                        [&line_num](Token& token) { line_num = token.m_line; },
+                        [&line_num](Token& token) { line_num = token.get_line_num(); },
                         [&symbols](NonTerminal& m) {
                             for (size_t i{0}; i < m.get_production()->m_body.size(); ++i) {
                                 symbols.push(m.move_symbol(i));
@@ -548,7 +548,9 @@ template <typename TypedNfaState, typename TypedDfaState>
 auto Lalr1Parser<TypedNfaState, TypedDfaState>::get_input_until_next_newline(Token* error_token)
         -> std::string {
     std::string rest_of_line;
-    bool next_is_end_token = (error_token->m_type_ids_ptr->at(0) == (uint32_t)SymbolId::TokenEnd);
+    bool next_is_end_token{
+            error_token->get_type_ids()->at(0) == static_cast<uint32_t>(SymbolId::TokenEnd)
+    };
     bool next_has_newline = (error_token->to_string().find('\n') != std::string::npos)
                             || (error_token->to_string().find('\r') != std::string::npos);
     while (!next_has_newline && !next_is_end_token) {
@@ -557,7 +559,8 @@ auto Lalr1Parser<TypedNfaState, TypedDfaState>::get_input_until_next_newline(Tok
                            || (token.to_string().find('\r') != std::string::npos);
         if (!next_has_newline) {
             rest_of_line += token.to_string();
-            next_is_end_token = (token.m_type_ids_ptr->at(0) == (uint32_t)SymbolId::TokenEnd);
+            next_is_end_token
+                    = token.get_type_ids()->at(0) == static_cast<uint32_t>(SymbolId::TokenEnd);
         }
     }
     rest_of_line += "\n";
@@ -581,7 +584,9 @@ auto Lalr1Parser<TypedNfaState, TypedDfaState>::report_error() -> std::string {
         error_indicator += " ";
     }
     error_indicator += "^\n";
-    if (token.m_type_ids_ptr->at(0) == (uint32_t)SymbolId::TokenEnd && consumed_input.empty()) {
+    if (token.get_type_ids()->at(0) == static_cast<uint32_t>(SymbolId::TokenEnd)
+        && consumed_input.empty())
+    {
         error_type = "empty file";
         error_indicator = "^\n";
     } else {
@@ -667,7 +672,7 @@ auto Lalr1Parser<TypedNfaState, TypedDfaState>::get_next_symbol() -> Token {
 template <typename TypedNfaState, typename TypedDfaState>
 auto Lalr1Parser<TypedNfaState, TypedDfaState>::parse_advance(Token& next_token, bool* accept)
         -> bool {
-    for (auto const type : *next_token.m_type_ids_ptr) {
+    for (auto const type : *next_token.get_type_ids()) {
         if (parse_symbol(type, next_token, accept)) {
             return *accept;
         }
@@ -718,12 +723,12 @@ auto Lalr1Parser<TypedNfaState, TypedDfaState>::parse_symbol(
                             m_parse_stack_matches.pop();
                         }
                         if (reduce->m_semantic_rule != nullptr) {
-                            if (0 == m_next_token->m_start_pos) {
+                            if (0 == m_next_token->get_start_pos()) {
                                 m_input_buffer.set_consumed_pos(
                                         m_input_buffer.storage().size() - 1
                                 );
                             } else {
-                                m_input_buffer.set_consumed_pos(m_next_token->m_start_pos - 1);
+                                m_input_buffer.set_consumed_pos(m_next_token->get_start_pos() - 1);
                             }
                             matched_non_terminal.set_parser_ast(
                                     reduce->m_semantic_rule(&matched_non_terminal)
diff --git a/src/log_surgeon/Lexer.tpp b/src/log_surgeon/Lexer.tpp
@@ -60,7 +60,7 @@ auto Lexer<TypedNfaState, TypedDfaState>::scan(ParserInputBuffer& input_buffer)
                     input_buffer.storage().size(),
                     m_match_line,
                     m_type_ids,
-                    std::move(m_dfa->release_reg_handler())
+                    m_dfa->release_reg_handler()
             };
             return {ErrorCode::Success, token};
         }
@@ -141,7 +141,7 @@ auto Lexer<TypedNfaState, TypedDfaState>::scan(ParserInputBuffer& input_buffer)
                         input_buffer.storage().size(),
                         m_match_line,
                         m_type_ids,
-                        std::move(m_dfa->release_reg_handler())
+                        m_dfa->release_reg_handler()
                 };
                 return {ErrorCode::Success, token};
             }
diff --git a/src/log_surgeon/LogEvent.cpp b/src/log_surgeon/LogEvent.cpp
@@ -56,15 +56,14 @@ auto LogEventView::get_logtype() const -> std::string {
     }
     for (uint32_t i{1}; i < m_log_output_buffer->pos(); ++i) {
         auto token_view{m_log_output_buffer->get_mutable_token(i)};
-        auto const rule_id{token_view.m_type_ids_ptr->at(0)};
+        auto const rule_id{token_view.get_type_ids()->at(0)};
         if (static_cast<uint32_t>(SymbolId::TokenUncaughtString) == rule_id) {
             logtype += token_view.to_string_view();
         } else {
             bool const is_first_token{false == m_log_output_buffer->has_timestamp() && 1 == i};
             if (static_cast<uint32_t>(SymbolId::TokenNewline) != rule_id && false == is_first_token)
             {
-                logtype += token_view.get_delimiter();
-                token_view.m_start_pos++;
+                logtype += token_view.release_delimiter();
             }
             if (auto const& optional_capture_ids{
                         m_log_parser.m_lexer.get_capture_ids_from_rule_id(rule_id)
@@ -91,13 +90,13 @@ auto LogEventView::get_logtype() const -> std::string {
                     if (false == start_positions.empty() && -1 < start_positions[0]
                         && false == end_positions.empty() && -1 < end_positions[0])
                     {
-                        capture_view.m_end_pos = start_positions[0];
+                        capture_view.set_end_pos(start_positions[0]);
                         logtype.append(capture_view.to_string_view());
                         logtype.append("<" + capture_name + ">");
-                        capture_view.m_start_pos = end_positions[0];
+                        capture_view.set_start_pos(end_positions[0]);
                     }
                 }
-                capture_view.m_end_pos = token_view.m_end_pos;
+                capture_view.set_end_pos(token_view.get_end_pos());
                 logtype.append(capture_view.to_string_view());
             } else {
                 logtype += "<" + m_log_parser.get_id_symbol(rule_id) + ">";
@@ -140,14 +139,14 @@ LogEvent::LogEvent(LogEventView const& src) : LogEventView{src.get_log_parser()}
                 m_buffer.data(),
                 buffer_size,
                 0,
-                token.m_type_ids_ptr
+                token.get_type_ids()
         };
         m_log_output_buffer->set_curr_token(copied_token);
         m_log_output_buffer->advance_to_next_token();
     }
     for (uint32_t i = 0; i < get_log_output_buffer()->pos(); i++) {
         Token& token = get_log_output_buffer()->get_mutable_token(i);
-        auto const& token_types = *token.m_type_ids_ptr;
+        auto const& token_types{*token.get_type_ids()};
         add_token(token_types[0], &token);
     }
 }
diff --git a/src/log_surgeon/LogParser.cpp b/src/log_surgeon/LogParser.cpp
@@ -137,39 +137,37 @@ auto LogParser::parse(LogParser::ParsingAction& parsing_action) -> ErrorCode {
             }
             next_token = optional_next_token.value();
             if (false == output_buffer->has_timestamp()
-                && next_token.m_type_ids_ptr->at(0) == (uint32_t)SymbolId::TokenNewlineTimestamp)
+                && next_token.get_type_ids()->at(0)
+                           == static_cast<uint32_t>(SymbolId::TokenNewlineTimestamp))
             {
                 // TODO: combine the below with found_start_of_next_message
                 // into 1 function
                 // Increment by 1 because the '\n' character is not part of the
                 // next log message
                 m_start_of_log_message = next_token;
-                if (m_start_of_log_message.m_start_pos == m_start_of_log_message.m_buffer_size - 1)
-                {
-                    m_start_of_log_message.m_start_pos = 0;
-                } else {
-                    m_start_of_log_message.m_start_pos++;
-                }
+                m_start_of_log_message.increment_start_pos();
                 // make a message with just the '\n' character
-                next_token.m_end_pos = next_token.m_start_pos + 1;
-                next_token.m_type_ids_ptr
-                        = &Lexer<ByteNfaState, ByteDfaState>::cTokenUncaughtStringTypes;
+                next_token.set_end_pos(next_token.get_next_pos());
+                next_token.set_type_ids(
+                        &Lexer<ByteNfaState, ByteDfaState>::cTokenUncaughtStringTypes
+                );
                 output_buffer->set_token(1, next_token);
                 output_buffer->set_pos(2);
-                m_input_buffer.set_consumed_pos(next_token.m_start_pos);
+                m_input_buffer.set_consumed_pos(next_token.get_start_pos());
                 m_has_start_of_log = true;
                 parsing_action = ParsingAction::Compress;
                 return ErrorCode::Success;
             }
         }
-        if (next_token.m_type_ids_ptr->at(0) == (uint32_t)SymbolId::TokenEnd) {
+        if (next_token.get_type_ids()->at(0) == static_cast<uint32_t>(SymbolId::TokenEnd)) {
             output_buffer->set_token(0, next_token);
             output_buffer->set_pos(1);
             parsing_action = ParsingAction::CompressAndFinish;
             return ErrorCode::Success;
         }
-        if (next_token.m_type_ids_ptr->at(0) == (uint32_t)SymbolId::TokenFirstTimestamp
-            || next_token.m_type_ids_ptr->at(0) == (uint32_t)SymbolId::TokenNewlineTimestamp)
+        if (next_token.get_type_ids()->at(0) == static_cast<uint32_t>(SymbolId::TokenFirstTimestamp)
+            || next_token.get_type_ids()->at(0)
+                       == static_cast<uint32_t>(SymbolId::TokenNewlineTimestamp))
         {
             output_buffer->set_has_timestamp(true);
             output_buffer->set_token(0, next_token);
@@ -189,11 +187,11 @@ auto LogParser::parse(LogParser::ParsingAction& parsing_action) -> ErrorCode {
         }
         Token next_token{optional_next_token.value()};
         output_buffer->set_curr_token(next_token);
-        auto token_type = next_token.m_type_ids_ptr->at(0);
+        auto token_type{next_token.get_type_ids()->at(0)};
         bool found_start_of_next_message
                 = (output_buffer->has_timestamp()
                    && token_type == (uint32_t)SymbolId::TokenNewlineTimestamp)
-                  || (!output_buffer->has_timestamp() && next_token.get_char(0) == '\n'
+                  || (false == output_buffer->has_timestamp() && next_token.get_delimiter() == "\n"
                       && token_type != (uint32_t)SymbolId::TokenNewline);
         if (token_type == (uint32_t)SymbolId::TokenEnd) {
             parsing_action = ParsingAction::CompressAndFinish;
@@ -202,7 +200,7 @@ auto LogParser::parse(LogParser::ParsingAction& parsing_action) -> ErrorCode {
         if (false == output_buffer->has_timestamp()
             && token_type == (uint32_t)SymbolId::TokenNewline)
         {
-            m_input_buffer.set_consumed_pos(output_buffer->get_curr_token().m_end_pos);
+            m_input_buffer.set_consumed_pos(output_buffer->get_curr_token().get_end_pos());
             output_buffer->advance_to_next_token();
             parsing_action = ParsingAction::Compress;
             return ErrorCode::Success;
@@ -211,22 +209,13 @@ auto LogParser::parse(LogParser::ParsingAction& parsing_action) -> ErrorCode {
             // increment by 1 because the '\n' character is not part of the next
             // log message
             m_start_of_log_message = output_buffer->get_curr_token();
-            if (m_start_of_log_message.m_start_pos == m_start_of_log_message.m_buffer_size - 1) {
-                m_start_of_log_message.m_start_pos = 0;
-            } else {
-                m_start_of_log_message.m_start_pos++;
-            }
+            auto const consumed_pos{m_start_of_log_message.increment_start_pos()};
             // make the last token of the current message the '\n' character
             Token curr_token = output_buffer->get_curr_token();
-            curr_token.m_end_pos = curr_token.m_start_pos + 1;
-            curr_token.m_type_ids_ptr
-                    = &Lexer<ByteNfaState, ByteDfaState>::cTokenUncaughtStringTypes;
+            curr_token.set_end_pos(curr_token.get_next_pos());
+            curr_token.set_type_ids(&Lexer<ByteNfaState, ByteDfaState>::cTokenUncaughtStringTypes);
             output_buffer->set_curr_token(curr_token);
-            if (0 == m_start_of_log_message.m_start_pos) {
-                m_input_buffer.set_consumed_pos(m_input_buffer.storage().size() - 1);
-            } else {
-                m_input_buffer.set_consumed_pos(m_start_of_log_message.m_start_pos - 1);
-            }
+            m_input_buffer.set_consumed_pos(consumed_pos);
             m_has_start_of_log = true;
             output_buffer->advance_to_next_token();
             parsing_action = ParsingAction::Compress;
@@ -255,7 +244,7 @@ auto LogParser::generate_log_event_view_metadata() -> void {
     uint32_t first_newline_pos{0};
     for (uint32_t i = start; i < m_log_event_view->m_log_output_buffer->pos(); i++) {
         Token* token = &m_log_event_view->m_log_output_buffer->get_mutable_token(i);
-        m_log_event_view->add_token(token->m_type_ids_ptr->at(0), token);
+        m_log_event_view->add_token(token->get_type_ids()->at(0), token);
         if (token->get_delimiter() == "\n" && first_newline_pos == 0) {
             first_newline_pos = i;
         }
diff --git a/src/log_surgeon/SchemaParser.cpp b/src/log_surgeon/SchemaParser.cpp
@@ -132,7 +132,7 @@ static auto schema_var_rule(NonTerminal* m) -> unique_ptr<SchemaVarAST> {
     return make_unique<SchemaVarAST>(
             identifier_ast.m_name,
             std::move(m->non_terminal_cast(3).get_parser_ast().get<unique_ptr<RegexASTByte>>()),
-            m->token_cast(2).m_line
+            m->token_cast(2).get_line_num()
     );
 }
 
diff --git a/src/log_surgeon/Token.cpp b/src/log_surgeon/Token.cpp
@@ -1,47 +1,78 @@
 #include "Token.hpp"
 
-#include <algorithm>
+#include <cstddef>
 #include <string>
 #include <string_view>
 
 namespace log_surgeon {
-auto Token::to_string() -> std::string {
-    if (m_start_pos <= m_end_pos) {
-        return {m_buffer + m_start_pos, m_buffer + m_end_pos};
-    }
-    if (m_wrap_around_string.empty()) {
-        m_wrap_around_string = std::string{m_buffer + m_start_pos, m_buffer + m_buffer_size}
-                               + std::string{m_buffer, m_buffer + m_end_pos};
+auto Token::get_cached_string() -> std::string const& {
+    if (m_cached_string.empty()) {
+        if (get_start_pos() <= get_end_pos()) {
+            auto const token{m_buffer.subspan(get_start_pos(), get_end_pos() - get_start_pos())};
+            m_cached_string = std::string{token.begin(), token.end()};
+        } else {
+            auto const token_start{
+                    m_buffer.subspan(get_start_pos(), get_buffer_size() - get_start_pos())
+            };
+            auto const token_end{m_buffer.subspan(0, get_end_pos())};
+            m_cached_string = std::string{token_start.begin(), token_start.end()}
+                              + std::string{token_end.begin(), token_end.end()};
+        }
     }
-    return {m_wrap_around_string};
+    return m_cached_string;
+}
+
+auto Token::to_string() -> std::string {
+    return {get_cached_string()};
 }
 
 auto Token::to_string_view() -> std::string_view {
-    if (m_start_pos <= m_end_pos) {
-        return {m_buffer + m_start_pos, m_end_pos - m_start_pos};
-    }
-    if (m_wrap_around_string.empty()) {
-        m_wrap_around_string = std::string{m_buffer + m_start_pos, m_buffer + m_buffer_size}
-                               + std::string{m_buffer, m_buffer + m_end_pos};
+    if (get_start_pos() <= get_end_pos()) {
+        auto const token{m_buffer.subspan(get_start_pos(), get_end_pos() - get_start_pos())};
+        return {token.begin(), token.end()};
     }
-    return {m_wrap_around_string};
+    return {get_cached_string()};
+}
+
+auto Token::get_delimiter() const -> std::string {
+    auto const delim{m_buffer.subspan(get_start_pos(), 1)};
+    return {delim.begin(), delim.end()};
 }
 
-auto Token::get_char(uint8_t i) const -> char {
-    if (m_start_pos + i < m_buffer_size) {
-        return m_buffer[m_start_pos + i];
+auto Token::get_length() const -> size_t {
+    if (get_start_pos() <= get_end_pos()) {
+        return get_end_pos() - get_start_pos();
     }
-    return m_buffer[i - (m_buffer_size - m_start_pos)];
+    return get_buffer_size() - get_start_pos() + get_end_pos();
 }
 
-auto Token::get_delimiter() const -> std::string {
-    return {m_buffer + m_start_pos, m_buffer + m_start_pos + 1};
+auto Token::release_delimiter() -> char {
+    auto const delim{m_buffer[get_start_pos()]};
+    increment_start_pos();
+    return delim;
+}
+
+auto Token::set_start_pos(size_t pos) -> void {
+    m_cached_string.clear();
+    m_start_pos = pos;
+}
+
+auto Token::set_end_pos(size_t pos) -> void {
+    m_cached_string.clear();
+    m_end_pos = pos;
+}
+
+auto Token::increment_start_pos() -> size_t {
+    auto const old_start_pos{get_start_pos()};
+    set_start_pos(get_next_pos());
+    return old_start_pos;
 }
 
-auto Token::get_length() const -> uint32_t {
-    if (m_start_pos <= m_end_pos) {
-        return m_end_pos - m_start_pos;
+auto Token::get_next_pos() const -> size_t {
+    auto next_pos{get_start_pos() + 1};
+    if (next_pos == get_buffer_size()) {
+        next_pos = 0;
     }
-    return m_buffer_size - m_start_pos + m_end_pos;
+    return next_pos;
 }
 }  // namespace log_surgeon
diff --git a/src/log_surgeon/Token.hpp b/src/log_surgeon/Token.hpp
diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml
diff --git a/tests/test-buffer-parser.cpp b/tests/test-buffer-parser.cpp

Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,7 @@ BufferParser::parse_next_event(char* buf, size_t size, size_t& offset, bool fini`
`32`	`32`	`if (0 != m_log_parser.get_log_event_view().m_log_output_buffer->pos()) {`
`33`	`33`	`offset = m_log_parser.get_log_event_view()`
`34`	`34`	`.m_log_output_buffer->get_token(0)`
`35`		`- .m_start_pos;`
	`35`	`+ .get_start_pos();`
`36`	`36`	`}`
`37`	`37`	`reset();`
`38`	`38`	`return error_code;`
Original file line number	Diff line number	Diff line change
`@@ -132,7 +132,7 @@ static auto schema_var_rule(NonTerminal* m) -> unique_ptr<SchemaVarAST> {`
`132`	`132`	`return make_unique<SchemaVarAST>(`
`133`	`133`	`identifier_ast.m_name,`
`134`	`134`	`std::move(m->non_terminal_cast(3).get_parser_ast().get<unique_ptr<RegexASTByte>>()),`
`135`		`- m->token_cast(2).m_line`
	`135`	`+ m->token_cast(2).get_line_num()`
`136`	`136`	`);`
`137`	`137`	`}`
`138`	`138`