Skip to content

Fix loganalyzer.py UnicodeDecodeError issue #6524

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ZhaohuiS merged 1 commit into from
Oct 12, 2022
Merged

Fix loganalyzer.py UnicodeDecodeError issue #6524

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions ansible/library/extract_log.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,9 @@ def extract_lines(directory, filename, target_string):
path = os.path.join(directory, filename)
file = None
if 'gz' in path:
file = gzip.open(path, mode='rt')
file = gzip.open(path, mode='rt', errors='ignore')
else:
file = open(path)
file = open(path, errors='ignore')
result = None
with file:
# This might be a gunzip file or logrotate issue, there has
Expand Down Expand Up @@ -226,17 +226,17 @@ def combine_logs_and_save(directory, filenames, start_string, target_string, tar
do_copy = False
line_processed = 0
line_copied = 0
with open(target_filename, 'w') as fp:
with open(target_filename, 'w', errors='ignore') as fp:
for filename in reversed(filenames):
path = os.path.join(directory, filename)
dt = datetime.datetime.fromtimestamp(os.path.getctime(path))
sz = os.path.getsize(path)
logger.debug("extract_log combine_logs from file {} create time {}, size {}".format(path, dt, sz))
file = None
if 'gz' in path:
file = gzip.open(path, mode='rt')
file = gzip.open(path, mode='rt', errors='ignore')
else:
file = open(path)
file = open(path, errors='ignore')

with file:
for line in file:
Expand Down
101 changes: 54 additions & 47 deletions ansible/roles/test/files/tools/loganalyzer/loganalyzer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
sudo python loganalyzer.py --out_dir /home/hrachya/projects/loganalyzer/log.analyzer.results --action analyze --run_id myTest114 --logs file3.log -m /home/hrachya/projects/loganalyzer/match.file.1.log,/home/hrachya/projects/loganalyzer/match.file.2.log -i ignore.file.1.log,ignore.file.2.log -v
'''

#---------------------------------------------------------------------
# ---------------------------------------------------------------------
# Global imports
#---------------------------------------------------------------------
# ---------------------------------------------------------------------
from __future__ import print_function
import sys
import getopt
Expand All @@ -30,15 +30,15 @@
import subprocess
from datetime import datetime

#---------------------------------------------------------------------
# ---------------------------------------------------------------------
# Global variables
#---------------------------------------------------------------------
# ---------------------------------------------------------------------
tokenizer = ','
comment_key = '#'
system_log_file = '/var/log/syslog'
re_rsyslog_pid = re.compile("PID:\s+(\d+)")

#-- List of ERROR codes to be returned by AnsibleLogAnalyzer
# -- List of ERROR codes to be returned by AnsibleLogAnalyzer
err_duplicate_start_marker = -1
err_duplicate_end_marker = -2
err_no_end_marker = -3
Expand All @@ -48,6 +48,7 @@
err_end_ignore_marker = -7
err_start_ignore_marker = -8


class AnsibleLogAnalyzer:
'''
@summary: Overview of functionality
Expand Down Expand Up @@ -84,36 +85,36 @@ class AnsibleLogAnalyzer:
def init_sys_logger(self):
logger = logging.getLogger('LogAnalyzer')
logger.setLevel(logging.DEBUG)
handler = logging.handlers.SysLogHandler(address = '/dev/log')
handler = logging.handlers.SysLogHandler(address='/dev/log')
logger.addHandler(handler)
return logger
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def __init__(self, run_id, verbose, start_marker = None):
def __init__(self, run_id, verbose, start_marker=None):
self.run_id = run_id
self.verbose = verbose
self.start_marker = start_marker
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def print_diagnostic_message(self, message):
if (not self.verbose):
return

print('[LogAnalyzer][diagnostic]:%s' % message)
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def create_start_marker(self):
if (self.start_marker is None) or (len(self.start_marker) == 0):
return self.start_marker_prefix + "-" + self.run_id
else:
return self.start_marker

#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def is_filename_stdin(self, file_name):
return file_name == "-"

#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def require_marker_check(self, file_path):
'''
Expand All @@ -127,19 +128,19 @@ def require_marker_check(self, file_path):
files_to_skip = ["sairedis.rec", "bgpd.log"]
return not any([target in file_path for target in files_to_skip])

#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def create_end_marker(self):
return self.end_marker_prefix + "-" + self.run_id
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def create_start_ignore_marker(self):
return self.start_ignore_marker_prefix + "-" + self.run_id
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def create_end_ignore_marker(self):
return self.end_ignore_marker_prefix + "-" + self.run_id
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def flush_rsyslogd(self):
'''
Expand Down Expand Up @@ -208,11 +209,11 @@ def wait_for_marker(self, marker, timeout=120, polling_interval=10):
prev_syslog_file = "/var/log/syslog.1"
last_dt = os.path.getctime(syslog_file)
while wait_time <= timeout:
with open(syslog_file, 'r') as fp:
with open(syslog_file, 'r', errors='ignore') as fp:
dt = os.path.getctime(syslog_file)
if last_dt != dt:
try:
with open(prev_syslog_file, 'r') as pfp:
with open(prev_syslog_file, 'r', errors='ignore') as pfp:
pfp.seek(last_check_pos)
for l in fp:
if marker in l:
Expand Down Expand Up @@ -251,7 +252,7 @@ def place_marker(self, log_file_list, marker, wait_for_marker=False):
raise RuntimeError("cannot find marker {} in /var/log/syslog".format(marker))

return
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def error_to_regx(self, error_string):
'''
Expand All @@ -271,11 +272,11 @@ def error_to_regx(self, error_string):
original_string = error_string
#-- Escapes out of all the meta characters --#
error_string = re.escape(error_string)
#-- Replaces a white space with the white space regular expression
# -- Replaces a white space with the white space regular expression
error_string = re.sub(r"(\\\s+)+", "\\\\s+", error_string)
#-- Replaces a digit number with the digit regular expression
# -- Replaces a digit number with the digit regular expression
error_string = re.sub(r"\b\d+\b", "\\\\d+", error_string)
#-- Replaces a hex number with the hex regular expression
# -- Replaces a hex number with the hex regular expression
error_string = re.sub(r"0x[0-9a-fA-F]+", "0x[0-9a-fA-F]+", error_string)
self.print_diagnostic_message('Built error string: %s' % error_string)

Expand All @@ -284,7 +285,7 @@ def error_to_regx(self, error_string):
error_string = '|'.join(map(self.error_to_regx, error_string))

return error_string
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def create_msg_regex(self, file_lsit):
'''
Expand All @@ -310,17 +311,17 @@ def create_msg_regex(self, file_lsit):
for index, row in enumerate(csvreader):
row = [item for item in row if item != ""]
self.print_diagnostic_message('[diagnostic]:processing row:%d' % index)
self.print_diagnostic_message('row:%s'% row)
self.print_diagnostic_message('row:%s' % row)
try:
#-- Ignore Empty Lines
# -- Ignore Empty Lines
if not row:
continue
#-- Ignore commented Lines
# -- Ignore commented Lines
if row[0].startswith(comment_key):
self.print_diagnostic_message('[diagnostic]:skipping row[0]:%s' % row[0])
continue

#-- ('s' | 'r') = (Raw String | Regular Expression)
# -- ('s' | 'r') = (Raw String | Regular Expression)
is_regex = row[0]
if ('s' == row[0]):
is_regex = False
Expand All @@ -329,7 +330,7 @@ def create_msg_regex(self, file_lsit):
else:
raise Exception('file:%s, malformed line:%d. '
'must be \'s\'(string) or \'r\'(regex)'
%(filename,index))
% (filename, index))

if (is_regex):
messages_regex.extend(row[1:])
Expand All @@ -346,7 +347,7 @@ def create_msg_regex(self, file_lsit):
else:
regex = None
return regex, messages_regex
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def line_matches(self, str, match_messages_regex, ignore_messages_regex):
'''
Expand Down Expand Up @@ -378,7 +379,7 @@ def line_matches(self, str, match_messages_regex, ignore_messages_regex):
ret_code = True

return ret_code
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def line_is_expected(self, str, expect_messages_regex):
'''
Expand Down Expand Up @@ -413,11 +414,10 @@ def analyze_file(self, log_file_path, match_messages_regex, ignore_messages_rege
@return: List of strings match search criteria.
'''

self.print_diagnostic_message('analyzing file: %s' % log_file_path)

self.print_diagnostic_message('analyzing file: %s'% log_file_path)

#-- indicates whether log analyzer currently is in the log range between start
#-- and end marker. see analyze_file method.
# -- indicates whether log analyzer currently is in the log range between start
# -- and end marker. see analyze_file method.
check_marker = self.require_marker_check(log_file_path)
in_analysis_range = not check_marker
stdin_as_input = self.is_filename_stdin(log_file_path)
Expand Down Expand Up @@ -503,7 +503,7 @@ def analyze_file(self, log_file_path, match_messages_regex, ignore_messages_rege
sys.exit(err_no_end_marker)

return matching_lines, expected_lines
#---------------------------------------------------------------------
# ---------------------------------------------------------------------

def analyze_file_list(self, log_file_list, match_messages_regex, ignore_messages_regex, expect_messages_regex):
'''
Expand All @@ -528,14 +528,16 @@ def analyze_file_list(self, log_file_list, match_messages_regex, ignore_messages
for log_file in log_file_list:
if not len(log_file):
continue
match_strings, expect_strings = self.analyze_file(log_file, match_messages_regex, ignore_messages_regex, expect_messages_regex)
match_strings, expect_strings = self.analyze_file(
log_file, match_messages_regex, ignore_messages_regex, expect_messages_regex)

match_strings.reverse()
expect_strings.reverse()
res[log_file] = [ match_strings, expect_strings ]
res[log_file] = [match_strings, expect_strings]

return res
#---------------------------------------------------------------------
# ---------------------------------------------------------------------


def usage():
print('loganalyzer input parameters:')
Expand Down Expand Up @@ -565,7 +567,8 @@ def usage():
print(' in one of specified log files during the analysis. Must be present')
print(' when action == analyze.')

#---------------------------------------------------------------------
# ---------------------------------------------------------------------


def check_action(action, log_files_in, out_dir, match_files_in, ignore_files_in, expect_files_in):
'''
Expand All @@ -588,13 +591,13 @@ def check_action(action, log_files_in, out_dir, match_files_in, ignore_files_in,
print('ERROR: missing required match_files_in for analyze action')
ret_code = False


else:
ret_code = False
print('ERROR: invalid action:%s specified' % action)

return ret_code
#---------------------------------------------------------------------
# ---------------------------------------------------------------------


def check_run_id(run_id):
'''
Expand All @@ -612,7 +615,8 @@ def check_run_id(run_id):
ret_code = False

return ret_code
#---------------------------------------------------------------------
# ---------------------------------------------------------------------


def write_result_file(run_id, out_dir, analysis_result_per_file, messages_regex_e, unused_regex_messages):
'''
Expand Down Expand Up @@ -666,7 +670,8 @@ def write_result_file(run_id, out_dir, analysis_result_per_file, messages_regex_

out_file.write("\n-------------------------------------------------\n\n")
out_file.flush()
#---------------------------------------------------------------------
# ---------------------------------------------------------------------


def write_summary_file(run_id, out_dir, analysis_result_per_file, unused_regex_messages):
'''
Expand Down Expand Up @@ -703,7 +708,8 @@ def write_summary_file(run_id, out_dir, analysis_result_per_file, unused_regex_m
out_file.write("-----------------------------------\n")
out_file.flush()
out_file.close()
#---------------------------------------------------------------------
# ---------------------------------------------------------------------


def main(argv):

Expand All @@ -718,7 +724,8 @@ def main(argv):
verbose = False

try:
opts, args = getopt.getopt(argv, "a:r:s:l:o:m:i:e:vh", ["action=", "run_id=", "start_marker=", "logs=", "out_dir=", "match_files_in=", "ignore_files_in=", "expect_files_in=", "verbose", "help"])
opts, args = getopt.getopt(argv, "a:r:s:l:o:m:i:e:vh", [
"action=", "run_id=", "start_marker=", "logs=", "out_dir=", "match_files_in=", "ignore_files_in=", "expect_files_in=", "verbose", "help"])

except getopt.GetoptError:
print("Invalid option specified")
Expand Down Expand Up @@ -799,11 +806,11 @@ def main(argv):
analyzer.place_marker(log_file_list, analyzer.create_end_ignore_marker(), wait_for_marker=True)
return 0


else:
print('Unknown action:%s specified' % action)
return len(result)
#---------------------------------------------------------------------
# ---------------------------------------------------------------------


if __name__ == "__main__":
main(sys.argv[1:])