diff --git a/ansible/library/extract_log.py b/ansible/library/extract_log.py index 982216aea5d..aef14a6c0cb 100644 --- a/ansible/library/extract_log.py +++ b/ansible/library/extract_log.py @@ -92,9 +92,9 @@ def extract_lines(directory, filename, target_string): path = os.path.join(directory, filename) file = None if 'gz' in path: - file = gzip.open(path, mode='rt') + file = gzip.open(path, mode='rt', errors='ignore') else: - file = open(path) + file = open(path, errors='ignore') result = None with file: # This might be a gunzip file or logrotate issue, there has @@ -226,7 +226,7 @@ def combine_logs_and_save(directory, filenames, start_string, target_string, tar do_copy = False line_processed = 0 line_copied = 0 - with open(target_filename, 'w') as fp: + with open(target_filename, 'w', errors='ignore') as fp: for filename in reversed(filenames): path = os.path.join(directory, filename) dt = datetime.datetime.fromtimestamp(os.path.getctime(path)) @@ -234,9 +234,9 @@ def combine_logs_and_save(directory, filenames, start_string, target_string, tar logger.debug("extract_log combine_logs from file {} create time {}, size {}".format(path, dt, sz)) file = None if 'gz' in path: - file = gzip.open(path, mode='rt') + file = gzip.open(path, mode='rt', errors='ignore') else: - file = open(path) + file = open(path, errors='ignore') with file: for line in file: diff --git a/ansible/roles/test/files/tools/loganalyzer/loganalyzer.py b/ansible/roles/test/files/tools/loganalyzer/loganalyzer.py index 2e04ef8fc1a..b4a85afedaf 100644 --- a/ansible/roles/test/files/tools/loganalyzer/loganalyzer.py +++ b/ansible/roles/test/files/tools/loganalyzer/loganalyzer.py @@ -13,9 +13,9 @@ sudo python loganalyzer.py --out_dir /home/hrachya/projects/loganalyzer/log.analyzer.results --action analyze --run_id myTest114 --logs file3.log -m /home/hrachya/projects/loganalyzer/match.file.1.log,/home/hrachya/projects/loganalyzer/match.file.2.log -i ignore.file.1.log,ignore.file.2.log -v ''' -#--------------------------------------------------------------------- +# --------------------------------------------------------------------- # Global imports -#--------------------------------------------------------------------- +# --------------------------------------------------------------------- from __future__ import print_function import sys import getopt @@ -30,15 +30,15 @@ import subprocess from datetime import datetime -#--------------------------------------------------------------------- +# --------------------------------------------------------------------- # Global variables -#--------------------------------------------------------------------- +# --------------------------------------------------------------------- tokenizer = ',' comment_key = '#' system_log_file = '/var/log/syslog' re_rsyslog_pid = re.compile("PID:\s+(\d+)") -#-- List of ERROR codes to be returned by AnsibleLogAnalyzer +# -- List of ERROR codes to be returned by AnsibleLogAnalyzer err_duplicate_start_marker = -1 err_duplicate_end_marker = -2 err_no_end_marker = -3 @@ -48,6 +48,7 @@ err_end_ignore_marker = -7 err_start_ignore_marker = -8 + class AnsibleLogAnalyzer: ''' @summary: Overview of functionality @@ -84,23 +85,23 @@ class AnsibleLogAnalyzer: def init_sys_logger(self): logger = logging.getLogger('LogAnalyzer') logger.setLevel(logging.DEBUG) - handler = logging.handlers.SysLogHandler(address = '/dev/log') + handler = logging.handlers.SysLogHandler(address='/dev/log') logger.addHandler(handler) return logger - #--------------------------------------------------------------------- + # --------------------------------------------------------------------- - def __init__(self, run_id, verbose, start_marker = None): + def __init__(self, run_id, verbose, start_marker=None): self.run_id = run_id self.verbose = verbose self.start_marker = start_marker - #--------------------------------------------------------------------- + # --------------------------------------------------------------------- def print_diagnostic_message(self, message): if (not self.verbose): return print('[LogAnalyzer][diagnostic]:%s' % message) - #--------------------------------------------------------------------- + # --------------------------------------------------------------------- def create_start_marker(self): if (self.start_marker is None) or (len(self.start_marker) == 0): @@ -108,12 +109,12 @@ def create_start_marker(self): else: return self.start_marker - #--------------------------------------------------------------------- + # --------------------------------------------------------------------- def is_filename_stdin(self, file_name): return file_name == "-" - #--------------------------------------------------------------------- + # --------------------------------------------------------------------- def require_marker_check(self, file_path): ''' @@ -127,19 +128,19 @@ def require_marker_check(self, file_path): files_to_skip = ["sairedis.rec", "bgpd.log"] return not any([target in file_path for target in files_to_skip]) - #--------------------------------------------------------------------- + # --------------------------------------------------------------------- def create_end_marker(self): return self.end_marker_prefix + "-" + self.run_id - #--------------------------------------------------------------------- + # --------------------------------------------------------------------- def create_start_ignore_marker(self): return self.start_ignore_marker_prefix + "-" + self.run_id - #--------------------------------------------------------------------- + # --------------------------------------------------------------------- def create_end_ignore_marker(self): return self.end_ignore_marker_prefix + "-" + self.run_id - #--------------------------------------------------------------------- + # --------------------------------------------------------------------- def flush_rsyslogd(self): ''' @@ -208,11 +209,11 @@ def wait_for_marker(self, marker, timeout=120, polling_interval=10): prev_syslog_file = "/var/log/syslog.1" last_dt = os.path.getctime(syslog_file) while wait_time <= timeout: - with open(syslog_file, 'r') as fp: + with open(syslog_file, 'r', errors='ignore') as fp: dt = os.path.getctime(syslog_file) if last_dt != dt: try: - with open(prev_syslog_file, 'r') as pfp: + with open(prev_syslog_file, 'r', errors='ignore') as pfp: pfp.seek(last_check_pos) for l in fp: if marker in l: @@ -251,7 +252,7 @@ def place_marker(self, log_file_list, marker, wait_for_marker=False): raise RuntimeError("cannot find marker {} in /var/log/syslog".format(marker)) return - #--------------------------------------------------------------------- + # --------------------------------------------------------------------- def error_to_regx(self, error_string): ''' @@ -271,11 +272,11 @@ def error_to_regx(self, error_string): original_string = error_string #-- Escapes out of all the meta characters --# error_string = re.escape(error_string) - #-- Replaces a white space with the white space regular expression + # -- Replaces a white space with the white space regular expression error_string = re.sub(r"(\\\s+)+", "\\\\s+", error_string) - #-- Replaces a digit number with the digit regular expression + # -- Replaces a digit number with the digit regular expression error_string = re.sub(r"\b\d+\b", "\\\\d+", error_string) - #-- Replaces a hex number with the hex regular expression + # -- Replaces a hex number with the hex regular expression error_string = re.sub(r"0x[0-9a-fA-F]+", "0x[0-9a-fA-F]+", error_string) self.print_diagnostic_message('Built error string: %s' % error_string) @@ -284,7 +285,7 @@ def error_to_regx(self, error_string): error_string = '|'.join(map(self.error_to_regx, error_string)) return error_string - #--------------------------------------------------------------------- + # --------------------------------------------------------------------- def create_msg_regex(self, file_lsit): ''' @@ -310,17 +311,17 @@ def create_msg_regex(self, file_lsit): for index, row in enumerate(csvreader): row = [item for item in row if item != ""] self.print_diagnostic_message('[diagnostic]:processing row:%d' % index) - self.print_diagnostic_message('row:%s'% row) + self.print_diagnostic_message('row:%s' % row) try: - #-- Ignore Empty Lines + # -- Ignore Empty Lines if not row: continue - #-- Ignore commented Lines + # -- Ignore commented Lines if row[0].startswith(comment_key): self.print_diagnostic_message('[diagnostic]:skipping row[0]:%s' % row[0]) continue - #-- ('s' | 'r') = (Raw String | Regular Expression) + # -- ('s' | 'r') = (Raw String | Regular Expression) is_regex = row[0] if ('s' == row[0]): is_regex = False @@ -329,7 +330,7 @@ def create_msg_regex(self, file_lsit): else: raise Exception('file:%s, malformed line:%d. ' 'must be \'s\'(string) or \'r\'(regex)' - %(filename,index)) + % (filename, index)) if (is_regex): messages_regex.extend(row[1:]) @@ -346,7 +347,7 @@ def create_msg_regex(self, file_lsit): else: regex = None return regex, messages_regex - #--------------------------------------------------------------------- + # --------------------------------------------------------------------- def line_matches(self, str, match_messages_regex, ignore_messages_regex): ''' @@ -378,7 +379,7 @@ def line_matches(self, str, match_messages_regex, ignore_messages_regex): ret_code = True return ret_code - #--------------------------------------------------------------------- + # --------------------------------------------------------------------- def line_is_expected(self, str, expect_messages_regex): ''' @@ -413,11 +414,10 @@ def analyze_file(self, log_file_path, match_messages_regex, ignore_messages_rege @return: List of strings match search criteria. ''' + self.print_diagnostic_message('analyzing file: %s' % log_file_path) - self.print_diagnostic_message('analyzing file: %s'% log_file_path) - - #-- indicates whether log analyzer currently is in the log range between start - #-- and end marker. see analyze_file method. + # -- indicates whether log analyzer currently is in the log range between start + # -- and end marker. see analyze_file method. check_marker = self.require_marker_check(log_file_path) in_analysis_range = not check_marker stdin_as_input = self.is_filename_stdin(log_file_path) @@ -503,7 +503,7 @@ def analyze_file(self, log_file_path, match_messages_regex, ignore_messages_rege sys.exit(err_no_end_marker) return matching_lines, expected_lines - #--------------------------------------------------------------------- + # --------------------------------------------------------------------- def analyze_file_list(self, log_file_list, match_messages_regex, ignore_messages_regex, expect_messages_regex): ''' @@ -528,14 +528,16 @@ def analyze_file_list(self, log_file_list, match_messages_regex, ignore_messages for log_file in log_file_list: if not len(log_file): continue - match_strings, expect_strings = self.analyze_file(log_file, match_messages_regex, ignore_messages_regex, expect_messages_regex) + match_strings, expect_strings = self.analyze_file( + log_file, match_messages_regex, ignore_messages_regex, expect_messages_regex) match_strings.reverse() expect_strings.reverse() - res[log_file] = [ match_strings, expect_strings ] + res[log_file] = [match_strings, expect_strings] return res - #--------------------------------------------------------------------- + # --------------------------------------------------------------------- + def usage(): print('loganalyzer input parameters:') @@ -565,7 +567,8 @@ def usage(): print(' in one of specified log files during the analysis. Must be present') print(' when action == analyze.') -#--------------------------------------------------------------------- +# --------------------------------------------------------------------- + def check_action(action, log_files_in, out_dir, match_files_in, ignore_files_in, expect_files_in): ''' @@ -588,13 +591,13 @@ def check_action(action, log_files_in, out_dir, match_files_in, ignore_files_in, print('ERROR: missing required match_files_in for analyze action') ret_code = False - else: ret_code = False print('ERROR: invalid action:%s specified' % action) return ret_code -#--------------------------------------------------------------------- +# --------------------------------------------------------------------- + def check_run_id(run_id): ''' @@ -612,7 +615,8 @@ def check_run_id(run_id): ret_code = False return ret_code -#--------------------------------------------------------------------- +# --------------------------------------------------------------------- + def write_result_file(run_id, out_dir, analysis_result_per_file, messages_regex_e, unused_regex_messages): ''' @@ -666,7 +670,8 @@ def write_result_file(run_id, out_dir, analysis_result_per_file, messages_regex_ out_file.write("\n-------------------------------------------------\n\n") out_file.flush() -#--------------------------------------------------------------------- +# --------------------------------------------------------------------- + def write_summary_file(run_id, out_dir, analysis_result_per_file, unused_regex_messages): ''' @@ -703,7 +708,8 @@ def write_summary_file(run_id, out_dir, analysis_result_per_file, unused_regex_m out_file.write("-----------------------------------\n") out_file.flush() out_file.close() -#--------------------------------------------------------------------- +# --------------------------------------------------------------------- + def main(argv): @@ -718,7 +724,8 @@ def main(argv): verbose = False try: - opts, args = getopt.getopt(argv, "a:r:s:l:o:m:i:e:vh", ["action=", "run_id=", "start_marker=", "logs=", "out_dir=", "match_files_in=", "ignore_files_in=", "expect_files_in=", "verbose", "help"]) + opts, args = getopt.getopt(argv, "a:r:s:l:o:m:i:e:vh", [ + "action=", "run_id=", "start_marker=", "logs=", "out_dir=", "match_files_in=", "ignore_files_in=", "expect_files_in=", "verbose", "help"]) except getopt.GetoptError: print("Invalid option specified") @@ -799,11 +806,11 @@ def main(argv): analyzer.place_marker(log_file_list, analyzer.create_end_ignore_marker(), wait_for_marker=True) return 0 - else: print('Unknown action:%s specified' % action) return len(result) -#--------------------------------------------------------------------- +# --------------------------------------------------------------------- + if __name__ == "__main__": main(sys.argv[1:])