Revert "Fix loganalyzer.py UnicodeDecodeError (sonic-net#6524)" (sonic-net#6577)

This reverts commit 94ac4a0.

Author: ZhaohuiS

ansible/library/extract_log.py

@@ -92,9 +92,9 @@ def extract_lines(directory, filename, target_string):
     path = os.path.join(directory, filename)
     file = None
     if 'gz' in path:
-        file = gzip.open(path, mode='rt', errors='ignore')
+        file = gzip.open(path, mode='rt')
     else:
-        file = open(path, errors='ignore')
+        file = open(path)
     result = None
     with file:
         # This might be a gunzip file or logrotate issue, there has
@@ -226,17 +226,17 @@ def combine_logs_and_save(directory, filenames, start_string, target_string, tar
     do_copy = False
     line_processed = 0
     line_copied = 0
-    with open(target_filename, 'w', errors='ignore') as fp:
+    with open(target_filename, 'w') as fp:
         for filename in reversed(filenames):
             path = os.path.join(directory, filename)
             dt = datetime.datetime.fromtimestamp(os.path.getctime(path))
             sz = os.path.getsize(path)
             logger.debug("extract_log combine_logs from file {} create time {}, size {}".format(path, dt, sz))
             file = None
             if 'gz' in path:
-                file = gzip.open(path, mode='rt', errors='ignore')
+                file = gzip.open(path, mode='rt')
             else:
-                file = open(path, errors='ignore')
+                file = open(path)
 
             with file:
                 for line in file:

ansible/roles/test/files/tools/loganalyzer/loganalyzer.py

@@ -13,9 +13,9 @@
                 sudo python loganalyzer.py  --out_dir /home/hrachya/projects/loganalyzer/log.analyzer.results --action analyze --run_id myTest114 --logs file3.log -m /home/hrachya/projects/loganalyzer/match.file.1.log,/home/hrachya/projects/loganalyzer/match.file.2.log  -i ignore.file.1.log,ignore.file.2.log -v
 '''
 
-# ---------------------------------------------------------------------
+#---------------------------------------------------------------------
 # Global imports
-# ---------------------------------------------------------------------
+#---------------------------------------------------------------------
 from __future__ import print_function
 import sys
 import getopt
@@ -30,15 +30,15 @@
 import subprocess
 from datetime import datetime
 
-# ---------------------------------------------------------------------
+#---------------------------------------------------------------------
 # Global variables
-# ---------------------------------------------------------------------
+#---------------------------------------------------------------------
 tokenizer = ','
 comment_key = '#'
 system_log_file = '/var/log/syslog'
 re_rsyslog_pid = re.compile("PID:\s+(\d+)")
 
-# -- List of ERROR codes to be returned by AnsibleLogAnalyzer
+#-- List of ERROR codes to be returned by AnsibleLogAnalyzer
 err_duplicate_start_marker = -1
 err_duplicate_end_marker = -2
 err_no_end_marker = -3
@@ -48,7 +48,6 @@
 err_end_ignore_marker = -7
 err_start_ignore_marker = -8
 
-
 class AnsibleLogAnalyzer:
     '''
     @summary: Overview of functionality
@@ -85,36 +84,36 @@ class AnsibleLogAnalyzer:
     def init_sys_logger(self):
         logger = logging.getLogger('LogAnalyzer')
         logger.setLevel(logging.DEBUG)
-        handler = logging.handlers.SysLogHandler(address='/dev/log')
+        handler = logging.handlers.SysLogHandler(address = '/dev/log')
         logger.addHandler(handler)
         return logger
-    # ---------------------------------------------------------------------
+    #---------------------------------------------------------------------
 
-    def __init__(self, run_id, verbose, start_marker=None):
+    def __init__(self, run_id, verbose, start_marker = None):
         self.run_id = run_id
         self.verbose = verbose
         self.start_marker = start_marker
-    # ---------------------------------------------------------------------
+    #---------------------------------------------------------------------
 
     def print_diagnostic_message(self, message):
         if (not self.verbose):
             return
 
         print('[LogAnalyzer][diagnostic]:%s' % message)
-    # ---------------------------------------------------------------------
+    #---------------------------------------------------------------------
 
     def create_start_marker(self):
         if (self.start_marker is None) or (len(self.start_marker) == 0):
             return self.start_marker_prefix + "-" + self.run_id
         else:
             return self.start_marker
 
-    # ---------------------------------------------------------------------
+    #---------------------------------------------------------------------
 
     def is_filename_stdin(self, file_name):
         return file_name == "-"
 
-    # ---------------------------------------------------------------------
+    #---------------------------------------------------------------------
 
     def require_marker_check(self, file_path):
         '''
@@ -128,19 +127,19 @@ def require_marker_check(self, file_path):
         files_to_skip = ["sairedis.rec", "bgpd.log"]
         return not any([target in file_path for target in files_to_skip])
 
-    # ---------------------------------------------------------------------
+    #---------------------------------------------------------------------
 
     def create_end_marker(self):
         return self.end_marker_prefix + "-" + self.run_id
-    # ---------------------------------------------------------------------
+    #---------------------------------------------------------------------
 
     def create_start_ignore_marker(self):
         return self.start_ignore_marker_prefix + "-" + self.run_id
-    # ---------------------------------------------------------------------
+    #---------------------------------------------------------------------
 
     def create_end_ignore_marker(self):
         return self.end_ignore_marker_prefix + "-" + self.run_id
-    # ---------------------------------------------------------------------
+    #---------------------------------------------------------------------
 
     def flush_rsyslogd(self):
         '''
@@ -209,11 +208,11 @@ def wait_for_marker(self, marker, timeout=120, polling_interval=10):
         prev_syslog_file = "/var/log/syslog.1"
         last_dt = os.path.getctime(syslog_file)
         while wait_time <= timeout:
-            with open(syslog_file, 'r', errors='ignore') as fp:
+            with open(syslog_file, 'r') as fp:
                 dt = os.path.getctime(syslog_file)
                 if last_dt != dt:
                     try:
-                        with open(prev_syslog_file, 'r', errors='ignore') as pfp:
+                        with open(prev_syslog_file, 'r') as pfp:
                             pfp.seek(last_check_pos)
                             for l in fp:
                                 if marker in l:
@@ -252,7 +251,7 @@ def place_marker(self, log_file_list, marker, wait_for_marker=False):
                 raise RuntimeError("cannot find marker {} in /var/log/syslog".format(marker))
 
         return
-    # ---------------------------------------------------------------------
+    #---------------------------------------------------------------------
 
     def error_to_regx(self, error_string):
         '''
@@ -272,11 +271,11 @@ def error_to_regx(self, error_string):
             original_string = error_string
             #-- Escapes out of all the meta characters --#
             error_string = re.escape(error_string)
-            # -- Replaces a white space with the white space regular expression
+            #-- Replaces a white space with the white space regular expression
             error_string = re.sub(r"(\\\s+)+", "\\\\s+", error_string)
-            # -- Replaces a digit number with the digit regular expression
+            #-- Replaces a digit number with the digit regular expression
             error_string = re.sub(r"\b\d+\b", "\\\\d+", error_string)
-            # -- Replaces a hex number with the hex regular expression
+            #-- Replaces a hex number with the hex regular expression
             error_string = re.sub(r"0x[0-9a-fA-F]+", "0x[0-9a-fA-F]+", error_string)
             self.print_diagnostic_message('Built error string: %s' % error_string)
 
@@ -285,7 +284,7 @@ def error_to_regx(self, error_string):
             error_string = '|'.join(map(self.error_to_regx, error_string))
 
         return error_string
-    # ---------------------------------------------------------------------
+    #---------------------------------------------------------------------
 
     def create_msg_regex(self, file_lsit):
         '''
@@ -311,17 +310,17 @@ def create_msg_regex(self, file_lsit):
                 for index, row in enumerate(csvreader):
                     row = [item for item in row if item != ""]
                     self.print_diagnostic_message('[diagnostic]:processing row:%d' % index)
-                    self.print_diagnostic_message('row:%s' % row)
+                    self.print_diagnostic_message('row:%s'% row)
                     try:
-                        # -- Ignore Empty Lines
+                        #-- Ignore Empty Lines
                         if not row:
                             continue
-                        # -- Ignore commented Lines
+                        #-- Ignore commented Lines
                         if row[0].startswith(comment_key):
                             self.print_diagnostic_message('[diagnostic]:skipping row[0]:%s' % row[0])
                             continue
 
-                        # -- ('s' | 'r') = (Raw String | Regular Expression)
+                        #-- ('s' | 'r') = (Raw String | Regular Expression)
                         is_regex = row[0]
                         if ('s' == row[0]):
                             is_regex = False
@@ -330,7 +329,7 @@ def create_msg_regex(self, file_lsit):
                         else:
                             raise Exception('file:%s, malformed line:%d. '
                                             'must be \'s\'(string) or \'r\'(regex)'
-                                            % (filename, index))
+                                            %(filename,index))
 
                         if (is_regex):
                             messages_regex.extend(row[1:])
@@ -347,7 +346,7 @@ def create_msg_regex(self, file_lsit):
         else:
             regex = None
         return regex, messages_regex
-    # ---------------------------------------------------------------------
+    #---------------------------------------------------------------------
 
     def line_matches(self, str, match_messages_regex, ignore_messages_regex):
         '''
@@ -379,7 +378,7 @@ def line_matches(self, str, match_messages_regex, ignore_messages_regex):
                 ret_code = True
 
         return ret_code
-    # ---------------------------------------------------------------------
+    #---------------------------------------------------------------------
 
     def line_is_expected(self, str, expect_messages_regex):
         '''
@@ -414,10 +413,11 @@ def analyze_file(self, log_file_path, match_messages_regex, ignore_messages_rege
         @return: List of strings match search criteria.
         '''
 
-        self.print_diagnostic_message('analyzing file: %s' % log_file_path)
 
-        # -- indicates whether log analyzer currently is in the log range between start
-        # -- and end marker. see analyze_file method.
+        self.print_diagnostic_message('analyzing file: %s'% log_file_path)
+
+        #-- indicates whether log analyzer currently is in the log range between start
+        #-- and end marker. see analyze_file method.
         check_marker = self.require_marker_check(log_file_path)
         in_analysis_range = not check_marker
         stdin_as_input = self.is_filename_stdin(log_file_path)
@@ -503,7 +503,7 @@ def analyze_file(self, log_file_path, match_messages_regex, ignore_messages_rege
                 sys.exit(err_no_end_marker)
 
         return matching_lines, expected_lines
-    # ---------------------------------------------------------------------
+    #---------------------------------------------------------------------
 
     def analyze_file_list(self, log_file_list, match_messages_regex, ignore_messages_regex, expect_messages_regex):
         '''
@@ -528,16 +528,14 @@ def analyze_file_list(self, log_file_list, match_messages_regex, ignore_messages
         for log_file in log_file_list:
             if not len(log_file):
                 continue
-            match_strings, expect_strings = self.analyze_file(
-                log_file, match_messages_regex, ignore_messages_regex, expect_messages_regex)
+            match_strings, expect_strings = self.analyze_file(log_file, match_messages_regex, ignore_messages_regex, expect_messages_regex)
 
             match_strings.reverse()
             expect_strings.reverse()
-            res[log_file] = [match_strings, expect_strings]
+            res[log_file] = [ match_strings, expect_strings ]
 
         return res
-    # ---------------------------------------------------------------------
-
+    #---------------------------------------------------------------------
 
 def usage():
     print('loganalyzer input parameters:')
@@ -567,8 +565,7 @@ def usage():
     print('                                 in one of specified log files during the analysis. Must be present')
     print('                                 when action == analyze.')
 
-# ---------------------------------------------------------------------
-
+#---------------------------------------------------------------------
 
 def check_action(action, log_files_in, out_dir, match_files_in, ignore_files_in, expect_files_in):
     '''
@@ -591,13 +588,13 @@ def check_action(action, log_files_in, out_dir, match_files_in, ignore_files_in,
             print('ERROR: missing required match_files_in for analyze action')
             ret_code = False
 
+
     else:
         ret_code = False
         print('ERROR: invalid action:%s specified' % action)
 
     return ret_code
-# ---------------------------------------------------------------------
-
+#---------------------------------------------------------------------
 
 def check_run_id(run_id):
     '''
@@ -615,8 +612,7 @@ def check_run_id(run_id):
         ret_code = False
 
     return ret_code
-# ---------------------------------------------------------------------
-
+#---------------------------------------------------------------------
 
 def write_result_file(run_id, out_dir, analysis_result_per_file, messages_regex_e, unused_regex_messages):
     '''
@@ -670,8 +666,7 @@ def write_result_file(run_id, out_dir, analysis_result_per_file, messages_regex_
 
         out_file.write("\n-------------------------------------------------\n\n")
         out_file.flush()
-# ---------------------------------------------------------------------
-
+#---------------------------------------------------------------------
 
 def write_summary_file(run_id, out_dir, analysis_result_per_file, unused_regex_messages):
     '''
@@ -708,8 +703,7 @@ def write_summary_file(run_id, out_dir, analysis_result_per_file, unused_regex_m
     out_file.write("-----------------------------------\n")
     out_file.flush()
     out_file.close()
-# ---------------------------------------------------------------------
-
+#---------------------------------------------------------------------
 
 def main(argv):
 
@@ -724,8 +718,7 @@ def main(argv):
     verbose = False
 
     try:
-        opts, args = getopt.getopt(argv, "a:r:s:l:o:m:i:e:vh", [
-                                   "action=", "run_id=", "start_marker=", "logs=", "out_dir=", "match_files_in=", "ignore_files_in=", "expect_files_in=", "verbose", "help"])
+        opts, args = getopt.getopt(argv, "a:r:s:l:o:m:i:e:vh", ["action=", "run_id=", "start_marker=", "logs=", "out_dir=", "match_files_in=", "ignore_files_in=", "expect_files_in=", "verbose", "help"])
 
     except getopt.GetoptError:
         print("Invalid option specified")
@@ -806,11 +799,11 @@ def main(argv):
         analyzer.place_marker(log_file_list, analyzer.create_end_ignore_marker(), wait_for_marker=True)
         return 0
 
+
     else:
         print('Unknown action:%s specified' % action)
     return len(result)
-# ---------------------------------------------------------------------
-
+#---------------------------------------------------------------------
 
 if __name__ == "__main__":
     main(sys.argv[1:])