From 29a5c663bb5b01babd23b0b445877f5258f229b0 Mon Sep 17 00:00:00 2001 From: marian-pritsak Date: Mon, 6 Feb 2017 13:04:42 +0200 Subject: [PATCH 1/3] [loganalyzer]: Refacor log analysis flow Make test invoking loganalysis fail if unexpected errors in logs are found Integrate sysdump call to loganalyzer Fetch resulting files to ansible machine --- .../tools/loganalyzer/loganalyzer_analyze.yml | 2 +- .../tools/loganalyzer/loganalyzer_end.yml | 53 +++++++++++++------ ansible/roles/test/tasks/fib.yml | 7 +-- ansible/roles/test/tasks/lag.yml | 7 +-- 4 files changed, 40 insertions(+), 29 deletions(-) diff --git a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml index 8cda128620d..bb2dae21996 100644 --- a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml +++ b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml @@ -8,7 +8,7 @@ - debug: msg={{cmd}} -- name: Initialize loganalyzer {{ testname }} +- name: Invoke loganalyzer analyse {{ testname }} become: True shell: "{{ cmd }}" args: diff --git a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_end.yml b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_end.yml index f5f570ac29e..5fa944b8afd 100644 --- a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_end.yml +++ b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_end.yml @@ -3,24 +3,45 @@ #------------------------------------------------ - set_fact: - result_file: result.loganalysis.{{testname_unique}}.log - summary_file: summary.loganalysis.{{testname_unique}}.log -- debug : msg="summary_file {{summary_file}}" + result_file: result.loganalysis.{{ testname_unique }}.log + summary_file: summary.loganalysis.{{ testname_unique }}.log + test_fetch_dir: test/{{ inventory_hostname }} + +# Output content of result files to ansible console +- shell: cat {{ test_out_dir }}/* + register: out +- debug: var=out.stdout_lines + +- name: Check if loganalyzer found any error + shell: grep "TOTAL MATCHES" "{{ test_out_dir }}/{{ summary_file }}" | sed -n "s/TOTAL MATCHES:[[:space:]]*//p" + register: errors_found + +- set_fact: + fail_in_logs: errors_found.stdout != "0" + +- name: Generate system dump + command: sonic_support + register: sonic_support + when: fail_in_logs + - name : Fetch result files from switch to ansible machine - fetch: src={{ test_out_dir }}/{{item}} dest=/tmp + fetch: + src: "{{ item }}" + dest: "{{ test_fetch_dir }}/{{ item | basename }}" + flat: yes with_items: - - "{{result_file}}" - - "{{summary_file}}" - -- name : Create destination directory on PTF host for loganalyzer results - file: path="{{ test_out_dir }}" state=directory - delegate_to: "{{ ptf_host }}" + - "{{ test_out_dir }}/{{result_file}}" + - "{{ test_out_dir }}/{{summary_file}}" + - "{{ sonic_support.stdout | default(omit) }}" + when: fail_in_logs -- name: Copy loganalyzer result files from ansible machine to PTF host - copy: src="/tmp/{{inventory_hostname}}/{{test_out_dir}}/{{item}}" dest="{{ test_out_dir }}/{{item}}" +- debug: msg="File {{ item }} saved to {{ test_fetch_dir }}" with_items: - - "{{ result_file }}" - - "{{ summary_file }}" - delegate_to: "{{ ptf_host }}" + - "{{result_file}}" + - "{{summary_file}}" + - "{{ sonic_support.stdout | default(omit) }}" + when: fail_in_logs -- debug: msg="Location of log files on PTF host - {{ test_out_dir }}" \ No newline at end of file +- name: Fail due to errors in logs + fail: + when: fail_in_logs diff --git a/ansible/roles/test/tasks/fib.yml b/ansible/roles/test/tasks/fib.yml index 4aba8bf8402..bd2df31ca08 100644 --- a/ansible/roles/test/tasks/fib.yml +++ b/ansible/roles/test/tasks/fib.yml @@ -79,9 +79,4 @@ always: - include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml - # Output content of result files to ansible console - - shell: cat {{ test_out_dir }}/* - register: out - - debug: var=out.stdout_lines - - - include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml \ No newline at end of file + - include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml diff --git a/ansible/roles/test/tasks/lag.yml b/ansible/roles/test/tasks/lag.yml index 64073697300..986dd53569f 100644 --- a/ansible/roles/test/tasks/lag.yml +++ b/ansible/roles/test/tasks/lag.yml @@ -74,9 +74,4 @@ always: - include: roles/test/files/tools/loganalyzer/loganalyzer_analyze.yml - # Output content of result files to ansible console - - shell: cat {{ test_out_dir }}/* - register: out - - debug: var=out.stdout_lines - - - include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml \ No newline at end of file + - include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml From 5ab4df05a19de0d395289db8f133bd4c9a373bf4 Mon Sep 17 00:00:00 2001 From: Marian Pritsak Date: Wed, 10 May 2017 18:15:33 +0300 Subject: [PATCH 2/3] Remove unnecessary include --- ansible/roles/test/tasks/fib.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/roles/test/tasks/fib.yml b/ansible/roles/test/tasks/fib.yml index d395bac546d..f14dbd604a4 100644 --- a/ansible/roles/test/tasks/fib.yml +++ b/ansible/roles/test/tasks/fib.yml @@ -44,5 +44,3 @@ - debug: var=out.stdout_lines when: out.rc != 0 - -- include: roles/test/files/tools/loganalyzer/loganalyzer_end.yml \ No newline at end of file From fbc6df3dbb5c0e4eea5a27e6b49d201db71deca3 Mon Sep 17 00:00:00 2001 From: Marian Pritsak Date: Tue, 16 May 2017 10:39:05 +0300 Subject: [PATCH 3/3] Force log rotation before initializing loganalyzer --- ansible/roles/test/files/tools/loganalyzer/loganalyzer_init.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_init.yml b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_init.yml index 586c2807b88..4ace815cefe 100644 --- a/ansible/roles/test/files/tools/loganalyzer/loganalyzer_init.yml +++ b/ansible/roles/test/files/tools/loganalyzer/loganalyzer_init.yml @@ -27,6 +27,8 @@ - name: create output directory for current test run file: path="{{ test_out_dir }}" state=directory +- name: Force log rotation to have most of log file available + command: logrotate -f /etc/logrotate.conf - debug: msg="starting loganalyzer_init.py" - debug: msg="python {{ run_dir }}/loganalyzer.py --action init --run_id {{ testname_unique }}"