From 356fd66c33e0facddbe632fde39d0016a4c0ecc8 Mon Sep 17 00:00:00 2001 From: okanchou9 Date: Fri, 1 Dec 2017 09:29:29 +0800 Subject: [PATCH 1/4] [snmp_memory] add SNMP memory test Signed-off-by: okanchou9 --- ansible/library/snmp_facts.py | 15 ++++ ansible/roles/test/tasks/snmp/memory.py | 18 ++++ ansible/roles/test/tasks/snmp/memory.yml | 105 +++++++++++++++++++++++ 3 files changed, 138 insertions(+) create mode 100644 ansible/roles/test/tasks/snmp/memory.py create mode 100644 ansible/roles/test/tasks/snmp/memory.yml diff --git a/ansible/library/snmp_facts.py b/ansible/library/snmp_facts.py index 2956e3f4de5..9c2f0b9bbdd 100644 --- a/ansible/library/snmp_facts.py +++ b/ansible/library/snmp_facts.py @@ -144,6 +144,10 @@ def __init__(self,dotprefix=False): # From Dell Private MIB self.ChStackUnitCpuUtil5sec = dp + "1.3.6.1.4.1.6027.3.10.1.2.9.1.2.1" + # Memory Check + self.sysTotalMemery = dp + "1.3.6.1.4.1.2021.4.5.0" + self.sysTotalFreeMemery = dp + "1.3.6.1.4.1.2021.4.6.0" + def decode_hex(hexstring): @@ -226,6 +230,7 @@ def main(): authkey=dict(required=False), privkey=dict(required=False), is_dell=dict(required=False, default=False, type='bool'), + is_ingrasys=dict(required=False, default=False, type='bool'), removeplaceholder=dict(required=False)), required_together = ( ['username','level','integrity','authkey'],['privacy','privkey'],), supports_check_mode=False) @@ -290,6 +295,9 @@ def main(): cmdgen.MibVariable(p.sysContact,), cmdgen.MibVariable(p.sysName,), cmdgen.MibVariable(p.sysLocation,), + cmdgen.MibVariable(p.sysTotalMemery,), + cmdgen.MibVariable(p.sysTotalFreeMemery,), + lookupMib=False, lexicographicMode=False ) if errorIndication: @@ -310,6 +318,10 @@ def main(): results['ansible_sysname'] = current_val elif current_oid == v.sysLocation: results['ansible_syslocation'] = current_val + elif current_oid == v.sysTotalMemery: + results['ansible_sysTotalMemery'] = decode_type(module, current_oid, val) + elif current_oid == v.sysTotalFreeMemery: + results['ansible_sysTotalFreeMemery'] = decode_type(module, current_oid, val) errorIndication, errorStatus, errorIndex, varTable = cmdGen.nextCmd( snmp_auth, @@ -325,6 +337,7 @@ def main(): cmdgen.MibVariable(p.ipAdEntIfIndex,), cmdgen.MibVariable(p.ipAdEntNetMask,), cmdgen.MibVariable(p.ifAlias,), + lookupMib=False, lexicographicMode=False ) if errorIndication: @@ -389,6 +402,7 @@ def main(): cmdgen.MibVariable(p.ifHCOutOctets,), cmdgen.MibVariable(p.ifInUcastPkts,), cmdgen.MibVariable(p.ifOutUcastPkts,), + lookupMib=False, lexicographicMode=False ) if errorIndication: @@ -446,6 +460,7 @@ def main(): snmp_auth, cmdgen.UdpTransportTarget((m_args['host'], 161)), cmdgen.MibVariable(p.ChStackUnitCpuUtil5sec,), + lookupMib=False, lexicographicMode=False ) if errorIndication: diff --git a/ansible/roles/test/tasks/snmp/memory.py b/ansible/roles/test/tasks/snmp/memory.py new file mode 100644 index 00000000000..57289a54a58 --- /dev/null +++ b/ansible/roles/test/tasks/snmp/memory.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python + +import sys +import time + +if len(sys.argv) != 2: + print "usage: fillmem " + sys.exit() + +count = int(sys.argv[1]) + +megabyte = (0,) * (1024 * 1024 / 8) + +data = megabyte * count + +while True: + time.sleep(1) + diff --git a/ansible/roles/test/tasks/snmp/memory.yml b/ansible/roles/test/tasks/snmp/memory.yml new file mode 100644 index 00000000000..0cf30712921 --- /dev/null +++ b/ansible/roles/test/tasks/snmp/memory.yml @@ -0,0 +1,105 @@ +# Test SNMP Memory Utilization +# +# - Check the difference percentage of Total Memory between SNMP and shell is 0 +# - Check the difference percentage of Total Free Memory between SNMP and shell small than 3% when: +# * Check free memory before stress test +# * Check free memory in stress test +# * Check free memory after stress test +# - Check memory leak issue, the difference percentage between Total Free Memory before stress test and after should small than 3% +# +# Requires: Ansible v2.0.0.2-1 +# +# Usage: +# +# ansible-playbook test_sonic.yml -i inventory --limit switch2 --become --tags snmp_memory -vvvvvv +# + +- block: + - name: Copy Memory stress script to the switch + copy: src={{ item.src }} dest={{ item.dest }} + with_items: + - { src: 'roles/test/tasks/snmp/memory.py', dest: '/tmp/memory.py' } + + - name: Gathering basic snmp facts about the device + snmp_facts: host={{ ansible_host }} version=v2c community={{ snmp_rocommunity }} + connection: local + + - name: Check total memory via shell + shell: grep MemTotal /proc/meminfo | awk '{print $2}' + register: shell_total_memory + become: yes + + - name: 'Validating SNMP total memory matches shell "/proc/meminfo" result' + assert: + that: "{{ (ansible_sysTotalMemery|int - shell_total_memory.stdout|int)|abs }} == 0" + + - name: Gathering basic snmp facts about the device + snmp_facts: host={{ ansible_host }} version=v2c community={{ snmp_rocommunity }} + connection: local + + - name: Check total free memory via shell + shell: grep MemFree /proc/meminfo | awk '{print $2}' + register: shell_total_free_memory + become: yes + + - name: Validating SNMP total free memory matches shell result before stress test + assert: + that: "{{ '%.6f'|format((ansible_sysTotalFreeMemery|int - shell_total_free_memory.stdout|int)|abs / ansible_sysTotalFreeMemery|int) }} <= 0.03" + + - set_fact: free_memery_before_test="{{ ansible_sysTotalFreeMemery }}" + + - set_fact: test_momory="{{ ((ansible_sysTotalFreeMemery - 512000) / 1024)|int }}" + + - name: Start memory stress generation + shell: python /tmp/memory.py {{ test_momory }} & + become: yes + + - name: Wait for memory stress to reflect in SNMP + pause: seconds=20 + + - name: Gathering basic snmp facts about the device + snmp_facts: host={{ ansible_host }} version=v2c community={{ snmp_rocommunity }} + connection: local + + - name: Check total free memory via shell + shell: grep MemFree /proc/meminfo | awk '{print $2}' + register: shell_total_free_memory + become: yes + + - name: Validating SNMP total free memory matches shell result in stress test + assert: + that: "{{ '%.6f'|format((ansible_sysTotalFreeMemery|int - shell_total_free_memory.stdout|int)|abs / ansible_sysTotalFreeMemery|int) }} <= 0.03" + + - name: Stop memory stress generation + shell: killall python /tmp/memory.py + become: yes + ignore_errors: true + + - name: Wait for memory recovery to reflect in SNMP + pause: seconds=20 + + - name: Gathering basic snmp facts about the device + snmp_facts: host={{ ansible_host }} version=v2c community={{ snmp_rocommunity }} + connection: local + + - name: Check total free memory via shell + shell: grep MemFree /proc/meminfo | awk '{print $2}' + register: shell_total_free_memory + become: yes + + - name: Validating SNMP total free memory matches shell result after stress test + assert: + that: "{{ '%.6f'|format((ansible_sysTotalFreeMemery|int - shell_total_free_memory.stdout|int)|abs / ansible_sysTotalFreeMemery|int) }} <= 0.03" + + - name: Validating memory leak issue on DUT + assert: + that: "{{ '%.6f'|format((free_memery_before_test|int - ansible_sysTotalFreeMemery|int)|abs / free_memery_before_test|int) }} <= 0.03" + + always: + - name: Stop memory stress generation + shell: killall python /tmp/memory.py + become: yes + ignore_errors: true + + - name: Remove stress test file from DUT + file: path="/tmp/memory.py" state=absent From 629eac205c238eb72a9b415603fb926bb3021493 Mon Sep 17 00:00:00 2001 From: okanchou9 Date: Tue, 5 Dec 2017 14:16:12 +0800 Subject: [PATCH 2/4] [snmp_memory] remove unnecessary SNMP query Signed-off-by: okanchou9 --- ansible/roles/test/tasks/snmp/memory.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ansible/roles/test/tasks/snmp/memory.yml b/ansible/roles/test/tasks/snmp/memory.yml index 0cf30712921..2ca876c3ab7 100644 --- a/ansible/roles/test/tasks/snmp/memory.yml +++ b/ansible/roles/test/tasks/snmp/memory.yml @@ -33,10 +33,6 @@ assert: that: "{{ (ansible_sysTotalMemery|int - shell_total_memory.stdout|int)|abs }} == 0" - - name: Gathering basic snmp facts about the device - snmp_facts: host={{ ansible_host }} version=v2c community={{ snmp_rocommunity }} - connection: local - - name: Check total free memory via shell shell: grep MemFree /proc/meminfo | awk '{print $2}' register: shell_total_free_memory From 52e50812deb2b49748e7b95a967d78ed5f729f90 Mon Sep 17 00:00:00 2001 From: okanchou9 Date: Tue, 12 Dec 2017 15:10:23 +0800 Subject: [PATCH 3/4] [snmp_memory] update script with request Signed-off-by: okanchou9 --- ansible/README.test.md | 6 ++++ ansible/roles/test/tasks/snmp/memory.yml | 46 +++++++++++++++++------- ansible/roles/test/tasks/sonic.yml | 5 +++ 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/ansible/README.test.md b/ansible/README.test.md index c3ac324a06c..63dbc8571b9 100644 --- a/ansible/README.test.md +++ b/ansible/README.test.md @@ -129,3 +129,9 @@ ansible-playbook test_sonic.yml -i inventory --limit {DUT_NAME} --tags bgp_multi This test only works for T1 related topologies(t1, t1-lag, ...) You might need to redeploy your VMs before you run this test due to the change for ToR VM router configuration changes `./testbed-cli.sh config-vm your-topo-name(vms1-1) your-vm-name(VM0108)` will do this for you + +### SNMP memory test +``` +ansible-playbook test_sonic.yml -i inventory --limit {DUT_NAME}, --become --tags snmp_memory -e "tolerance=0.05" -e "min_memory_size=512000" +``` + diff --git a/ansible/roles/test/tasks/snmp/memory.yml b/ansible/roles/test/tasks/snmp/memory.yml index 2ca876c3ab7..81d97f44b4d 100644 --- a/ansible/roles/test/tasks/snmp/memory.yml +++ b/ansible/roles/test/tasks/snmp/memory.yml @@ -1,20 +1,35 @@ +########################################################################## # Test SNMP Memory Utilization # # - Check the difference percentage of Total Memory between SNMP and shell is 0 -# - Check the difference percentage of Total Free Memory between SNMP and shell small than 3% when: +# - Check the difference percentage of Total Free Memory between SNMP and shell small than 5% when: # * Check free memory before stress test # * Check free memory in stress test # * Check free memory after stress test -# - Check memory leak issue, the difference percentage between Total Free Memory before stress test and after should small than 3% +# - Tester can modify the value of tolerance and min_memory_size for different platforms by Ansible extra variable +# * Default value of tolerance is 0.05(5%) +# * Default value of min_memory_size is 512000(512M) # # Requires: Ansible v2.0.0.2-1 # # Usage: # -# ansible-playbook test_sonic.yml -i inventory --limit switch2 --become --tags snmp_memory -vvvvvv +# ansible-playbook test_sonic.yml -i inventory --limit switch2 --become --tags snmp_memory -e "tolerance=0.05" -e "min_memory_size=512000" # +# Author: +# +# Kenie Liu(okanchou9) mailto:kenie7@gmail.com +########################################################################## - block: + - name: Check tolerance is set or not. Default is 0.05(5%) + set_fact: tolerance=0.05 + when: tolerance is not defined + + - name: Check min_memory_size is set or not. Default is 512000(512M) + set_fact: min_memory_size=512000 + when: min_memory_size is not defined + - name: Copy Memory stress script to the switch copy: src={{ item.src }} dest={{ item.dest }} with_items: @@ -40,56 +55,63 @@ - name: Validating SNMP total free memory matches shell result before stress test assert: - that: "{{ '%.6f'|format((ansible_sysTotalFreeMemery|int - shell_total_free_memory.stdout|int)|abs / ansible_sysTotalFreeMemery|int) }} <= 0.03" + that: "{{ '%.6f'|format((ansible_sysTotalFreeMemery|int - shell_total_free_memory.stdout|int)|abs / ansible_sysTotalFreeMemery|int) }} <= {{ tolerance|float }}" - - set_fact: free_memery_before_test="{{ ansible_sysTotalFreeMemery }}" + - set_fact: test_momory="{{ ((ansible_sysTotalFreeMemery - min_memory_size|int) / 1024)|int }}" - - set_fact: test_momory="{{ ((ansible_sysTotalFreeMemery - 512000) / 1024)|int }}" + - debug: msg="Memory stress test will skip since system total free momory({{ ansible_sysTotalFreeMemery }}) is less than min_memory_size({{ min_memory_size }})" + when: test_momory|int <= 0 - name: Start memory stress generation shell: python /tmp/memory.py {{ test_momory }} & become: yes + when: test_momory|int > 0 - name: Wait for memory stress to reflect in SNMP pause: seconds=20 + when: test_momory|int > 0 - name: Gathering basic snmp facts about the device snmp_facts: host={{ ansible_host }} version=v2c community={{ snmp_rocommunity }} connection: local + when: test_momory|int > 0 - name: Check total free memory via shell shell: grep MemFree /proc/meminfo | awk '{print $2}' register: shell_total_free_memory become: yes + when: test_momory|int > 0 - name: Validating SNMP total free memory matches shell result in stress test assert: - that: "{{ '%.6f'|format((ansible_sysTotalFreeMemery|int - shell_total_free_memory.stdout|int)|abs / ansible_sysTotalFreeMemery|int) }} <= 0.03" + that: "{{ '%.6f'|format((ansible_sysTotalFreeMemery|int - shell_total_free_memory.stdout|int)|abs / ansible_sysTotalFreeMemery|int) }} <= {{ tolerance|float }}" + when: test_momory|int > 0 - name: Stop memory stress generation shell: killall python /tmp/memory.py become: yes ignore_errors: true + when: test_momory|int > 0 - name: Wait for memory recovery to reflect in SNMP pause: seconds=20 + when: test_momory|int > 0 - name: Gathering basic snmp facts about the device snmp_facts: host={{ ansible_host }} version=v2c community={{ snmp_rocommunity }} connection: local + when: test_momory|int > 0 - name: Check total free memory via shell shell: grep MemFree /proc/meminfo | awk '{print $2}' register: shell_total_free_memory become: yes + when: test_momory|int > 0 - name: Validating SNMP total free memory matches shell result after stress test assert: - that: "{{ '%.6f'|format((ansible_sysTotalFreeMemery|int - shell_total_free_memory.stdout|int)|abs / ansible_sysTotalFreeMemery|int) }} <= 0.03" - - - name: Validating memory leak issue on DUT - assert: - that: "{{ '%.6f'|format((free_memery_before_test|int - ansible_sysTotalFreeMemery|int)|abs / free_memery_before_test|int) }} <= 0.03" + that: "{{ '%.6f'|format((ansible_sysTotalFreeMemery|int - shell_total_free_memory.stdout|int)|abs / ansible_sysTotalFreeMemery|int) }} <= {{ tolerance|float }}" + when: test_momory|int > 0 always: - name: Stop memory stress generation diff --git a/ansible/roles/test/tasks/sonic.yml b/ansible/roles/test/tasks/sonic.yml index d254ed6dbe2..959898e6f68 100644 --- a/ansible/roles/test/tasks/sonic.yml +++ b/ansible/roles/test/tasks/sonic.yml @@ -169,3 +169,8 @@ - name: neighbor mac change test without using ptf include: neighbour-mac-noptf.yml tags: neighbour_mac_noptf + +- name: Test SNMP Memory + include: snmp/memory.yml + tags: snmp_memory + From 035beb5a541cfe616d02afdd754a51ed17577238 Mon Sep 17 00:00:00 2001 From: Guohan Lu Date: Fri, 20 Mar 2020 09:30:05 +0000 Subject: [PATCH 4/4] remove text Signed-off-by: Guohan Lu --- ansible/README.test.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/ansible/README.test.md b/ansible/README.test.md index c3a494ad777..b05dc486c81 100644 --- a/ansible/README.test.md +++ b/ansible/README.test.md @@ -214,9 +214,6 @@ ansible-playbook test_sonic.yml -i {INVENTORY} --limit {DUT_NAME} -e testcase_na ``` ansible-playbook test_sonic.yml -i {INVENTORY} --limit {DUT_NAME} -e testcase_name=neighbor_mac_noptf -e testbed_name={TESTBED_NAME} ``` -This test only works for T1 related topologies(t1, t1-lag, ...) -You might need to redeploy your VMs before you run this test due to the change for ToR VM router configuration changes -`./testbed-cli.sh config-vm your-topo-name(vms1-1) your-vm-name(VM0108)` will do this for you - Requires switch connected to a VM set or PTF testbed ### SNMP memory test