From 74a00457d5040177f0ec6923143d334fceed0355 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Sun, 13 Feb 2022 16:36:24 +0200 Subject: [PATCH 1/4] [auto-ts] add memory check Signed-off-by: Stepan Blyschak --- files/build_templates/init_cfg.json.j2 | 5 ++- files/image_config/monit/conf.d/sonic-host | 3 ++ .../tests/auto_techsupport.json | 7 ++++ .../tests_config/auto_techsupport.json | 36 +++++++++++++++++++ .../yang-models/sonic-auto_techsupport.yang | 18 ++++++++++ 5 files changed, 68 insertions(+), 1 deletion(-) diff --git a/files/build_templates/init_cfg.json.j2 b/files/build_templates/init_cfg.json.j2 index e1320214e02..d50ce825105 100644 --- a/files/build_templates/init_cfg.json.j2 +++ b/files/build_templates/init_cfg.json.j2 @@ -72,6 +72,8 @@ "rate_limit_interval" : "180", "max_techsupport_limit" : "10.0", "max_core_limit" : "5.0", + "available_mem_threshold": "10.0", + "min_available_mem": "204800", "since" : "2 days ago" } }, @@ -81,7 +83,8 @@ {%- if enable_auto_tech_support == "y" %} "state" : "enabled", {% else %} "state" : "disabled", {% endif %} - "rate_limit_interval" : "600" + "rate_limit_interval" : "600", + "available_mem_threshold": "10.0" }{%if not loop.last %},{% endif -%} {% endfor %} } diff --git a/files/image_config/monit/conf.d/sonic-host b/files/image_config/monit/conf.d/sonic-host index ceebf1003eb..8c7e1db2e52 100644 --- a/files/image_config/monit/conf.d/sonic-host +++ b/files/image_config/monit/conf.d/sonic-host @@ -46,3 +46,6 @@ check program vnetRouteCheck with path "/usr/local/bin/vnet_route_check.py" every 5 cycles if status != 0 for 3 cycle then alert repeat every 1 cycles +# memory_check tool that verifies that memory usage does not cross the threshold or invokes techsupport. +check program memory_check with path "/usr/local/bin/memory_check.py" + if status == 2 for 10 times within 20 cycles then exec "/usr/local/bin/memory_check_handler.py" diff --git a/src/sonic-yang-models/tests/yang_model_tests/tests/auto_techsupport.json b/src/sonic-yang-models/tests/yang_model_tests/tests/auto_techsupport.json index 8a65fec2b6d..239f51205cb 100644 --- a/src/sonic-yang-models/tests/yang_model_tests/tests/auto_techsupport.json +++ b/src/sonic-yang-models/tests/yang_model_tests/tests/auto_techsupport.json @@ -23,5 +23,12 @@ }, "AUTO_TECHSUPPORT_RATE_LIMIT_INTERVAL_TEST": { "desc" : "Configure and test the valid configuration" + }, + "AUTO_TECHSUPPORT_AVAILABLE_MEM_THRESHOLD": { + "desc" : "Configure and test the valid configuration" + }, + "AUTO_TECHSUPPORT_INVALID_AVAILABLE_MEM_THRESHOLD": { + "desc" : "Configure a value for available_mem_threshold inside the range [0, 100) but with 3 fractional digits", + "eStrKey": "InvalidValue" } } diff --git a/src/sonic-yang-models/tests/yang_model_tests/tests_config/auto_techsupport.json b/src/sonic-yang-models/tests/yang_model_tests/tests_config/auto_techsupport.json index 43ac4ce8239..f93a2cdb172 100644 --- a/src/sonic-yang-models/tests/yang_model_tests/tests_config/auto_techsupport.json +++ b/src/sonic-yang-models/tests/yang_model_tests/tests_config/auto_techsupport.json @@ -81,5 +81,41 @@ ] } } + }, + "AUTO_TECHSUPPORT_AVAILABLE_MEM_THRESHOLD": { + "sonic-auto_techsupport:sonic-auto_techsupport": { + "sonic-auto_techsupport:AUTO_TECHSUPPORT_FEATURE": { + "AUTO_TECHSUPPORT_FEATURE_LIST": [ + { + "feature_name" : "bgp", + "state" : "enabled", + "available_mem_threshold": "10.0" + }, + { + "feature_name" : "swss", + "state" : "disabled", + "available_mem_threshold": "10.0" + } + ] + } + } + }, + "AUTO_TECHSUPPORT_INVALID_AVAILABLE_MEM_THRESHOLD": { + "sonic-auto_techsupport:sonic-auto_techsupport": { + "sonic-auto_techsupport:AUTO_TECHSUPPORT_FEATURE": { + "AUTO_TECHSUPPORT_FEATURE_LIST": [ + { + "feature_name" : "bgp", + "state" : "enabled", + "available_mem_threshold": "11.111" + }, + { + "feature_name" : "swss", + "state" : "disabled", + "available_mem_threshold": "10.0" + } + ] + } + } } } diff --git a/src/sonic-yang-models/yang-models/sonic-auto_techsupport.yang b/src/sonic-yang-models/yang-models/sonic-auto_techsupport.yang index 94934d7ab20..c4b8f48cef1 100644 --- a/src/sonic-yang-models/yang-models/sonic-auto_techsupport.yang +++ b/src/sonic-yang-models/yang-models/sonic-auto_techsupport.yang @@ -59,6 +59,18 @@ module sonic-auto_techsupport { description "Max Limit in percentage for the cummulative size of core dumps. No cleanup is performed if the value isn't congiured or is 0.0"; type decimal-repr; } + + leaf available_mem_threshold { + description "Memory threshold; 0 to disable techsupport invocation on memory usage threshold crossing"; + type decimal-repr; + default 10.0; + } + + leaf min_available_mem { + description "Minimum free memory amount in Kb when techsupport will be executed"; + type uint32; + default 204800; + } leaf since { /* @@ -96,6 +108,12 @@ module sonic-auto_techsupport { type stypes:admin_mode; } + leaf available_mem_threshold { + description "Memory threshold; 0 to disable techsupport invocation on memory usage threshold crossing"; + type decimal-repr; + default 10.0; + } + leaf rate_limit_interval { description "Rate limit interval for the corresponding feature. Configure 0 to explicitly disable"; type uint16; From 68bc3ceb4f691414ebfd6b3ad3dbf4913cf8bd21 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 18 Feb 2022 13:07:20 +0200 Subject: [PATCH 2/4] fix internal review comments Signed-off-by: Stepan Blyschak --- files/image_config/monit/conf.d/sonic-host | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/files/image_config/monit/conf.d/sonic-host b/files/image_config/monit/conf.d/sonic-host index 8c7e1db2e52..d6532520754 100644 --- a/files/image_config/monit/conf.d/sonic-host +++ b/files/image_config/monit/conf.d/sonic-host @@ -47,5 +47,5 @@ check program vnetRouteCheck with path "/usr/local/bin/vnet_route_check.py" if status != 0 for 3 cycle then alert repeat every 1 cycles # memory_check tool that verifies that memory usage does not cross the threshold or invokes techsupport. -check program memory_check with path "/usr/local/bin/memory_check.py" - if status == 2 for 10 times within 20 cycles then exec "/usr/local/bin/memory_check_handler.py" +check program memory_check with path "/usr/local/bin/memory_threshold_check.py" + if status == 2 for 10 times within 20 cycles then exec "/usr/local/bin/memory_threshold_check_handler.py" From 6ffc9ea14422ca6cc9c22effe5c82aef7bb6c37b Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 25 Mar 2022 02:18:05 +0200 Subject: [PATCH 3/4] review commetns Signed-off-by: Stepan Blyschak --- files/build_templates/init_cfg.json.j2 | 2 +- files/image_config/monit/conf.d/sonic-host | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/files/build_templates/init_cfg.json.j2 b/files/build_templates/init_cfg.json.j2 index d50ce825105..d1738c33950 100644 --- a/files/build_templates/init_cfg.json.j2 +++ b/files/build_templates/init_cfg.json.j2 @@ -73,7 +73,7 @@ "max_techsupport_limit" : "10.0", "max_core_limit" : "5.0", "available_mem_threshold": "10.0", - "min_available_mem": "204800", + "min_available_mem": "200", "since" : "2 days ago" } }, diff --git a/files/image_config/monit/conf.d/sonic-host b/files/image_config/monit/conf.d/sonic-host index d6532520754..54815fc1c4e 100644 --- a/files/image_config/monit/conf.d/sonic-host +++ b/files/image_config/monit/conf.d/sonic-host @@ -48,4 +48,4 @@ check program vnetRouteCheck with path "/usr/local/bin/vnet_route_check.py" # memory_check tool that verifies that memory usage does not cross the threshold or invokes techsupport. check program memory_check with path "/usr/local/bin/memory_threshold_check.py" - if status == 2 for 10 times within 20 cycles then exec "/usr/local/bin/memory_threshold_check_handler.py" + if status == 2 then exec "/usr/local/bin/memory_threshold_check_handler.py" From 78a59643ed5a462a83d0b69b42ab96e6691b8e0e Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 25 Mar 2022 02:19:00 +0200 Subject: [PATCH 4/4] review commetns Signed-off-by: Stepan Blyschak --- files/image_config/monit/conf.d/sonic-host | 2 +- src/sonic-yang-models/yang-models/sonic-auto_techsupport.yang | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/files/image_config/monit/conf.d/sonic-host b/files/image_config/monit/conf.d/sonic-host index 54815fc1c4e..d6532520754 100644 --- a/files/image_config/monit/conf.d/sonic-host +++ b/files/image_config/monit/conf.d/sonic-host @@ -48,4 +48,4 @@ check program vnetRouteCheck with path "/usr/local/bin/vnet_route_check.py" # memory_check tool that verifies that memory usage does not cross the threshold or invokes techsupport. check program memory_check with path "/usr/local/bin/memory_threshold_check.py" - if status == 2 then exec "/usr/local/bin/memory_threshold_check_handler.py" + if status == 2 for 10 times within 20 cycles then exec "/usr/local/bin/memory_threshold_check_handler.py" diff --git a/src/sonic-yang-models/yang-models/sonic-auto_techsupport.yang b/src/sonic-yang-models/yang-models/sonic-auto_techsupport.yang index c4b8f48cef1..02e29463d59 100644 --- a/src/sonic-yang-models/yang-models/sonic-auto_techsupport.yang +++ b/src/sonic-yang-models/yang-models/sonic-auto_techsupport.yang @@ -67,9 +67,9 @@ module sonic-auto_techsupport { } leaf min_available_mem { - description "Minimum free memory amount in Kb when techsupport will be executed"; + description "Minimum Free memory (in MB) that should be available for the techsupport execution to start"; type uint32; - default 204800; + default 200; } leaf since {