Skip to content

Commit 06ada15

Browse files
stephenxsmssonicbld
stephenxs
authored and
mssonicbld
committed
[Mellanox] Support DSCP remapping in dual ToR topo on T0 switch (sonic-net#12605)
- Why I did it Support DSCP remapping in dual ToR topo on T0 switch for SKU Mellanox-SN4600c-C64, Mellanox-SN4600c-D48C40, Mellanox-SN2700, Mellanox-SN2700-D48C8. - How I did it Regarding buffer settings, originally, there are two lossless PGs and queues 3, 4. In dual ToR scenario, the lossless traffic from the leaf switch to the uplink of the ToR switch can be bounced back. To avoid PFC deadlock, we need to map the bounce-back lossless traffic to different PGs and queues. Therefore, 2 additional lossless PGs and queues are allocated on uplink ports on ToR switches. On uplink ports, map DSCP 2/6 to TC 2/6 respectively On downlink ports, both DSCP 2/6 are still mapped to TC 1 Buffer adjusted according to the ports information: Mellanox-SN4600c-C64: 56 downlinks 50G + 8 uplinks 100G Mellanox-SN4600c-D48C40, Mellanox-SN2700, Mellanox-SN2700-D48C8: 24 downlinks 50G + 8 uplinks 100G - How to verify it Unit test. Signed-off-by: Stephen Sun <stephens@nvidia.com>
1 parent e6fde1d commit 06ada15

18 files changed

Lines changed: 16919 additions & 344 deletions

‎device/mellanox/x86_64-mlnx_msn2700-r0/Mellanox-SN2700-D48C8/buffers_defaults_t0.j2‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{#
2-
Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES.
2+
Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES.
33
Apache-2.0
44
55
Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,10 +15,17 @@
1515
limitations under the License.
1616
#}
1717
{% set default_cable = '5m' %}
18-
{% set ingress_lossless_pool_size = '7719936' %}
19-
{% set ingress_lossless_pool_xoff = '1032192' %}
18+
{%- if ((SYSTEM_DEFAULTS is defined) and ('tunnel_qos_remap' in SYSTEM_DEFAULTS) and (SYSTEM_DEFAULTS['tunnel_qos_remap']['status'] == 'enabled')) -%}
19+
{% set ingress_lossless_pool_size = '9047040' %}
20+
{% set ingress_lossless_pool_xoff = '851968' %}
2021
{% set egress_lossless_pool_size = '13945824' %}
21-
{% set egress_lossy_pool_size = '7719936' %}
22+
{% set egress_lossy_pool_size = '9047040' %}
23+
{%- else -%}
24+
{% set ingress_lossless_pool_size = '9595904' %}
25+
{% set ingress_lossless_pool_xoff = '614400' %}
26+
{% set egress_lossless_pool_size = '13945824' %}
27+
{% set egress_lossy_pool_size = '9595904' %}
28+
{%- endif -%}
2229

2330
{% import 'buffers_defaults_objects.j2' as defs with context %}
2431

@@ -30,10 +37,18 @@
3037
{{ defs.generate_profile_lists(port_names_active, port_names_inactive) }}
3138
{%- endmacro %}
3239

40+
{%- macro generate_queue_buffers_with_extra_lossless_queues_with_inactive_ports(port_names_active, port_names_extra_queues, port_names_inactive) %}
41+
{{ defs.generate_queue_buffers_with_extra_lossless_queues(port_names_active, port_names_extra_queues, port_names_inactive) }}
42+
{%- endmacro %}
43+
3344
{%- macro generate_queue_buffers_with_inactive_ports(port_names_active, port_names_inactive) %}
3445
{{ defs.generate_queue_buffers(port_names_active, port_names_inactive) }}
3546
{%- endmacro %}
3647

48+
{%- macro generate_pg_profiles_with_extra_lossless_pgs_with_inactive_ports(port_names_active, port_names_extra_pgs, port_names_inactive) %}
49+
{{ defs.generate_pg_profiles_with_extra_lossless_pgs(port_names_active, port_names_extra_pgs, port_names_inactive) }}
50+
{%- endmacro %}
51+
3752
{%- macro generate_pg_profiles_with_inactive_ports(port_names_active, port_names_inactive) %}
3853
{{ defs.generate_pg_profiles(port_names_active, port_names_inactive) }}
3954
{%- endmacro %}

‎device/mellanox/x86_64-mlnx_msn2700-r0/Mellanox-SN2700-D48C8/qos.json.j2‎

Lines changed: 0 additions & 17 deletions
This file was deleted.

‎device/mellanox/x86_64-mlnx_msn2700-r0/Mellanox-SN2700-D48C8/qos.json.j2‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../x86_64-mlnx_msn4600c-r0/Mellanox-SN4600C-C64/qos.json.j2

‎device/mellanox/x86_64-mlnx_msn2700-r0/Mellanox-SN2700/buffers_defaults_t0.j2‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{#
2-
Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES.
2+
Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES.
33
Apache-2.0
44
55
Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,10 +15,17 @@
1515
limitations under the License.
1616
#}
1717
{% set default_cable = '5m' %}
18-
{% set ingress_lossless_pool_size = '10177536' %}
19-
{% set ingress_lossless_pool_xoff = '688128' %}
18+
{%- if ((SYSTEM_DEFAULTS is defined) and ('tunnel_qos_remap' in SYSTEM_DEFAULTS) and (SYSTEM_DEFAULTS['tunnel_qos_remap']['status'] == 'enabled')) -%}
19+
{% set ingress_lossless_pool_size = '9047040' %}
20+
{% set ingress_lossless_pool_xoff = '851968' %}
2021
{% set egress_lossless_pool_size = '13945824' %}
21-
{% set egress_lossy_pool_size = '10177536' %}
22+
{% set egress_lossy_pool_size = '9047040' %}
23+
{%- else -%}
24+
{% set ingress_lossless_pool_size = '9595904' %}
25+
{% set ingress_lossless_pool_xoff = '614400' %}
26+
{% set egress_lossless_pool_size = '13945824' %}
27+
{% set egress_lossy_pool_size = '9595904' %}
28+
{%- endif -%}
2229

2330
{% import 'buffers_defaults_objects.j2' as defs with context %}
2431

@@ -30,10 +37,18 @@
3037
{{ defs.generate_profile_lists(port_names_active, port_names_inactive) }}
3138
{%- endmacro %}
3239

40+
{%- macro generate_queue_buffers_with_extra_lossless_queues_with_inactive_ports(port_names_active, port_names_extra_queues, port_names_inactive) %}
41+
{{ defs.generate_queue_buffers_with_extra_lossless_queues(port_names_active, port_names_extra_queues, port_names_inactive) }}
42+
{%- endmacro %}
43+
3344
{%- macro generate_queue_buffers_with_inactive_ports(port_names_active, port_names_inactive) %}
3445
{{ defs.generate_queue_buffers(port_names_active, port_names_inactive) }}
3546
{%- endmacro %}
3647

48+
{%- macro generate_pg_profiles_with_extra_lossless_pgs_with_inactive_ports(port_names_active, port_names_extra_pgs, port_names_inactive) %}
49+
{{ defs.generate_pg_profiles_with_extra_lossless_pgs(port_names_active, port_names_extra_pgs, port_names_inactive) }}
50+
{%- endmacro %}
51+
3752
{%- macro generate_pg_profiles_with_inactive_ports(port_names_active, port_names_inactive) %}
3853
{{ defs.generate_pg_profiles(port_names_active, port_names_inactive) }}
3954
{%- endmacro %}

‎device/mellanox/x86_64-mlnx_msn2700-r0/Mellanox-SN2700/qos.json.j2‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
../ACS-MSN2700/qos.json.j2
1+
../../x86_64-mlnx_msn4600c-r0/Mellanox-SN4600C-C64/qos.json.j2

‎device/mellanox/x86_64-mlnx_msn4600c-r0/Mellanox-SN4600C-C64/buffers_defaults_t0.j2‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{#
2-
Copyright (c) 2018-2021 NVIDIA CORPORATION & AFFILIATES.
2+
Copyright (c) 2018-2023 NVIDIA CORPORATION & AFFILIATES.
33
Apache-2.0
44
55
Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,10 +15,17 @@
1515
limitations under the License.
1616
#}
1717
{% set default_cable = '5m' %}
18-
{% set ingress_lossless_pool_size = '49905664' %}
19-
{% set ingress_lossless_pool_xoff = '3702784' %}
18+
{%- if ((SYSTEM_DEFAULTS is defined) and ('tunnel_qos_remap' in SYSTEM_DEFAULTS) and (SYSTEM_DEFAULTS['tunnel_qos_remap']['status'] == 'enabled')) -%}
19+
{% set ingress_lossless_pool_size = '50667520' %}
20+
{% set ingress_lossless_pool_xoff = '2629632' %}
2021
{% set egress_lossless_pool_size = '60817392' %}
21-
{% set egress_lossy_pool_size = '49905664' %}
22+
{% set egress_lossy_pool_size = '50667520' %}
23+
{%- else -%}
24+
{% set ingress_lossless_pool_size = '51511296' %}
25+
{% set ingress_lossless_pool_xoff = '2097152' %}
26+
{% set egress_lossless_pool_size = '60817392' %}
27+
{% set egress_lossy_pool_size = '51511296' %}
28+
{%- endif -%}
2229

2330
{% import 'buffers_defaults_objects.j2' as defs with context %}
2431

@@ -30,10 +37,18 @@
3037
{{ defs.generate_profile_lists(port_names_active, port_names_inactive) }}
3138
{%- endmacro %}
3239

40+
{%- macro generate_queue_buffers_with_extra_lossless_queues_with_inactive_ports(port_names_active, port_names_extra_queues, port_names_inactive) %}
41+
{{ defs.generate_queue_buffers_with_extra_lossless_queues(port_names_active, port_names_extra_queues, port_names_inactive) }}
42+
{%- endmacro %}
43+
3344
{%- macro generate_queue_buffers_with_inactive_ports(port_names_active, port_names_inactive) %}
3445
{{ defs.generate_queue_buffers(port_names_active, port_names_inactive) }}
3546
{%- endmacro %}
3647

48+
{%- macro generate_pg_profiles_with_extra_lossless_pgs_with_inactive_ports(port_names_active, port_names_extra_pgs, port_names_inactive) %}
49+
{{ defs.generate_pg_profiles_with_extra_lossless_pgs(port_names_active, port_names_extra_pgs, port_names_inactive) }}
50+
{%- endmacro %}
51+
3752
{%- macro generate_pg_profiles_with_inactive_ports(port_names_active, port_names_inactive) %}
3853
{{ defs.generate_pg_profiles(port_names_active, port_names_inactive) }}
3954
{%- endmacro %}

‎device/mellanox/x86_64-mlnx_msn4600c-r0/Mellanox-SN4600C-C64/qos.json.j2‎

Lines changed: 118 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{#
2-
Copyright (c) 2020-2021 NVIDIA CORPORATION & AFFILIATES.
2+
Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES.
33
Apache-2.0
44
55
Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,18 +14,20 @@
1414
See the License for the specific language governing permissions and
1515
limitations under the License.
1616
#}
17-
{% if ('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'LeafRouter') %}
17+
18+
{% if (('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'LeafRouter') or
19+
('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'ToRRouter') and
20+
('subtype' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['subtype'] == 'DualToR')) %}
1821
{% set different_dscp_to_tc_map = true %}
19-
{%- macro generate_dscp_to_tc_map() %}
20-
"DSCP_TO_TC_MAP": {
21-
"AZURE": {
22+
{%- macro generate_normal_dscp_to_tc_map(tc_mapped_from_dscp_33) -%}
23+
{
2224
"0" : "1",
2325
"1" : "1",
24-
"2" : "2",
26+
"2" : "1",
2527
"3" : "3",
2628
"4" : "4",
2729
"5" : "1",
28-
"6" : "6",
30+
"6" : "1",
2931
"7" : "1",
3032
"8" : "0",
3133
"9" : "1",
@@ -52,7 +54,7 @@
5254
"30": "1",
5355
"31": "1",
5456
"32": "1",
55-
"33": "1",
57+
"33": "{{tc_mapped_from_dscp_33}}",
5658
"34": "1",
5759
"35": "1",
5860
"36": "1",
@@ -83,15 +85,17 @@
8385
"61": "1",
8486
"62": "1",
8587
"63": "1"
86-
},
87-
"AZURE_UPLINK": {
88+
}
89+
{%- endmacro -%}
90+
{%- macro generate_dscp_to_tc_map_with_addition_lossless_pgs_n_queues(tc_mapped_from_dscp_33) -%}
91+
{
8892
"0" : "1",
8993
"1" : "1",
90-
"2" : "1",
94+
"2" : "2",
9195
"3" : "3",
9296
"4" : "4",
9397
"5" : "1",
94-
"6" : "1",
98+
"6" : "6",
9599
"7" : "1",
96100
"8" : "0",
97101
"9" : "1",
@@ -118,7 +122,7 @@
118122
"30": "1",
119123
"31": "1",
120124
"32": "1",
121-
"33": "1",
125+
"33": "{{tc_mapped_from_dscp_33}}",
122126
"34": "1",
123127
"35": "1",
124128
"36": "1",
@@ -150,10 +154,52 @@
150154
"62": "1",
151155
"63": "1"
152156
}
157+
{%- endmacro -%}
158+
{%- macro generate_dscp_to_tc_map() %}
159+
"DSCP_TO_TC_MAP": {
160+
{% if ('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'LeafRouter') %}
161+
"AZURE_UPLINK":
162+
{{ generate_normal_dscp_to_tc_map("1") }},
163+
"AZURE":
164+
{{ generate_dscp_to_tc_map_with_addition_lossless_pgs_n_queues("1") }}
165+
{% endif %}
166+
{% if ('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'ToRRouter') %}
167+
"AZURE_UPLINK":
168+
{{ generate_dscp_to_tc_map_with_addition_lossless_pgs_n_queues("8") }},
169+
"AZURE_TUNNEL":
170+
{{ generate_normal_dscp_to_tc_map("8") }},
171+
"AZURE":
172+
{{ generate_normal_dscp_to_tc_map("8") }}
173+
{% endif %}
153174
},
154175
{%- endmacro %}
155176
{%- macro generate_tc_to_pg_map() %}
156177
"TC_TO_PRIORITY_GROUP_MAP": {
178+
{% if (('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'ToRRouter') and
179+
('subtype' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['subtype'] == 'DualToR')) %}
180+
"AZURE_TUNNEL": {
181+
"0": "0",
182+
"1": "0",
183+
"2": "0",
184+
"3": "2",
185+
"4": "6",
186+
"5": "0",
187+
"6": "0",
188+
"7": "0",
189+
"8": "0"
190+
},
191+
"AZURE": {
192+
"0": "0",
193+
"1": "0",
194+
"2": "2",
195+
"3": "3",
196+
"4": "4",
197+
"5": "0",
198+
"6": "6",
199+
"7": "0",
200+
"8": "0"
201+
}
202+
{% else %}
157203
"AZURE": {
158204
"0": "0",
159205
"1": "0",
@@ -164,8 +210,67 @@
164210
"6": "6",
165211
"7": "0"
166212
}
213+
{% endif %}
214+
},
215+
{%- endmacro %}
216+
{%- macro generate_tc_to_queue_map() %}
217+
"TC_TO_QUEUE_MAP": {
218+
{% if (('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'ToRRouter') and
219+
('subtype' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['subtype'] == 'DualToR')) %}
220+
"AZURE_TUNNEL": {
221+
"0": "0",
222+
"1": "1",
223+
"2": "2",
224+
"3": "2",
225+
"4": "6",
226+
"5": "5",
227+
"6": "6",
228+
"7": "7",
229+
"8": "1"
230+
},
231+
"AZURE": {
232+
"0": "0",
233+
"1": "1",
234+
"2": "2",
235+
"3": "3",
236+
"4": "4",
237+
"5": "5",
238+
"6": "6",
239+
"7": "7",
240+
"8": "1"
241+
}
242+
{% else %}
243+
"AZURE": {
244+
"0": "0",
245+
"1": "1",
246+
"2": "2",
247+
"3": "3",
248+
"4": "4",
249+
"5": "5",
250+
"6": "6",
251+
"7": "7"
252+
}
253+
{% endif %}
254+
},
255+
{%- endmacro %}
256+
{% if (('type' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['type'] == 'ToRRouter') and
257+
('subtype' in DEVICE_METADATA['localhost'] and DEVICE_METADATA['localhost']['subtype'] == 'DualToR')) %}
258+
{%- macro generate_tc_to_dscp_map() %}
259+
"TC_TO_DSCP_MAP": {
260+
"AZURE_TUNNEL": {
261+
"0": "8",
262+
"1": "0",
263+
"2": "2",
264+
"3": "2",
265+
"4": "6",
266+
"5": "46",
267+
"6": "6",
268+
"7": "48",
269+
"8": "33"
270+
}
167271
},
168272
{%- endmacro %}
273+
{%- endif %}
169274
{%- macro generate_global_dscp_to_tc_map() %}
170275
{# This is an empty macro since the global DSCP_TO_TC map is not required #}
171276
{%- endmacro %}

0 commit comments

Comments
 (0)