[pcieutil] Add 'pcie-aer' sub-command to display AER stats (sonic-net#1169)

- What I did
Add new "pcie-aer" sub-command in pcieutil to display the AER stats.
"pcieutil pcie-aer" has four sub-commands - 'all', 'correctable', 'fatal' and 'non-fatal'.

'all' command displays the AER stats for all severities.
'correctable', 'fatal' and 'non-fatal' commands display the AER stats of respective severity.
'device', 'no-zero' options for pcie-aer sub commands

```
root@sonic:/home/admin# pcieutil pcie-aer
Usage: pcieutil pcie-aer [OPTIONS] COMMAND [ARGS]...

  Display PCIe AER status

Options:
  --help  Show this message and exit.

Commands:
  all          Show all PCIe AER attributes
  correctable  Show PCIe AER correctable attributes
  fatal        Show PCIe AER fatal attributes
  non-fatal    Show PCIe AER non-fatal attributes
root@sonic:/home/admin#
root@sonic:/home/admin# pcieutil pcie-aer all --help
Usage: pcieutil pcie-aer all [OPTIONS]

  Show all PCIe AER attributes

Options:
  -d, --device <BUS>:<DEV>.<FN>  Display stats only for the specified device
  -nz, --no-zero                 Display non-zero AER stats
  --help                         Show this message and exit.
root@sonic:/home/admin#
```
Depends on: sonic-net/sonic-platform-daemons#100

- How I did it
Add new functions in pcieutil, to implement sub-commands for retrieving AER stats from STATE_DB and output it in tabular format.

Author: ArunSaravananBalachandran

pcieutil/main.py

@@ -7,10 +7,15 @@
 
 try:
     import os
+    import re
     import sys
+    from collections import OrderedDict
 
     import click
     from sonic_py_common import device_info, logger
+    from swsssdk import SonicV2Connector
+    from tabulate import tabulate
+    import utilities_common.cli as clicommon
 except ImportError as e:
     raise ImportError("%s - required module not found" % str(e))
 
@@ -105,6 +110,168 @@ def show():
         click.echo("bus:dev.fn %s:%s.%s - dev_id=0x%s, %s" % (Bus, Dev, Fn, Id, Name))
 
 
+# PCIe AER stats helpers
+
+aer_fields = {
+    "correctable": ['RxErr', 'BadTLP', 'BadDLLP', 'Rollover', 'Timeout', 'NonFatalErr', 'CorrIntErr', 'HeaderOF', 'TOTAL_ERR_COR'],
+    "fatal": ['Undefined', 'DLP', 'SDES', 'TLP', 'FCP', 'CmpltTO', 'CmpltAbrt', 'UnxCmplt', 'RxOF', 'MalfTLP', 'ECRC', 'UnsupReq',
+              'ACSViol', 'UncorrIntErr', 'BlockedTLP', 'AtomicOpBlocked', 'TLPBlockedErr', 'TOTAL_ERR_FATAL'],
+    "non_fatal": ['Undefined', 'DLP', 'SDES', 'TLP', 'FCP', 'CmpltTO', 'CmpltAbrt', 'UnxCmplt', 'RxOF', 'MalfTLP', 'ECRC', 'UnsupReq',
+                  'ACSViol', 'UncorrIntErr', 'BlockedTLP', 'AtomicOpBlocked', 'TLPBlockedErr', 'TOTAL_ERR_NONFATAL']
+}
+
+
+class PcieDevice(click.ParamType):
+    name = "<Bus>:<Dev>.<Fn>"
+
+    def convert(self, value, param, ctx):
+        match = re.match(r'([0-9A-Fa-f]{1,2}):([0-9A-Fa-f]{1,2})\.([0-9A-Fa-f])', value)
+
+        if not match:
+            self.fail('{} is not in <Bus>:<Dev>.<Fn> format'.format(value), param, ctx)
+
+        Bus, Dev, Fn = [int(val, 16) for val in match.groups()]
+        if Bus > 255:
+            self.fail('Invalid Bus number', param, ctx)
+
+        if Dev > 31:
+            self.fail('Invalid Dev number', param, ctx)
+
+        if Fn > 7:
+            self.fail('Invalid Fn number', param, ctx)
+
+        return "%02x:%02x.%d" % (Bus, Dev, Fn)
+
+
+_pcie_aer_click_options = [
+    click.Option(['-d', '--device', 'device_key'],
+                 type=PcieDevice(),
+                 help="Display stats only for the specified device"),
+    click.Option(['-v', '--verbose'],
+                 is_flag=True,
+                 help="Display all stats")
+]
+
+
+class PcieAerCommand(click.Command):
+    '''This subclass of click.Command provides common options, help
+    and short help text for PCIe AER commands'''
+
+    def __init__(self, *args, **kwargs):
+        super(PcieAerCommand, self).__init__(*args, **kwargs)
+        self.params = _pcie_aer_click_options
+
+    def format_help_text(self, ctx, formatter):
+        formatter.write_paragraph()
+        with formatter.indentation():
+            formatter.write_text("Show {} PCIe AER attributes".format(self.name.replace("_", "-")))
+            formatter.write_text("(Default: Display only non-zero attributes)")
+
+    def get_short_help_str(self, limit):
+        return "Show {} PCIe AER attributes".format(self.name.replace("_", "-"))
+
+
+def pcie_aer_display(ctx, severity):
+    device_key = ctx.params['device_key']
+    no_zero = not ctx.params['verbose']
+    header = ["AER - " + severity.upper().replace("_", "")]
+    fields = aer_fields[severity]
+    pcie_dev_list = list()
+    dev_found = False
+
+    statedb = SonicV2Connector()
+    statedb.connect(statedb.STATE_DB)
+
+    table = OrderedDict()
+    for field in fields:
+        table[field] = [field]
+
+    if device_key:
+        pcie_dev_list = ["PCIE_DEVICE|%s" % device_key]
+    else:
+        keys = statedb.keys(statedb.STATE_DB, "PCIE_DEVICE|*")
+        if keys:
+            pcie_dev_list = sorted(keys)
+
+    for pcie_dev_key in pcie_dev_list:
+        aer_attribute = statedb.get_all(statedb.STATE_DB, pcie_dev_key)
+        if not aer_attribute:
+            continue
+
+        if device_key:
+            dev_found = True
+
+        if no_zero and all(val == '0' for key, val in aer_attribute.items() if key.startswith(severity)):
+            continue
+
+        pcie_dev = pcie_dev_key.split("|")[1]
+        Id = aer_attribute['id']
+
+        # Tabulate Header
+        device_name = "%s\n%s" % (pcie_dev, Id)
+        header.append(device_name)
+
+        # Tabulate Row
+        for field in fields:
+            key = severity + "|" + field
+            table[field].append(aer_attribute.get(key, 'NA'))
+
+    if device_key and not dev_found:
+        ctx.exit("Device not found in DB")
+
+    # Strip fields with no non-zero value
+    if no_zero:
+        for field in fields:
+            if all(val == '0' for val in table[field][1:]):
+                del table[field]
+
+    if not (no_zero and (len(header) == 1)):
+        if ctx.obj:
+            click.echo("")
+
+        click.echo(tabulate(list(table.values()), header, tablefmt="grid"))
+        ctx.obj = True
+
+
+# Show PCIe AER status
+@cli.group(cls=clicommon.AliasedGroup)
+@click.pass_context
+def pcie_aer(ctx):
+    '''Display PCIe AER status'''
+    # Set True to insert a line between severities in 'all' context
+    ctx.obj = False
+
+
+@pcie_aer.command(cls=PcieAerCommand)
+@click.pass_context
+def correctable(ctx, device_key, verbose):
+    '''Show correctable PCIe AER attributes'''
+    pcie_aer_display(ctx, "correctable")
+
+
+@pcie_aer.command(cls=PcieAerCommand)
+@click.pass_context
+def fatal(ctx, device_key, verbose):
+    '''Show fatal PCIe AER attributes'''
+    pcie_aer_display(ctx, "fatal")
+
+
+@pcie_aer.command(cls=PcieAerCommand)
+@click.pass_context
+def non_fatal(ctx, device_key, verbose):
+    '''Show non-fatal PCIe AER attributes'''
+    pcie_aer_display(ctx, "non_fatal")
+
+
+@pcie_aer.command(name='all', cls=PcieAerCommand)
+@click.pass_context
+def all_errors(ctx, device_key, verbose):
+    '''Show all PCIe AER attributes'''
+    pcie_aer_display(ctx, "correctable")
+    pcie_aer_display(ctx, "fatal")
+    pcie_aer_display(ctx, "non_fatal")
+
+
 #  Show PCIE Vender ID and Device ID
 @cli.command()
 def check():

tests/mock_tables/state_db.json

@@ -455,5 +455,103 @@
     "CHASSIS_MIDPLANE_TABLE|LINE-CARD1": {
         "ip_address": "192.168.1.2",
         "access": "False"
+    },
+    "PCIE_DEVICE|00:01.0": {
+        "correctable|BadDLLP": "0",
+        "correctable|BadTLP": "0",
+        "correctable|BadTLP": "1",
+        "correctable|CorrIntErr": "0",
+        "correctable|HeaderOF": "0",
+        "correctable|NonFatalErr": "0",
+        "correctable|Rollover": "0",
+        "correctable|RxErr": "0",
+        "correctable|TOTAL_ERR_COR": "0",
+        "correctable|TOTAL_ERR_COR": "1",
+        "correctable|Timeout": "0",
+        "fatal|ACSViol": "0",
+        "fatal|AtomicOpBlocked": "0",
+        "fatal|BlockedTLP": "0",
+        "fatal|CmpltAbrt": "0",
+        "fatal|CmpltTO": "0",
+        "fatal|DLP": "0",
+        "fatal|ECRC": "0",
+        "fatal|FCP": "0",
+        "fatal|MalfTLP": "0",
+        "fatal|RxOF": "0",
+        "fatal|SDES": "0",
+        "fatal|TLP": "0",
+        "fatal|TLPBlockedErr": "0",
+        "fatal|TOTAL_ERR_FATAL": "0",
+        "fatal|UncorrIntErr": "0",
+        "fatal|Undefined": "0",
+        "fatal|UnsupReq": "0",
+        "fatal|UnxCmplt": "0",
+        "id": "0x0001",
+        "non_fatal|ACSViol": "0",
+        "non_fatal|AtomicOpBlocked": "0",
+        "non_fatal|BlockedTLP": "0",
+        "non_fatal|CmpltAbrt": "0",
+        "non_fatal|CmpltTO": "0",
+        "non_fatal|DLP": "0",
+        "non_fatal|ECRC": "0",
+        "non_fatal|FCP": "0",
+        "non_fatal|MalfTLP": "1",
+        "non_fatal|RxOF": "0",
+        "non_fatal|SDES": "0",
+        "non_fatal|TLP": "0",
+        "non_fatal|TLPBlockedErr": "0",
+        "non_fatal|TOTAL_ERR_NONFATAL": "1",
+        "non_fatal|UncorrIntErr": "0",
+        "non_fatal|Undefined": "0",
+        "non_fatal|UnsupReq": "0",
+        "non_fatal|UnxCmplt": "0"
+    },
+    "PCIE_DEVICE|01:00.0": {
+        "correctable|BadDLLP": "0",
+        "correctable|BadTLP": "0",
+        "correctable|CorrIntErr": "0",
+        "correctable|HeaderOF": "0",
+        "correctable|NonFatalErr": "0",
+        "correctable|Rollover": "0",
+        "correctable|RxErr": "1",
+        "correctable|TOTAL_ERR_COR": "1",
+        "correctable|Timeout": "0",
+        "fatal|ACSViol": "0",
+        "fatal|AtomicOpBlocked": "0",
+        "fatal|BlockedTLP": "0",
+        "fatal|CmpltAbrt": "0",
+        "fatal|CmpltTO": "0",
+        "fatal|DLP": "0",
+        "fatal|ECRC": "0",
+        "fatal|FCP": "0",
+        "fatal|MalfTLP": "0",
+        "fatal|RxOF": "0",
+        "fatal|SDES": "0",
+        "fatal|TLP": "0",
+        "fatal|TLPBlockedErr": "0",
+        "fatal|TOTAL_ERR_FATAL": "0",
+        "fatal|UncorrIntErr": "0",
+        "fatal|Undefined": "0",
+        "fatal|UnsupReq": "0",
+        "fatal|UnxCmplt": "0",
+        "id": "0x0002",
+        "non_fatal|ACSViol": "0",
+        "non_fatal|AtomicOpBlocked": "0",
+        "non_fatal|BlockedTLP": "0",
+        "non_fatal|CmpltAbrt": "0",
+        "non_fatal|CmpltTO": "0",
+        "non_fatal|DLP": "0",
+        "non_fatal|ECRC": "0",
+        "non_fatal|FCP": "0",
+        "non_fatal|MalfTLP": "0",
+        "non_fatal|RxOF": "0",
+        "non_fatal|SDES": "0",
+        "non_fatal|TLP": "0",
+        "non_fatal|TLPBlockedErr": "0",
+        "non_fatal|TOTAL_ERR_NONFATAL": "0",
+        "non_fatal|UncorrIntErr": "0",
+        "non_fatal|Undefined": "0",
+        "non_fatal|UnsupReq": "0",
+        "non_fatal|UnxCmplt": "0"
     }
 }