[meta] Add enum ancestry check on SAI includes

.gitignore

@@ -1,6 +1,7 @@
 # metadata build artifacts
 meta/xml
 meta/html
+meta/temp
 
 meta/*.o
 meta/*.lo

meta/Makefile

@@ -81,6 +81,7 @@ SYMBOLS = $(OBJ:=.symbols)
 all: toolsversions saisanitycheck saimetadatatest saiserializetest saidepgraph.svg $(SYMBOLS)
 	./checkheaders.pl ../inc ../inc
 	./checkenumlock.sh
+	./checkancestry.sh
 	./saimetadatatest >/dev/null
 	./saiserializetest >/dev/null
 	./saisanitycheck
@@ -140,4 +141,4 @@ clean:
 	rm -f *.o *~ .*~ *.tmp .*.swp .*.swo *.bak sai*.gv sai*.svg *.o.symbols
 	rm -f saimetadata.h saimetadata.c saimetadatatest.c
 	rm -f saisanitycheck saimetadatatest saiserializetest saidepgraphgen
-	rm -rf xml html dist
+	rm -rf xml html dist temp

meta/ancestry.pl

@@ -0,0 +1,243 @@
+#!/usr/bin/perl
+#
+# Copyright (c) 2021 Microsoft Open Technologies, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+#    THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR
+#    CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT
+#    LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS
+#    FOR A PARTICULAR PURPOSE, MERCHANTABILITY OR NON-INFRINGEMENT.
+#
+#    See the Apache Version 2.0 License for specific language governing
+#    permissions and limitations under the License.
+#
+#    Microsoft would like to thank the following companies for their review and
+#    assistance with these files: Intel Corporation, Mellanox Technologies Ltd,
+#    Dell Products, L.P., Facebook, Inc., Marvell International Ltd.
+#
+# @file    ancestry.pl
+#
+# @brief   This module defines enum ancestry check for SAI headers
+#
+
+BEGIN { push @INC,'.'; }
+
+use strict;
+use warnings;
+use diagnostics;
+use sort 'stable'; # for enum initializers sort
+
+use Getopt::Std;
+use Data::Dumper;
+use utils;
+
+my %options = ();
+
+getopts("dsASl", \%options);
+
+our $optionPrintDebug        = 1 if defined $options{d};
+our $optionDisableAspell     = 1 if defined $options{A};
+our $optionUseXmlSimple      = 1 if defined $options{s};
+our $optionDisableStyleCheck = 1 if defined $options{S};
+our $optionShowLogCaller     = 1 if defined $options{l};
+
+$SIG{__DIE__} = sub
+{
+    LogError "FATAL ERROR === MUST FIX === : @_";
+    exit 1;
+};
+
+our $INCLUDE_DIR = "temp";
+our %SAI_ENUMS = ();
+our %SAI_DEFINES = ();
+our %HISTORY = ();
+our %IGNORED = ();
+
+sub ProcessSingleHeader
+{
+    my $header = shift;
+
+    my $data = ReadHeaderFile $header;
+
+    my @lines = split/\n/,$data;
+
+    my $currentEnum = "undefined";
+    my $currentEnumPrefix = "undefined";
+
+    my $ignore = "";
+
+    for my $line (@lines)
+    {
+        if ($line =~ /#define\s+(SAI_\w+)\s+(\(?".*"|$NUMBER_REGEX\)?)$/)
+        {
+            LogDebug "Defined $1 = $2";
+
+            $SAI_DEFINES{$1} = $2;
+            next;
+        }
+
+        if ($line =~ /^\s*typedef\s+enum\s+_((sai_\w+_)t)/)
+        {
+            $currentEnum = $1;
+            $currentEnumPrefix = uc($2);
+
+            my @values = ();
+            my @inits = ();
+
+            $SAI_ENUMS{$currentEnum}->{values} = \@values;
+            $SAI_ENUMS{$currentEnum}->{inits} = \@inits;
+
+            LogDebug "enum found $currentEnum";
+            next;
+        }
+
+        $ignore = "ignore" if $line =~ /\@ignore/;
+
+        if ($line =~ /^\s*(${currentEnumPrefix}\w+)(.*)$/)
+        {
+            my $enumName = $1;
+            my $init = (defined $2) ? $2 : "";
+
+            $init =~ s!\s*/\*.*\*/!!;    # remove potential comment
+            $init =~ s/^\s*=\s*/= /;      # remove assigner
+            $init =~ s/\s*,\s*$//;      # remove comma
+
+            push @{ $SAI_ENUMS{$currentEnum}->{values} }, $enumName;
+            push @{ $SAI_ENUMS{$currentEnum}->{inits} }, $init;
+
+            $IGNORED{$enumName} = $init if $ignore ne "";
+
+            $ignore = "";
+        }
+    }
+}
+
+sub ProcessHeaders
+{
+    my $commit = shift;
+
+    my @headers = GetHeaderFiles("temp/commit-$commit/inc");
+
+    for my $header (@headers)
+    {
+        LogDebug "Processing $header";
+
+        ProcessSingleHeader "temp/commit-$commit/inc/$header";
+    }
+}
+
+sub ProcessAllEnumInitializers
+{
+    for my $enumTypeName (sort keys %SAI_ENUMS)
+    {
+        LogDebug $enumTypeName;
+
+        my $arr_ref = $SAI_ENUMS{$enumTypeName}->{values};
+        my $ini_ref = $SAI_ENUMS{$enumTypeName}->{inits};
+
+        ProcessEnumInitializers($arr_ref, $ini_ref, $enumTypeName, \%SAI_DEFINES);
+    }
+}
+
+sub BuildCommitHistory
+{
+    my $commit = shift;
+
+    for my $enumTypeName (sort keys %SAI_ENUMS)
+    {
+        LogDebug $enumTypeName;
+
+        my $arr_ref = $SAI_ENUMS{$enumTypeName}->{values};
+        my $ini_ref = $SAI_ENUMS{$enumTypeName}->{inits};
+
+        my $count = scalar @$arr_ref;
+
+        for (my $idx = 0; $idx < $count; $idx++)
+        {
+            my $enumName = $arr_ref->[$idx];
+            my $enumValue = $ini_ref->[$idx];
+
+            # CheckAllEnumsEndings make sure _START match _END
+
+            next if $enumName =~ /_START$/;
+            next if $enumName =~ /_END$/;
+            next if $enumName =~ /_RANGE_BASE$/;
+
+            next if $enumName eq "SAI_API_MAX";
+            next if $enumName eq "SAI_OBJECT_TYPE_MAX";
+
+            LogError "wrong initializer on $enumName $enumValue" if not $enumValue =~ /^0x[0-9a-f]{8}$/;
+
+            if (defined $HISTORY{$enumTypeName}{$enumName} and $HISTORY{$enumTypeName}{$enumName}{value} eq $enumValue)
+            {
+                # ok, value is the same
+            }
+            elsif (not defined $HISTORY{$enumTypeName} or not defined $HISTORY{$enumTypeName}{$enumName})
+            {
+                $HISTORY{$enumTypeName}{$enumName}{name} = $enumName;
+                $HISTORY{$enumTypeName}{$enumName}{value} = $enumValue;
+                $HISTORY{$enumTypeName}{$enumName}{commit} = $commit;
+
+                if (not defined $HISTORY{$enumTypeName}{$enumValue})
+                {
+                    $HISTORY{$enumTypeName}{$enumValue} = $enumName;
+                }
+                elsif ($HISTORY{$enumTypeName}{$enumValue} eq $enumName)
+                {
+                    # ok this is the same enum in history
+                }
+                elsif (defined $IGNORED{$enumName})
+                {
+                    # ok, values are the sam, but enum is ignored (left for backward compatibility)
+                    # but we don't check if ignored value changed, it potentially switch to different ignore
+                }
+                else # 2 enums have same integer value
+                {
+                    #print "elsif (defined $enumName $IGNORED{$enumName} and $IGNORED{$enumName} eq $HISTORY{$enumTypeName}{$enumName}{name})";
+
+                    LogWarning "Both enums have the same value $enumName and $HISTORY{$enumTypeName}{$enumValue} = $enumValue";
+                }
+            }
+            else
+            {
+                LogError "check ! $enumName value is $enumValue, but on was $HISTORY{$enumTypeName}{$enumName}{value} on commit $HISTORY{$enumTypeName}{$enumName}{commit}";
+
+                $HISTORY{$enumTypeName}{$enumName}{value} = $enumValue;
+                $HISTORY{$enumTypeName}{$enumName}{commit} = $commit;
+            }
+        }
+    }
+}
+
+sub CleanData
+{
+    %SAI_ENUMS = ();
+    %SAI_DEFINES = ();
+    %IGNORED = ();
+}
+
+#
+# MAIN
+#
+
+for my $commit (@ARGV)
+{
+    # reset
+
+    LogInfo "processing commit $commit";
+
+    CleanData();
+
+    ProcessHeaders $commit;
+
+    ProcessAllEnumInitializers();
+
+    # print Dumper \%SAI_ENUMS;
+
+    BuildCommitHistory $commit;
+}
+
+ExitOnErrorsOrWarnings();

meta/checkancestry.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 Microsoft Open Technologies, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+#    THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR
+#    CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT
+#    LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS
+#    FOR A PARTICULAR PURPOSE, MERCHANTABILITY OR NON-INFRINGEMENT.
+#
+#    See the Apache Version 2.0 License for specific language governing
+#    permissions and limitations under the License.
+#
+#    Microsoft would like to thank the following companies for their review and
+#    assistance with these files: Intel Corporation, Mellanox Technologies Ltd,
+#    Dell Products, L.P., Facebook, Inc., Marvell International Ltd.
+#
+# @file    checkancestry.sh
+#
+# @brief   This module defines ancestry script
+#
+
+
+# To list git ancestry all comitts (even if there is a tree not single line)
+# this can be usefull to build histroy of enums from root (enum lock) to the
+# current origin/master and current commit - and it will be possible to fix
+# mistakes.
+
+# examples below are to show how to get correct git history tree
+# git log --graph --oneline --ancestry-path c388490^..0b90765 | cat
+# git rev-list --ancestry-path  c388490^..0b90765
+
+# If we will have our base commit, we will assume that each previous commit
+# followed metadata check, and then we can use naive approach for parsing enum
+# values instead of doing gcc compile whch can take long time. With this
+# approach we should be able to build entire history from base commit throug
+# all commits up to the current PR. This will sure that there will be no
+# abnormalities if some enums will be removed and then added again with
+# different value. This will also help to track the issue if two PRs will pass
+# validation but after they will be merged they could potentially cause enum
+# value issue and this approach will catch that.
+#
+# Working throug 25 commits takes about 0.4 seconds + parsing so it seems like
+# not a hudge time to make sure all commits are safe and even if we get at some
+# point that this will be "too slow", having all history, we can sometimes
+# produce "known" history with enum values and keep that file as a reference
+# and load it at begin, and start checking commits from one of the future
+# commits, basicially reducing processing time to zero.
+
+# Just for sanity we can also keep headers check to 1 commit back and alse
+# maybe we can add one gcc check current to history,
+
+set -e
+
+# 1. get all necessary data to temp directory for future processing
+# 2. pass all interesting commits to processor to build history
+
+function clean_temp_dir()
+{
+    rm -rf temp
+}
+
+function create_temp_dir()
+{
+    mkdir temp
+}
+
+function checkout_inc_directories()
+{
+    echo "git checkout work tree commits:" $LIST
+
+    for commit in $LIST
+    do
+        #echo working on commit $commit
+
+        mkdir temp/commit-$commit
+        mkdir temp/commit-$commit/inc
+
+        git --work-tree=temp/commit-$commit checkout $commit inc 2>/dev/null
+
+    done
+}
+
+function create_commit_list()
+{
+    local begin=$1
+    local end=$2
+
+    echo "git rev list from $begin to $end"
+
+    LIST=$(git rev-list --ancestry-path ${begin}^..${end} | xargs -n 1 git rev-parse --short | tac)
+}
+
+function check_enum_history()
+{
+    perl ancestry.pl $LIST
+}
+
+#
+# MAIN
+#
+
+# BEGIN_COMMIT is the commit from we want enums to be backward compatible
+
+BEGIN_COMMIT=3132018
+END_COMMIT=HEAD
+
+clean_temp_dir
+create_temp_dir
+create_commit_list $BEGIN_COMMIT $END_COMMIT
+checkout_inc_directories
+check_enum_history