[meta] Add check for struct/union binary backward compatibility

meta/Makefile

@@ -83,8 +83,10 @@ SYMBOLS = $(OBJ:=.symbols)
 
 all: toolsversions saisanitycheck saimetadatatest saiserializetest saidepgraph.svg $(SYMBOLS)
 	./checkheaders.pl ../inc ../inc
+	./aspellcheck.pl
 	./checkenumlock.sh
 	./checkancestry.sh
+	./checkstructs.sh
 	./saimetadatatest >/dev/null
 	./saiserializetest >/dev/null
 	./saisanitycheck

meta/aspell.en.pws

@@ -8,6 +8,7 @@ apache
 api
 APIs
 Args
+armhf
 attr
 attrid
 attrs
@@ -68,6 +69,7 @@ HQoS
 HQOS
 http
 idriver
+ifdef
 INET
 ingressing
 inout

meta/aspellcheck.pl

@@ -0,0 +1,206 @@
+#!/usr/bin/perl
+#
+# Copyright (c) 2023 Microsoft Open Technologies, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+#    THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR
+#    CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT
+#    LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS
+#    FOR A PARTICULAR PURPOSE, MERCHANTABILITY OR NON-INFRINGEMENT.
+#
+#    See the Apache Version 2.0 License for specific language governing
+#    permissions and limitations under the License.
+#
+#    Microsoft would like to thank the following companies for their review and
+#    assistance with these files: Intel Corporation, Mellanox Technologies Ltd,
+#    Dell Products, L.P., Facebook, Inc., Marvell International Ltd.
+#
+# @file    aspellcheck.pl
+#
+# @brief   This module run aspell on meta source and headers
+#
+
+BEGIN { push @INC,'.'; }
+
+use strict;
+use warnings;
+use diagnostics;
+
+use Term::ANSIColor;
+
+use utils;
+use style;
+
+our $errors = 0;
+our $warnings = 0;
+
+sub GetSourceFilesAndHeaders
+{
+    my @files = `find . -name "*.cpp" -o -name "*.[ch]"`;
+
+    return @files;
+}
+
+sub ReadFile
+{
+    my $filename = shift;
+
+    local $/ = undef;
+
+    # first search file in meta directory
+
+    open FILE, $filename or die "Couldn't open file $filename: $!";
+
+    binmode FILE;
+
+    my $string = <FILE>;
+
+    close FILE;
+
+    return $string;
+}
+
+sub ExtractComments
+{
+    my $input = shift;
+
+    my $comments = "";
+
+    # good enough comments extractor C/C++ source
+
+    while ($input =~ m!(".*?")|//.*?[\r\n]|/\*.*?\*/!s)
+    {
+        $input = $';
+
+        $comments .= $& if not $1;
+    }
+
+    return $comments;
+}
+
+sub RunAspell
+{
+    my $hash = shift;
+
+    my %wordsToCheck = %{ $hash };
+
+    if (not -e "/usr/bin/aspell")
+    {
+        LogError "ASPELL IS NOT PRESENT, please install aspell";
+        return;
+    }
+
+    LogInfo "Running Aspell";
+
+    my @keys = sort keys %wordsToCheck;
+
+    my $count = @keys;
+
+    my $all = "@keys";
+
+    LogInfo "Words to check: $count";
+
+    my @result = `echo "$all" | /usr/bin/aspell -l en -a -p ./aspell.en.pws 2>&1`;
+
+    for my $res (@result)
+    {
+        if ($res =~ /error/i)
+        {
+            LogError "aspell error: $res";
+            last;
+        }
+
+        chomp $res;
+        next if $res =~ /^\*?$/;
+
+        print "$res\n";
+        next if not $res =~ /^\s*&\s*(\S+)/;
+
+        my $word = $1;
+
+        next if $word =~ /^wred$/i;
+
+        chomp $res;
+
+        my $where = "??";
+
+        if (not defined $wordsToCheck{$word})
+        {
+            for my $k (@keys)
+            {
+                if ($k =~ /(^$word|$word$)/)
+                {
+                    $where = $wordsToCheck{$k};
+                    last;
+                }
+
+                $where = $wordsToCheck{$k} if ($k =~ /$word/);
+            }
+        }
+        else
+        {
+            $where = $wordsToCheck{$word};
+        }
+
+        LogWarning "Word '$word' is misspelled $where";
+    }
+}
+
+my @acronyms = GetAcronyms();
+
+my %spellAcronyms = ();
+
+$spellAcronyms{$_} = 1 for @acronyms;
+
+my @exceptions = qw/ IPv4 IPv6 0xFF IPv SAIMETADATALOGGER auth objecttype saimetadatalogger sak /;
+
+my %spellExceptions = map { $_ => $_ } @exceptions;
+
+my @files = GetSourceFilesAndHeaders();
+
+my %wordsToCheck = ();
+
+for my $file (@files)
+{
+    chomp $file;
+    next if $file =~ m!saiswig.cpp!;
+    next if $file =~ m!temp!;
+    next if $file =~ m!xml!;
+    next if $file =~ m!saimetadata.[ch]!;
+
+    my $data = ReadFile $file;
+
+    my $comments = ExtractComments $data;
+
+    $comments =~ s!github.com\S+! !g;
+    $comments =~ s!l2mc! !g;
+    $comments =~ s!\s+\d+(th|nd) ! !g;
+    $comments =~ s!(/\*|\*/)! !g;
+    $comments =~ s!//! !g;
+    $comments =~ s!\s+\*\s+! !g;
+    $comments =~ s![^a-zA-Z0-9_]! !g;
+
+    my @words = split/\s+/,$comments;
+
+    for my $word (@words)
+    {
+        next if defined $spellAcronyms{$word};
+        next if defined $spellExceptions{$word};
+
+        next if $word =~ /_/;
+        next if $word =~ /xYYY+/;
+        next if $word =~ /fe\d+/;
+        next if $word =~ /ebe\d+/;
+        next if $word =~ /Werror/;
+        next if $word =~ /^[A-Za-z][a-z]+([A-Z][a-z]+)+$/; # fooBar FooBar
+
+        $wordsToCheck{$word} = $file;
+    }
+}
+
+RunAspell(\%wordsToCheck);
+
+exit 1 if ($warnings > 0 or $errors > 0);

meta/checkstructs.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 Microsoft Open Technologies, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+#    THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR
+#    CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT
+#    LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS
+#    FOR A PARTICULAR PURPOSE, MERCHANTABILITY OR NON-INFRINGEMENT.
+#
+#    See the Apache Version 2.0 License for specific language governing
+#    permissions and limitations under the License.
+#
+#    Microsoft would like to thank the following companies for their review and
+#    assistance with these files: Intel Corporation, Mellanox Technologies Ltd,
+#    Dell Products, L.P., Facebook, Inc., Marvell International Ltd.
+#
+# @file    checkstructs.sh
+#
+# @brief   This module defines check structs script
+#
+
+
+# To list git ancestry all comitts (even if there is a tree not single line)
+# this can be usefull to build histroy of structs from root (struct lock) to the
+# current origin/master and current commit - and it will be possible to fix
+# mistakes.
+
+# examples below are to show how to get correct git history tree
+# git log --graph --oneline --ancestry-path c388490^..0b90765 | cat
+# git rev-list --ancestry-path  c388490^..0b90765
+
+# If we will have our base commit, we will assume that each previous commit
+# followed metadata check, and then we can use naive approach for parsing struct
+# values instead of doing gcc compile whch can take long time. With this
+# approach we should be able to build entire history from base commit throug
+# all commits up to the current PR. This will sure that there will be no
+# abnormalities if some structs will be removed and then added again with
+# different value. This will also help to track the issue if two PRs will pass
+# validation but after they will be merged they could potentially cause struct
+# value issue and this approach will catch that.
+#
+# Working throug 25 commits takes about 0.4 seconds + parsing so it seems like
+# not a hudge time to make sure all commits are safe and even if we get at some
+# point that this will be "too slow", having all history, we can sometimes
+# produce "known" history with structs values and keep that file as a reference
+# and load it at begin, and start checking commits from one of the future
+# commits, basicially reducing processing time to zero.
+
+# Just for sanity we can also keep headers check to 1 commit back and also
+# maybe we can add one gcc check current to history,
+
+set -e
+
+# 1. get all necessary data to temp directory for future processing
+# 2. pass all interesting commits to processor to build history
+
+function clean_temp_dir()
+{
+    rm -rf temp
+}
+
+function create_temp_dir()
+{
+    mkdir temp
+}
+
+function checkout_inc_directories()
+{
+    echo "git checkout work tree commits:" $LIST
+
+    for commit in $LIST
+    do
+        #echo working on commit $commit
+
+        mkdir temp/commit-$commit
+        mkdir temp/commit-$commit/inc
+
+        git --work-tree=temp/commit-$commit checkout $commit inc 2>/dev/null
+
+    done
+}
+
+function create_commit_list()
+{
+    local begin=$1
+    local end=$2
+
+    echo "ancestry graph"
+
+    # NOTE: origin/master should be changed if this file is on different branch
+
+    git --no-pager log --graph --oneline --ancestry-path  origin/master^..HEAD
+
+    echo "git rev list from $begin to $end"
+
+    LIST=$(git rev-list --ancestry-path ${begin}^..${end} | xargs -n 1 git rev-parse --short | tac)
+}
+
+function check_structs_history()
+{
+    perl structs.pl $LIST
+}
+
+#
+# MAIN
+#
+
+# BEGIN_COMMIT is the commit from we want structs to be backward compatible
+
+BEGIN_COMMIT=97a1e02 # v1.11.0
+END_COMMIT=HEAD
+
+clean_temp_dir
+create_temp_dir
+create_commit_list $BEGIN_COMMIT $END_COMMIT
+checkout_inc_directories
+check_structs_history