Modern Testing (capstone-engine#2456)

Author: Rot127

.github/workflows/CITest.yml

@@ -6,7 +6,7 @@ on:
       - "docs/**"
       - "ChangeLog"
       - "CREDITS.TXT"
-      - "COMPILE.TXT"
+      - "COMPILE_MAKE.TXT"
       - "COMPILE_MSVC.TXT"
       - "COMPILE_CMAKE.TXT"
       - "HACK.TXT"
@@ -34,41 +34,45 @@ jobs:
               os: ubuntu-22.04,
               arch: x64,
               build-system: 'make',
+              diet-build: 'OFF',
               enable-asan: 'OFF'
             }
           - {
               name: 'ubuntu-22.04 x64 cmake',
               os: ubuntu-22.04,
               arch: x64,
               build-system: 'cmake',
+              diet-build: 'OFF',
               enable-asan: 'OFF'
             }
           - {
-              name: 'ubuntu-22.04 x64 ASAN',
-              os: ubuntu-latest,
+              name: 'ubuntu-24.04 x64 ASAN',
+              os: ubuntu-24.04,
               arch: x64,
               build-system: 'cmake',
+              diet-build: 'OFF',
               enable-asan: 'ON'
             }
 
     steps:
     - uses: actions/checkout@v3
 
-    - name: prepare
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.config.python-version }}
+
+    - name: Prepare fuzzing
       run: |
         export LD_LIBRARY_PATH=`pwd`/tests/:$LD_LIBRARY_PATH
         wget https://github.com/groundx/capstonefuzz/raw/master/corpus/corpus-libFuzzer-capstone_fuzz_disasmnext-latest.zip
         unzip -q corpus-libFuzzer-capstone_fuzz_disasmnext-latest.zip -d suite/fuzz
-        git clone https://git.cryptomilk.org/projects/cmocka.git suite/cstest/cmocka
-        chmod +x suite/cstest/build_cstest.sh
 
     - name: make
       if: startsWith(matrix.config.build-system, 'make')
       run: |
         ./make.sh
-        make check
         sudo make install
-        cp libcapstone.so.5 libcapstone.so.5.0
 
     - name: cmake
       if: startsWith(matrix.config.build-system, 'cmake')
@@ -82,47 +86,55 @@ jobs:
         # build shared library
         cmake -DCAPSTONE_INSTALL=1 -DBUILD_SHARED_LIBS=1 -DCMAKE_INSTALL_PREFIX=/usr -DCAPSTONE_BUILD_CSTEST=ON -DENABLE_ASAN=${asan} ..
         sudo cmake --build . --config Release --target install
-        cp libcapstone.* ../
-        cp libcapstone.* ../tests/
-        cp test_* ../tests/
 
     - name: Lower number of KASL randomized address bits
       run: |
         # Work-around ASAN bug https://github.com/google/sanitizers/issues/1716
         sudo sysctl vm.mmap_rnd_bits=28
 
-    - name: "Compatibility header test build"
-      if: matrix.config.diet-build == 'OFF'
-      env:
-        asan: ${{ matrix.config.enable-asan }}
+    - name: "Compatibility header test"
+      if: startsWith(matrix.config.build-system, 'cmake') && matrix.config.diet-build == 'OFF'
       run: |
-        cd "$(git rev-parse --show-toplevel)/suite/auto-sync/c_tests/"
-        if [ "$asan" = "ON" ]; then
-          clang -lcapstone -fsanitize=address src/test_arm64_compatibility_header.c -o test_arm64_compatibility_header
-        else
-          clang -lcapstone src/test_arm64_compatibility_header.c -o test_arm64_compatibility_header
-        fi
-        ./test_arm64_compatibility_header
+        ctest --test-dir build --output-on-failure -R ASCompatibilityHeaderTest
 
     - name: cstool - reaches disassembler engine
       run: |
         sh suite/run_invalid_cstool.sh
 
-    - name: cstest (cmake)
+    - name: cstest unit tests
       if: startsWith(matrix.config.build-system, 'cmake')
       run: |
-        python suite/cstest/cstest_report.py -D -d suite/MC
-        python suite/cstest/cstest_report.py -D -f suite/cstest/issues.cs
-        python suite/cstest/cstest_report.py -D -f tests/cs_details/issue.cs
+        ctest --test-dir build --output-on-failure -R UnitCSTest
 
-    - name: cstest (make)
-      if: startsWith(matrix.config.build-system, 'make')
+    - name: cstest integration tests
+      if: startsWith(matrix.config.build-system, 'cmake')
+      run: |
+        ctest --test-dir build --output-on-failure -R IntegrationCSTest
+
+    - name: cstest MC
+      if: startsWith(matrix.config.build-system, 'cmake')
+      run: |
+        ctest --test-dir build --output-on-failure -R MCTests
+
+    - name: cstest details
+      if: startsWith(matrix.config.build-system, 'cmake')
+      run: |
+        ctest --test-dir build --output-on-failure -R DetailTests
+
+    - name: cstest issues
+      if: startsWith(matrix.config.build-system, 'cmake')
+      run: |
+        ctest --test-dir build --output-on-failure -R IssueTests
+
+    - name: cstest features
+      if: startsWith(matrix.config.build-system, 'cmake')
+      run: |
+        ctest --test-dir build --output-on-failure -R FeaturesTests
+
+    - name: Legacy integration tests
+      if: startsWith(matrix.config.build-system, 'cmake')
       run: |
-        cd suite/cstest && ./build_cstest.sh
-        python cstest_report.py -D -t build/cstest -d ../MC
-        python cstest_report.py -D -t build/cstest -f issues.cs
-        python cstest_report.py -D -t build/cstest -f ../../tests/cs_details/issue.cs
-        cd ../../
+        ctest --test-dir build --output-on-failure -R legacy*
 
   Windows:
     runs-on: ${{ matrix.config.os }}

.github/workflows/auto-sync.yml

@@ -3,11 +3,17 @@ on:
   push:
     paths:
       - "suite/auto-sync/**"
+      - ".github/workflows/auto-sync.yml"
   pull_request:
 
+# Stop previous runs on the same branch on new push
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     defaults:
       run:
         working-directory: suite/auto-sync/
@@ -30,16 +36,40 @@ jobs:
         run: |
           python3.11 -m black --check src/autosync
 
-      - name: Build llvm-tblgen
+      - name: Install llvm-mc
+        run: |
+          sudo apt install llvm-18
+          llvm-mc-18 --version
+          FileCheck-18 --version
+          sudo ln -s $(whereis -b llvm-mc-18 | grep -Eo "/.*") /usr/local/bin/llvm-mc
+          sudo ln -s $(whereis -b FileCheck-18 | grep -Eo "/.*") /usr/local/bin/FileCheck
+          llvm-mc --version
+          FileCheck --version
+
+      - name: Clone llvm-capstone
         run: |
           git clone https://github.com/capstone-engine/llvm-capstone.git vendor/llvm_root
+
+      - name: Build llvm-tblgen
+        run: |
           cd vendor/llvm_root
           mkdir build
           cd build
           cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug ../llvm
           cmake --build . --target llvm-tblgen --config Debug
           cd ../../../
 
+      - name: Test Header patcher
+        run: |
+          python -m unittest src/autosync/Tests/test_header_patcher.py
+          python -m unittest src/autosync/Tests/test_mcupdater.py
+
+      - name: Remove llvm-mc
+        run: |
+          sudo apt remove llvm-18
+          sudo rm /usr/local/bin/llvm-mc
+          sudo rm /usr/local/bin/FileCheck
+
       - name: Test generation of inc files
         run: |
           ./src/autosync/ASUpdater.py -d -a AArch64 -s IncGen
@@ -63,11 +93,9 @@ jobs:
           ./src/autosync/ASUpdater.py --ci -d -a PPC -s Translate
           ./src/autosync/ASUpdater.py --ci -d -a LoongArch -s Translate
 
-      - name: Test Header patcher
-        run: |
-          python -m unittest src/autosync/Tests/test_header_patcher.py
-          python -m unittest src/autosync/Tests/test_mcupdater.py
-
       - name: Differ - Test save file is up-to-date
         run: |
           ./src/autosync/cpptranslator/Differ.py -a AArch64 --check_saved
+          ./src/autosync/cpptranslator/Differ.py -a ARM --check_saved
+          ./src/autosync/cpptranslator/Differ.py -a PPC --check_saved
+          ./src/autosync/cpptranslator/Differ.py -a LoongArch --check_saved

.github/workflows/clang-tidy.yml

@@ -8,11 +8,11 @@ on:
 
 jobs:
   analyze:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
 
     name: clang-tidy
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Install clang-tidy
@@ -26,13 +26,18 @@ jobs:
         CC=clang sudo cmake --build . --config Release
         cd ..
 
-    - name: Install clang-tidy-15
+    - name: Install clang-tidy-18
       run: |
-        sudo apt install clang-tidy-15
+        sudo apt install clang-tidy-18
 
     - name: Check for warnings
       env:
         base_sha: ${{ github.event.pull_request.base.sha }}
         head_sha: ${{ github.event.pull_request.head.sha }}
       run: |
         ./run-clang-tidy.sh build
+
+    - uses: actions/upload-artifact@v4
+      if: ${{ failure() }}
+      with:
+        path: ct-warnings.txt

.github/workflows/python-tests.yml

@@ -28,5 +28,30 @@ jobs:
     - name: Build and install capstone
       run: pip install ./bindings/python
 
-    - name: Run tests
+    - name: Install cstest_py
+      run: pip install ./bindings/python/cstest_py
+
+    - name: Run legacy tests
       run: python ./bindings/python/tests/test_all.py
+
+    - name: cstest_py integration tests
+      run: |
+        cd suite/cstest/test/
+        python3 ./integration_tests.py cstest_py
+        cd ../../../
+
+    - name: cstest_py MC
+      run: |
+        cstest_py tests/MC/
+
+    - name: cstest_py details
+      run: |
+        cstest_py tests/details/
+
+    - name: cstest_py issues
+      run: |
+        cstest_py tests/issues/
+
+    - name: cstest_py features
+      run: |
+        cstest_py tests/features/

.gitignore

@@ -25,6 +25,7 @@
 # python
 bindings/python/build/
 bindings/python/capstone.egg-info/
+bindings/python/cstest_py/src/cstest_py.egg**
 bindings/cython/capstone.egg-info/
 *.pyc
 
@@ -133,7 +134,6 @@ fuzz_disasm
 fuzz_decode_platform
 capstone_get_setup
 suite/fuzz/corpus
-suite/cstest/cmocka/
 
 *.s
 
@@ -147,3 +147,6 @@ android-ndk-*
 
 # Auto-sync files
 suite/auto-sync/src/autosync.egg-info
+
+# clangd cache
+/.cache