MSTICPy CI build and check #3048
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will install Python dependencies, run tests and lint with a variety of Python versions | |
| # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions | |
| name: MSTICPy CI build and check | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main, release/*] | |
| merge_group: | |
| schedule: | |
| - cron: "0 0 * * 0,2,4" | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| permissions: read-all | |
| strategy: | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13"] | |
| steps: | |
| # Print out details about the run | |
| - name: Dump GitHub context | |
| env: | |
| GITHUB_CONTEXT: ${{ toJSON(github) }} | |
| run: echo "$GITHUB_CONTEXT" | |
| - name: Dump job context | |
| env: | |
| JOB_CONTEXT: ${{ toJSON(job) }} | |
| run: echo "$JOB_CONTEXT" | |
| # end print details | |
| - uses: actions/checkout@v4 | |
| - name: Free up disk space | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc | |
| sudo rm -rf /usr/share/locale/* | |
| sudo rm -rf /usr/share/i18n/* | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache pip | |
| uses: actions/cache@v4 | |
| with: | |
| # This path is specific to Ubuntu | |
| path: ~/.cache/pip | |
| # Look to see if there is a cache hit for the corresponding requirements file | |
| key: ${{ runner.os }}-pip-${{ hashFiles('setup.py', 'requirements.txt', 'requirements-dev.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip-${{ hashFiles('setup.py', 'requirements.txt', 'requirements-dev.txt') }} | |
| ${{ runner.os }}-pip | |
| - name: Install dependencies | |
| # Installs core install + all extras ("all") + dev/test dependencies ("dev") | |
| # via the combined "test" extra defined in setup.py. | |
| run: | | |
| python -m pip install --upgrade pip wheel setuptools | |
| python -m pip install --use-pep517 -e ".[test]" | |
| - name: Install test dependencies | |
| # ToDo - remove pip install xgboost when flaml is fixed | |
| run: | | |
| python -m pip install "pandas>=1.3.0" "pygeohash>=1.2.0" | |
| python -m pip install "xgboost" | |
| - name: Prepare test dummy data | |
| run: | | |
| mkdir ~/.msticpy | |
| mkdir ~/.msticpy/mordor | |
| cp ./tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy | |
| touch ~/.msticpy/GeoLite2-City.mmdb | |
| cp -r ./tests/testdata/mordor/* ~/.msticpy/mordor | |
| touch ~/.msticpy/mordor/mitre_tact_cache.pkl | |
| touch ~/.msticpy/mordor/mitre_tech_cache.pkl | |
| touch ~/.msticpy/mordor/mordor_cache.pkl | |
| - name: Pytest | |
| env: | |
| MAXMIND_AUTH: ${{ secrets.MAXMIND_AUTH }} | |
| IPSTACK_AUTH: ${{ secrets.IPSTACK_AUTH }} | |
| MSTICPYCONFIG: ./tests/msticpyconfig-test.yaml | |
| MSTICPY_BUILD_SOURCE: fork | |
| JUPYTER_PLATFORM_DIRS: 1 | |
| run: | | |
| jupyter --paths | |
| pytest tests -n auto --junitxml=junit/test-${{ matrix.python-version }}-results.xml --cov=msticpy --cov-report=xml | |
| if: ${{ always() }} | |
| - name: Upload pytest test results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pytest-results-${{ matrix.python-version }} | |
| path: junit/test-${{ matrix.python-version }}-results.xml | |
| # Use always() to always run this step to publish test results when there are test failures | |
| if: ${{ always() }} | |
| docs: | |
| runs-on: ubuntu-latest | |
| permissions: read-all | |
| strategy: | |
| matrix: | |
| python-version: ["3.12"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Free up disk space | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc | |
| sudo rm -rf /usr/share/locale/* | |
| sudo rm -rf /usr/share/i18n/* | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache pip | |
| uses: actions/cache@v4 | |
| with: | |
| # This path is specific to Ubuntu | |
| path: ~/.cache/pip | |
| # Look to see if there is a cache hit for the corresponding requirements file | |
| key: ${{ runner.os }}-pip-docs-${{ hashFiles('docs/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip-docs-${{ hashFiles('docs/requirements.txt') }} | |
| ${{ runner.os }}-pip-docs | |
| ${{ runner.os }}-pip | |
| - name: Sphinx Read the Docs build | |
| working-directory: docs | |
| run: | | |
| echo "Building docs from ${{ github.workspace }}/docs" | |
| python -m pip install sphinx readthedocs-sphinx-ext>=2.1.4 sphinx-rtd-theme>=1.0.0 | |
| python -m pip install -r ${{ github.workspace }}/docs/requirements.txt | |
| make html | |
| env: | |
| SPHINX_NOGEN: "true" | |
| lint: | |
| runs-on: ubuntu-latest | |
| permissions: read-all | |
| strategy: | |
| matrix: | |
| python-version: ["3.12"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Free up disk space | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc | |
| sudo rm -rf /usr/share/locale/* | |
| sudo rm -rf /usr/share/i18n/* | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache pip | |
| uses: actions/cache@v4 | |
| with: | |
| # This path is specific to Ubuntu | |
| path: ~/.cache/pip | |
| # Look to see if there is a cache hit for the corresponding requirements file | |
| key: ${{ runner.os }}-pip-lint-${{ hashFiles('setup.py', 'requirements.txt', 'requirements-dev.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip-lint-${{ hashFiles('setup.py', 'requirements.txt', 'requirements-dev.txt') }} | |
| ${{ runner.os }}-pip-lint | |
| ${{ runner.os }}-pip | |
| - name: Install dependencies | |
| # Installs core install + all extras ("all") + dev/test dependencies ("dev") | |
| # via the combined "test" extra defined in setup.py. | |
| run: | | |
| python -m pip install --upgrade pip wheel setuptools | |
| python -m pip install -e ".[test]" | |
| - name: ruff | |
| run: | | |
| ruff check msticpy --ignore PLW0603 | |
| if: ${{ always() }} | |
| - name: pylint | |
| run: | | |
| pylint msticpy --disable=duplicate-code --disable=E1135,E1101,E1133,W0101 | |
| if: ${{ always() }} | |
| - name: Cache/restore MyPy data | |
| id: cache-mypy | |
| uses: actions/cache@v4 | |
| with: | |
| # MyPy cache files are stored in `~/.mypy_cache` | |
| path: .mypy_cache | |
| key: ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }} | |
| ${{ runner.os }}-build-mypy-${{ github.ref }} | |
| ${{ runner.os }}-build-mypy | |
| - name: mypy | |
| # --python-version is set explicitly to the job's Python version because mypy | |
| # otherwise infers its syntax target from setup.cfg's python_requires (the | |
| # package's minimum supported version), which can cause false positives on | |
| # newer-syntax stubs (e.g. PEP 695 `type` statements in recent numpy releases). | |
| run: | | |
| mypy --ignore-missing-imports --follow-imports=silent --show-column-numbers --show-error-end --show-error-context --disable-error-code annotation-unchecked --python-version ${{ matrix.python-version }} --junit-xml junit/mypy-test-${{ matrix.python-version }}-results.xml msticpy | |
| if: ${{ always() }} | |
| - name: Upload mypy test results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Mypy results ${{ matrix.python-version }} | |
| path: junit/mypy-test-${{ matrix.python-version }}-results.xml | |
| # Use always() to always run this step to publish test results when there are test failures | |
| if: ${{ always() }} | |
| - name: bandit | |
| run: | | |
| bandit -x tests -r -s B303,B404,B603,B607,B608,B113 msticpy | |
| if: ${{ always() }} | |
| - name: pyroma | |
| run: | | |
| pyroma --min 10 . | |
| if: ${{ always() }} | |
| check_status: | |
| runs-on: ubuntu-latest | |
| permissions: read-all | |
| needs: [build, lint, docs] | |
| steps: | |
| - name: File build fail issue | |
| if: ${{ env.GITHUB_REF_NAME == 'main' && ( needs.build.result == 'failure' || needs.lint.result == 'failure' ) }} | |
| uses: dacbd/create-issue-action@v1 | |
| with: | |
| token: ${{ github.token }} | |
| title: "Build failed for main branch" | |
| body: The build failed on branch ${{ github.ref }}. Please investigate | |
| labels: build_break, bug, high_severity |