Benchmark #140
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| jobs: | |
| mot-metrics-benchmark: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ['3.12'] | |
| tracker: ["ocsort", "bytetrack", "botsort", "deepocsort", "hybridsort", "strongsort", "boosttrack"] | |
| timeout-minutes: 50 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: 'pip' | |
| - name: Install requirements | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y jq curl unzip | |
| python -m pip install --upgrade pip setuptools wheel uv | |
| sed -i'' -e 's/index = "torch-gpu"/index = "torch-cpu"/g' pyproject.toml | |
| uv sync --extra yolo | |
| # Restore the cached dataset (if available) | |
| - name: Restore MOT17 dataset cache | |
| id: cache-restore | |
| uses: actions/cache@v4 | |
| with: | |
| path: boxmot/engine/TrackEval/MOT17-ablation.zip | |
| key: mot17-ablation-dataset-cache-v3 | |
| # Cache data for future runs (only if the cache was not already restored) | |
| - name: Cache MOT17.zip if not already cached | |
| if: steps.cache-restore.outputs.cache-hit != 'true' | |
| uses: actions/cache@v4 | |
| with: | |
| path: boxmot/engine/TrackEval/MOT17-ablation.zip | |
| key: mot17-ablation-dataset-cache-v3 | |
| - name: Evaluation and Summarize Results | |
| run: | | |
| source .venv/bin/activate | |
| if boxmot eval --classes 0 --yolo-model yolox_x_MOT17_ablation.pt --reid-model lmbn_n_duke.pt --tracking-method ${{ matrix.tracker }} --ci --verbose --source MOT17-ablation; then | |
| STATUS="✅" | |
| else | |
| STATUS="❌" | |
| fi | |
| if [ -f ${{ matrix.tracker }}_output.json ]; then | |
| HOTA=$(jq -r '.HOTA' ${{ matrix.tracker }}_output.json) | |
| MOTA=$(jq -r '.MOTA' ${{ matrix.tracker }}_output.json) | |
| IDF1=$(jq -r '.IDF1' ${{ matrix.tracker }}_output.json) | |
| else | |
| HOTA="" | |
| MOTA="" | |
| IDF1="" | |
| fi | |
| mkdir -p results | |
| TRACKER_NAME=$(echo "${{ matrix.tracker }}" | awk '{print tolower($0)}') | |
| if [ "$STATUS" = "❌" ]; then | |
| fps="" | |
| else | |
| declare -A tracker_fps | |
| tracker_fps["deepocsort"]=12 | |
| tracker_fps["bytetrack"]=1265 | |
| tracker_fps["ocsort"]=1483 | |
| tracker_fps["strongsort"]=17 | |
| tracker_fps["botsort"]=46 | |
| tracker_fps["hybridsort"]=25 | |
| tracker_fps["boosttrack"]=25 | |
| fps=${tracker_fps[$TRACKER_NAME]} | |
| fi | |
| echo "$TRACKER_NAME,$STATUS,$HOTA,$MOTA,$IDF1,$fps" > results/${{ matrix.tracker }}.txt | |
| - name: Show Results | |
| run: cat results/${{ matrix.tracker }}.txt | |
| - name: Upload Results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: results-${{ github.run_id }}-${{ matrix.tracker }} | |
| path: results/${{ matrix.tracker }}.txt | |
| combine-results: | |
| runs-on: ubuntu-latest | |
| needs: mot-metrics-benchmark | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download all results artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: results | |
| - name: Check downloaded files | |
| run: | | |
| echo "Downloaded files in the results directory:" | |
| ls -la results/*/ || true | |
| - name: Combine results | |
| run: | | |
| touch combined_results.csv | |
| for file in results/*/*; do | |
| if [ -f "$file" ]; then | |
| cat "$file" >> combined_results.csv | |
| fi | |
| done | |
| sort -t, -k3 -nr combined_results.csv > sorted_results.csv | |
| column -s, -t sorted_results.csv > pretty_results.txt | |
| - name: Show Combined Results | |
| run: cat pretty_results.txt | |
| - name: Set up Git | |
| run: | | |
| git config --local user.email "[email protected]" | |
| git config --local user.name "mikel-brostrom" | |
| - name: Update README with tracker results | |
| run: | | |
| RESULTS_FILE="pretty_results.txt" | |
| README_FILE="README.md" | |
| cp "$README_FILE" "${README_FILE}.bak" | |
| declare -A paper_links | |
| paper_links["boosttrack"]="https://arxiv.org/abs/2408.13003" | |
| paper_links["deepocsort"]="https://arxiv.org/abs/2302.11813" | |
| paper_links["bytetrack"]="https://arxiv.org/abs/2110.06864" | |
| paper_links["ocsort"]="https://arxiv.org/abs/2203.14360" | |
| paper_links["strongsort"]="https://arxiv.org/abs/2202.13514" | |
| paper_links["botsort"]="https://arxiv.org/abs/2206.14651" | |
| paper_links["hybridsort"]="https://arxiv.org/abs/2308.00783" | |
| new_table="| Tracker | Status | HOTA↑ | MOTA↑ | IDF1↑ | FPS |\n" | |
| new_table+="| :-----: | :-----: | :---: | :---: | :---: | :---: |\n" | |
| while read -r line; do | |
| tracker=$(echo "$line" | awk '{print $1}') | |
| status=$(echo "$line" | awk '{print $2}') | |
| hota=$(echo "$line" | awk '{print $3}') | |
| mota=$(echo "$line" | awk '{print $4}') | |
| idf1=$(echo "$line" | awk '{print $5}') | |
| fps=$(echo "$line" | awk '{print $6}') | |
| paper_url=${paper_links[$tracker]} | |
| tracker_link="[$tracker](${paper_url:-#})" | |
| new_table+="| $tracker_link | $status | $hota | $mota | $idf1 | $fps |\n" | |
| done < "$RESULTS_FILE" | |
| start_marker="<!-- START TRACKER TABLE -->" | |
| end_marker="<!-- END TRACKER TABLE -->" | |
| awk -v start_marker="$start_marker" -v end_marker="$end_marker" -v new_table="$new_table" ' | |
| $0 == start_marker { print $0; print new_table; in_table=1; next } | |
| $0 == end_marker { in_table=0; print $0; next } | |
| !in_table | |
| ' "$README_FILE" > temp_readme.md | |
| mv temp_readme.md "$README_FILE" | |
| - name: Check for changes in README.md | |
| id: check_changes | |
| run: | | |
| if git diff --quiet README.md; then | |
| echo "changed=false" >> $GITHUB_ENV | |
| else | |
| BRANCH_NAME="update-tracker-results-$(date +%Y%m%d%H%M%S)" | |
| echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV | |
| echo "changed=true" >> $GITHUB_ENV | |
| fi | |
| - name: Create Pull Request | |
| if: env.changed == 'true' | |
| uses: peter-evans/create-pull-request@v7 | |
| with: | |
| add-paths: README.md | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| branch: ${{ env.BRANCH_NAME }} | |
| commit-message: Overwrite tracker results in README.md | |
| title: Overwrite tracker results in README.md | |
| body: "This PR updates the tracker results table in the README.md." |