Sync

Author: Bumber64

.github/workflows/build-windows.yml

@@ -32,6 +32,9 @@ on:
       platform-files:
         type: boolean
         default: true
+      plugins:
+        type: boolean
+        default: true
       common-files:
         type: boolean
         default: true
@@ -104,7 +107,7 @@ jobs:
             win-msvc
       - name: Cross-compile
         env:
-          CMAKE_EXTRA_ARGS: -DBUILD_LIBRARY=${{ inputs.platform-files }} -DBUILD_STONESENSE:BOOL=${{ inputs.stonesense }} -DBUILD_DOCS:BOOL=${{ inputs.docs }} -DBUILD_DOCS_NO_HTML:BOOL=${{ !inputs.html }} -DINSTALL_DATA_FILES:BOOL=${{ inputs.common-files }} -DINSTALL_SCRIPTS:BOOL=${{ inputs.common-files }} -DBUILD_DFLAUNCH:BOOL=${{ inputs.launchdf }} -DBUILD_TESTS:BOOL=${{ inputs.tests }} -DBUILD_XMLDUMP:BOOL=${{ inputs.xml-dump-type-sizes }} ${{ inputs.xml-dump-type-sizes && '-DINSTALL_XMLDUMP:BOOL=1' || '' }}
+          CMAKE_EXTRA_ARGS: -DBUILD_LIBRARY=${{ inputs.platform-files }} -DBUILD_PLUGINS:BOOL=${{ inputs.platform-files && inputs.plugins }} -DBUILD_STONESENSE:BOOL=${{ inputs.stonesense }} -DBUILD_DOCS:BOOL=${{ inputs.docs }} -DBUILD_DOCS_NO_HTML:BOOL=${{ !inputs.html }} -DINSTALL_DATA_FILES:BOOL=${{ inputs.common-files }} -DINSTALL_SCRIPTS:BOOL=${{ inputs.common-files }} -DBUILD_DFLAUNCH:BOOL=${{ inputs.launchdf }} -DBUILD_TESTS:BOOL=${{ inputs.tests }} -DBUILD_XMLDUMP:BOOL=${{ inputs.xml-dump-type-sizes }} ${{ inputs.xml-dump-type-sizes && '-DINSTALL_XMLDUMP:BOOL=1' || '' }}
         run: |
           cd build
           bash -x build-win64-from-linux.sh

.github/workflows/generate-symbols.yml

@@ -8,7 +8,7 @@ on:
         required: true
         default: master
       version:
-        description: DF version
+        description: DF version (can be "auto" if channel is steam)
         required: true
       platform:
         description: Target OS platform
@@ -44,7 +44,7 @@ on:
         type: string
 
 jobs:
-  package:
+  package-linux:
     uses: ./.github/workflows/build-linux.yml
     if: inputs.platform == 'all' || inputs.platform == 'linux'
     with:
@@ -57,12 +57,25 @@ jobs:
       plugins: false
     secrets: inherit
 
+  package-win64:
+    uses: ./.github/workflows/build-windows.yml
+    if: (inputs.platform == 'all' || inputs.platform == 'windows') && inputs.version == 'auto'
+    with:
+      dfhack_ref: ${{ github.ref }}
+      structures_ref: ${{ inputs.structures_ref }}
+      artifact-name: dfhack-symbols-windows64-build
+      append-date-and-hash: false
+      cache-id: release
+      cache-readonly: true
+      plugins: false
+    secrets: inherit
+
   generate-linux:
     name: Generate linux64 symbols
     runs-on: ubuntu-latest
     if: inputs.platform == 'all' || inputs.platform == 'linux'
     needs:
-    - package
+    - package-linux
     steps:
     - name: Install dependencies
       run: |
@@ -119,11 +132,29 @@ jobs:
           "+app_update 975370 $BETA_PARAMS validate" \
           +quit
         tar xjf dfhack-symbols-linux64-build.tar.bz2 -C DF_steam
-        xml/symbols_gen_linux.sh ${{ inputs.version }} STEAM DF_steam
+        xml/symbols_gen_linux.sh ${{ inputs.version == 'auto' && '50.0' || inputs.version }} STEAM DF_steam
+        if [ "${{ inputs.version }}" = "auto" ]; then
+          while pgrep dwarfort; do
+            echo "waiting for DF to exit"
+            sleep 0.5
+          done
+          cp xml/symbols.xml DF_steam/hack
+          cd DF_steam
+          DFHACK_DISABLE_CONSOLE=1 ./dfhack &
+          while ! ./dfhack-run lua 'print(scr)' | fgrep 'viewscreen_titlest' 2>/dev/null; do
+            echo "waiting for DF to start"
+            sleep 0.5
+          done
+          df_ver=`./dfhack-run lua 'print(dfhack.gui.getDFViewscreen(true).str_version)' | ansifilter`
+          echo "Found version string: '$df_ver'"
+          echo "DETECTED_DF_VER=$df_ver" >>$GITHUB_ENV
+          sed -i "s/v0.50.0 linux64 STEAM/v0.$df_ver linux64 STEAM/" ../xml/symbols.xml
+          ./dfhack-run die || true
+        fi
 
     # Itch
     - name: Generate Itch symbols
-      if: inputs.channel == 'all' || inputs.channel == 'itch'
+      if: (inputs.channel == 'all' || inputs.channel == 'itch') && inputs.version != 'auto'
       env:
         DISPLAY: :0
         ITCH_API_KEY: ${{ secrets.ITCH_API_KEY }}
@@ -139,7 +170,7 @@ jobs:
 
     # Classic
     - name: Generate Classic symbols
-      if: inputs.channel == 'all' || inputs.channel == 'classic'
+      if: (inputs.channel == 'all' || inputs.channel == 'classic') && inputs.version != 'auto'
       env:
         DISPLAY: :0
       run: |
@@ -164,7 +195,7 @@ jobs:
     - name: Commit symbol updates
       uses: stefanzweifel/git-auto-commit-action@v5
       with:
-        commit_message: Auto-update symbols
+        commit_message: Auto-update symbols for Linux DF version ${{ env.DETECTED_DF_VER || inputs.version }}
         repository: xml
         commit_user_name: DFHack-Urist via GitHub Actions
         commit_user_email: [email protected]
@@ -227,11 +258,11 @@ jobs:
           +login $STEAM_USERNAME \
           "+app_update 975370 $BETA_PARAMS validate" \
           +quit
-        xml/symbols_gen_windows.sh ${{ inputs.version }} STEAM DF_steam
+        xml/symbols_gen_windows.sh ${{ inputs.version == 'auto' && '50.0' || inputs.version }} STEAM DF_steam
 
     # Itch
     - name: Generate Itch symbols
-      if: inputs.channel == 'all' || inputs.channel == 'itch'
+      if: (inputs.channel == 'all' || inputs.channel == 'itch') && inputs.version != 'auto'
       env:
         ITCH_API_KEY: ${{ secrets.ITCH_API_KEY }}
       run: |
@@ -245,7 +276,7 @@ jobs:
 
     # Classic
     - name: Generate Classic symbols
-      if: inputs.channel == 'all' || inputs.channel == 'classic'
+      if: (inputs.channel == 'all' || inputs.channel == 'classic') && inputs.version != 'auto'
       run: |
         mkdir DF_classic
         minor=$(echo "${{ inputs.version }}" | cut -d. -f1)
@@ -267,7 +298,86 @@ jobs:
     - name: Commit symbol updates
       uses: stefanzweifel/git-auto-commit-action@v5
       with:
-        commit_message: Auto-update symbols
+        commit_message: Auto-update symbols for Windows DF version ${{ inputs.version }}
+        repository: xml
+        commit_user_name: DFHack-Urist via GitHub Actions
+        commit_user_email: [email protected]
+
+  auto-ver-windows:
+    name: Autodetect DF version string (Windows)
+    if: (inputs.platform == 'all' || inputs.platform == 'windows') && (inputs.channel == 'all' || inputs.channel == 'steam') && inputs.version == 'auto'
+    needs:
+      - package-win64
+      - generate-windows
+    runs-on: windows-latest
+    steps:
+    - name: Clone structures
+      uses: actions/checkout@v4
+      with:
+        repository: DFHack/df-structures
+        ref: ${{ inputs.structures_ref }}
+        token: ${{ secrets.DFHACK_GITHUB_TOKEN }}
+        path: xml
+    - name: Download DFHack
+      uses: actions/download-artifact@v4
+      with:
+        name: dfhack-symbols-windows64-build
+    - name: Setup steamcmd
+      id: steamcmd
+      uses: CyberAndrii/setup-steamcmd@v1
+    - name: Update DF version string
+      env:
+        STEAM_USERNAME: ${{ secrets.STEAM_USERNAME }}
+        STEAM_CONFIG_VDF: ${{ secrets.STEAM_CONFIG_VDF }}
+        STEAM_DF_TESTING: ${{ secrets.STEAM_DF_TESTING }}
+        STEAM_DF_ADVENTURE_TEST: ${{ secrets.STEAM_DF_ADVENTURE_TEST }}
+      shell: bash
+      run: |
+        mkdir DF_steam
+        echo "$STEAM_CONFIG_VDF" | base64 -d >${{ steps.steamcmd.outputs.directory }}/config/config.vdf
+        echo "DF steam branch: ${{ inputs.df_steam_branch }}"
+        if [ "${{ inputs.df_steam_branch }}" = "default" ]; then
+          BETA_PARAMS=""
+        elif [ "${{ inputs.df_steam_branch }}" = "testing" ]; then
+          BETA_PARAMS="-beta testing -betapassword $STEAM_DF_TESTING"
+        elif [ "${{ inputs.df_steam_branch }}" = "adventure_test" ]; then
+          BETA_PARAMS="-beta adventure_test -betapassword $STEAM_DF_ADVENTURE_TEST"
+        else
+          BETA_PARAMS="-beta ${{ inputs.df_steam_branch }}"
+        fi
+        ${{ steps.steamcmd.outputs.executable }} \
+          +@ShutdownOnFailedCommand 1 \
+          +@sSteamCmdForcePlatformType windows \
+          +force_install_dir $PWD/DF_steam \
+          +login $STEAM_USERNAME \
+          "+app_update 975370 $BETA_PARAMS validate" \
+          +quit
+        tar xjf dfhack-symbols-windows64-build.tar.bz2 -C DF_steam
+        cp xml/symbols.xml DF_steam/hack
+        cd DF_steam
+        "./Dwarf Fortress.exe" &
+        while ! ./dfhack-run.exe lua 'print(scr)' | fgrep 'viewscreen_titlest' 2>/dev/null; do
+          echo "waiting for DF to start"
+          sleep 0.5
+        done
+        df_ver=`./dfhack-run.exe lua 'print(dfhack.gui.getDFViewscreen(true).str_version)'`
+        echo "Found version string: '$df_ver'"
+        echo "DETECTED_DF_VER=$df_ver" >>$GITHUB_ENV
+        sed -i "s/v0.50.0 win64 STEAM/v0.$df_ver win64 STEAM/" ../xml/symbols.xml
+        ./dfhack-run.exe die || true
+    - name: Merge updates
+      shell: bash
+      run: |
+        cd xml
+        if ! git diff --exit-code; then
+          git stash
+          git pull
+          git stash pop
+        fi
+    - name: Commit symbol updates
+      uses: stefanzweifel/git-auto-commit-action@v5
+      with:
+        commit_message: Auto-update Windows DF version to ${{ env.DETECTED_DF_VER }} in symbols
         repository: xml
         commit_user_name: DFHack-Urist via GitHub Actions
         commit_user_email: [email protected]
@@ -277,28 +387,32 @@ jobs:
     runs-on: ubuntu-latest
     needs:
     - generate-linux
-    - generate-windows
+    - auto-ver-windows
     if: ${{ ! failure() }}
     steps:
     - name: Clone DFHack
       uses: actions/checkout@v4
       with:
         token: ${{ secrets.DFHACK_GITHUB_TOKEN }}
     - name: Update ref
+      shell: bash
       run: |
         git submodule update --init --no-single-branch library/xml
         cd library/xml
         git checkout ${{ inputs.structures_ref }}
         git pull
+        df_ver=`grep -E 'symbol-table.*STEAM' symbols.xml | head -n1 | sed -r "s/.*name='v0.([^ ]+) .*/\1/"`
+        echo "using DF version: $df_ver"
+        echo "DETECTED_DF_VER=$df_ver" >>$GITHUB_ENV
     - name: Commit ref update
       uses: stefanzweifel/git-auto-commit-action@v5
       with:
-        commit_message: Auto-update structures ref
+        commit_message: Auto-update structures ref for ${{ env.DETECTED_DF_VER }}
         commit_user_name: DFHack-Urist via GitHub Actions
         commit_user_email: [email protected]
     - name: Launch steam-deploy
       if: inputs.steam_branch
       shell: bash
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: gh workflow run steam-deploy.yml -R DFHack/dfhack -r ${{ github.ref }} -f version=${{ inputs.version }} -f steam_branch=${{ inputs.steam_branch }}
+      run: gh workflow run steam-deploy.yml -R DFHack/dfhack -r ${{ github.ref }} -f version=${{ env.DETECTED_DF_VER }} -f steam_branch=${{ inputs.steam_branch }}

.github/workflows/steam-deploy.yml

@@ -74,13 +74,15 @@ jobs:
           rm ${name}-depot.tar.bz2
           cd ..
         done
+    - name: Get short SHA of commit
+      run: echo "SHORT_SHA=`echo ${{ github.sha }} | cut -c1-8`" >>$GITHUB_ENV
     - name: Steam deploy
       uses: game-ci/steam-deploy@v3
       with:
         username: ${{ secrets.STEAM_USERNAME }}
         configVdf: ${{ secrets.STEAM_CONFIG_VDF}}
         appId: 2346660
-        buildDescription: ${{ github.event.inputs && github.event.inputs.version || github.ref_name }}
+        buildDescription: ${{ github.event.inputs && github.event.inputs.version || github.ref_name }} (${{ env.SHORT_SHA }})
         rootPath: .
         depot1Path: common-depot
         depot2Path: win64-depot

.github/workflows/watch-df-steam.yml

@@ -14,32 +14,27 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # dfhack_ref: leave blank if no structures update is desired
-        # structures_ref: leave blank to default to master
-        # steam_branch: leave blank if no steam push is desired
+        # df_steam_branch: which DF Steam branch to watch
         # platform: leave blank to default to all
-        # channel: leave blank to default to all; should usually be steam
+        # structures_ref: leave blank to default to master
+        # dfhack_ref: leave blank if no structures update is desired
+        # steam_branch: leave blank if no DFHack steam push is desired
         include:
           - df_steam_branch: public
-            version: public
           - df_steam_branch: beta
-            version: 51.01-beta
-            channel: steam
-            dfhack_ref: adv-beta
             structures_ref: adv-beta
+            dfhack_ref: adv-beta
             steam_branch: adventure-beta
           - df_steam_branch: testing
-            version: testing
-            channel: steam
-            dfhack_ref: testing
             structures_ref: testing
+            dfhack_ref: testing
     steps:
     - name: Fetch state
       uses: actions/cache/restore@v4
       with:
         path: state
-        key: watchstate-${{ matrix.version }}
-        restore-keys: watchstate-${{ matrix.version }}
+        key: watchstate-${{ matrix.df_steam_branch }}
+        restore-keys: watchstate-${{ matrix.df_steam_branch }}
     - name: Compare branch metadata
       uses: nick-fields/retry@v3
       with:
@@ -77,14 +72,14 @@ jobs:
           -R DFHack/dfhack \
           -r ${{ matrix.dfhack_ref }} \
           -f structures_ref=${{ matrix.structures_ref }} \
-          -f version=${{ matrix.version }} \
+          -f version=auto \
           -f platform=${{ matrix.platform }} \
-          -f channel=${{ matrix.channel }} \
+          -f channel=steam \
           -f df_steam_branch=${{ matrix.df_steam_branch }} \
           -f steam_branch=${{ matrix.steam_branch }}
     - name: Save state
       uses: actions/cache/save@v4
       if: env.TIMESTAMP
       with:
         path: state
-        key: watchstate-${{ matrix.version }}-${{ env.TIMESTAMP }}
+        key: watchstate-${{ matrix.df_steam_branch }}-${{ env.TIMESTAMP }}

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ ci:
 repos:
 # shared across repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.6.0
+  rev: v5.0.0
   hooks:
     - id: check-added-large-files
     - id: check-case-conflict
@@ -20,7 +20,7 @@ repos:
       args: ['--fix=lf']
     - id: trailing-whitespace
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.2
+  rev: 0.30.0
   hooks:
     - id: check-github-workflows
 - repo: https://github.com/Lucas-C/pre-commit-hooks

CMakeLists.txt

@@ -56,7 +56,7 @@ endif()
 
 if(WIN32)
     set(MSVC_MIN_VER 1930)
-    set(MSVC_MAX_VER 1941)
+    set(MSVC_MAX_VER 1942)
     if(NOT MSVC)
         message(SEND_ERROR "No MSVC found! MSVC 2022 version ${MSVC_MIN_VER} to ${MSVC_MAX_VER} is required.")
     elseif((MSVC_VERSION LESS MSVC_MIN_VER) OR (MSVC_VERSION GREATER MSVC_MAX_VER))

build/build-win64-from-linux.sh

@@ -32,9 +32,7 @@ if [[ $(id -u) -eq 0 ]]; then
     chown -R $builder_uid win64-cross
 fi
 
-# Assumes you built a container image called dfhack-build-msvc from
-# https://github.com/BenLubar/build-env/tree/master/msvc, see
-# docs/dev/compile/Compile.rst.
+# Pulls the MSVC build env container from the GitHub registry
 #
 # NOTE: win64-cross is mounted in /src/build due to the hardcoded `cmake ..` in
 # the Dockerfile
@@ -44,7 +42,7 @@ if ! docker run --rm -i -v "$srcdir":/src -v "$srcdir/build/win64-cross/":/src/b
     -e steam_username \
     -e steam_password \
     --name dfhack-win \
-    ghcr.io/dfhack/build-env:msvc \
+    ghcr.io/dfhack/build-env:master \
     bash -c "cd /src/build && dfhack-configure windows 64 Release -DCMAKE_INSTALL_PREFIX=/src/build/output -DBUILD_DOCS=1 $CMAKE_EXTRA_ARGS && dfhack-make -j$jobs install" \
     ; then
     echo

data/base_command_counts.json

@@ -98,7 +98,7 @@
 	"gui/mechanisms": 1,
 	"gui/pathable": 1,
 	"hotkeys": 1,
-	"infiniteSky": 1,
+	"infinite-sky": 1,
 	"force": 1,
 	"hermit": 1,
 	"strangemood": 1,