diff --git a/.github/workflows/autoqa-reliability.yml b/.github/workflows/autoqa-reliability.yml new file mode 100644 index 0000000000..759e937173 --- /dev/null +++ b/.github/workflows/autoqa-reliability.yml @@ -0,0 +1,121 @@ +name: AutoQA Reliability (Manual) + +on: + workflow_dispatch: + inputs: + source_type: + description: 'App source type (url)' + required: true + type: choice + options: [url] + default: url + jan_app_windows_source: + description: 'Windows installer URL path (used when source_type=url or to select artifact)' + required: true + type: string + default: 'https://catalog.jan.ai/windows/Jan_0.6.8_x64-setup.exe' + jan_app_ubuntu_source: + description: 'Ubuntu .deb URL path' + required: true + type: string + default: 'https://delta.jan.ai/nightly/Jan-nightly_0.6.4-728_amd64.deb' + jan_app_macos_source: + description: 'macOS .dmg URL path' + required: true + type: string + default: 'https://delta.jan.ai/nightly/Jan-nightly_0.6.4-728_universal.dmg' + is_nightly: + description: 'Is the app a nightly build?' + required: true + type: boolean + default: true + reliability_phase: + description: 'Reliability phase' + required: true + type: choice + options: [development, deployment] + default: development + reliability_runs: + description: 'Custom runs (0 uses phase default)' + required: true + type: number + default: 0 + reliability_test_path: + description: 'Test file path (relative to autoqa working directory)' + required: true + type: string + default: 'tests/base/settings/app-data.txt' + +jobs: + reliability-windows: + runs-on: windows-11-nvidia-gpu + timeout-minutes: 60 + env: + DEFAULT_JAN_APP_URL: 'https://catalog.jan.ai/windows/Jan_0.6.8_x64-setup.exe' + DEFAULT_IS_NIGHTLY: 'false' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Python 3.13 + uses: actions/setup-python@v4 + with: + python-version: '3.13' + + - name: Clean existing Jan installations + shell: powershell + run: | + .\autoqa\scripts\windows_cleanup.ps1 -IsNightly "${{ inputs.is_nightly }}" + + - name: Download/Prepare Jan app + shell: powershell + run: | + .\autoqa\scripts\windows_download.ps1 ` + -WorkflowInputUrl "${{ inputs.jan_app_windows_source }}" ` + -WorkflowInputIsNightly "${{ inputs.is_nightly }}" ` + -RepoVariableUrl "${{ vars.JAN_APP_URL }}" ` + -RepoVariableIsNightly "${{ vars.IS_NIGHTLY }}" ` + -DefaultUrl "$env:DEFAULT_JAN_APP_URL" ` + -DefaultIsNightly "$env:DEFAULT_IS_NIGHTLY" + + - name: Install Jan app + shell: powershell + run: | + .\autoqa\scripts\windows_install.ps1 -IsNightly "$env:IS_NIGHTLY" + + - name: Install Python dependencies + working-directory: autoqa + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run reliability tests + working-directory: autoqa + shell: powershell + run: | + $runs = "${{ inputs.reliability_runs }}" + $runsArg = "" + if ([int]$runs -gt 0) { $runsArg = "--reliability-runs $runs" } + python main.py --enable-reliability-test --reliability-phase "${{ inputs.reliability_phase }}" --reliability-test-path "${{ inputs.reliability_test_path }}" $runsArg + + - name: Upload screen recordings + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: reliability-recordings-${{ github.run_number }}-${{ runner.os }} + path: autoqa/recordings/ + + - name: Upload trajectories + if: always() + uses: actions/upload-artifact@v4 + continue-on-error: true + with: + name: reliability-trajectories-${{ github.run_number }}-${{ runner.os }} + path: autoqa/trajectories/ + + - name: Cleanup after tests + if: always() + shell: powershell + run: | + .\autoqa\scripts\windows_post_cleanup.ps1 -IsNightly "${{ inputs.is_nightly }}"