diff --git a/.github/workflows/check-sdk-version.yml b/.github/workflows/check-sdk-version.yml index 83ba70aba..9c4255475 100644 --- a/.github/workflows/check-sdk-version.yml +++ b/.github/workflows/check-sdk-version.yml @@ -34,7 +34,6 @@ jobs: if [ -n "$EXISTS" ]; then echo "Error: Version $PKG_VERSION of $PKG_NAME already exists on PyPI." >&2 echo "This change will NOT trigger a new release. Please bump the version in pyproject.toml." >&2 - sleep 1 exit 1 else echo "Version $PKG_VERSION does not exist on PyPI. This change will trigger a new release." diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 7d6bbf12d..000000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,76 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ "main" ] - paths: - - 'api/**' - pull_request: - # The branches below must be a subset of the branches above - branches: [ "main" ] - paths: - - 'api/**' - schedule: - - cron: '44 16 * * 4' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'javascript' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - # â„šī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index b6409d5c6..c79ec27ec 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,14 +1,6 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# +# Unified CodeQL security scanning for both Python and JavaScript/TypeScript. +# Replaces the former separate codeql-analysis.yml workflow. + name: "CodeQL" on: @@ -23,16 +15,15 @@ on: schedule: - cron: '30 21 * * 0' +concurrency: + group: ${{ github.workflow }}-${{ matrix.language }}-${{ github.ref }} + cancel-in-progress: true + jobs: analyze: - name: Analyze - # Runner size impacts CodeQL analysis time. To learn more, please see: - # - https://gh.io/recommended-hardware-resources-for-running-codeql - # - https://gh.io/supported-runners-and-hardware-resources - # - https://gh.io/using-larger-runners - # Consider using larger runners for possible analysis time improvements. - runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} - timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + name: Analyze (${{ matrix.language }}) + runs-on: ubuntu-latest + timeout-minutes: 60 permissions: actions: read contents: read @@ -41,44 +32,20 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'python' ] - # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] - # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both - # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + language: [ 'javascript-typescript', 'python' ] steps: - name: Checkout repository uses: actions/checkout@v4 - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). - # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v3 - # â„šī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 with: diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 34a1de537..3bcaabbaf 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -7,19 +7,20 @@ on: jobs: build-and-push: runs-on: ubuntu-latest + timeout-minutes: 60 defaults: run: working-directory: api steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Log in to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 457baace3..4e162b8a4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,12 +2,21 @@ name: Lint on: [push] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm - name: Install modules - run: npm install + run: npm ci - name: Run ESLint run: npm run lint diff --git a/.github/workflows/netlify-docs-build.yml b/.github/workflows/netlify-docs-build.yml deleted file mode 100644 index 8476c0141..000000000 --- a/.github/workflows/netlify-docs-build.yml +++ /dev/null @@ -1,22 +0,0 @@ -# This workflow triggers a Netlify docs build when a new GitHub release is published -# TEMPORARILY DISABLED -name: Trigger Netlify Docs Build on Release - -on: - workflow_dispatch: # Temporarily disable: Allows manual triggering only -# release: -# types: [published] - -jobs: - build-docs: - runs-on: ubuntu-latest - steps: - - name: Trigger Netlify Build Hook - env: - NETLIFY_BUILD_HOOK: ${{ secrets.NETLIFY_BUILD_HOOK }} - run: | - if [ -z "$NETLIFY_BUILD_HOOK" ]; then - echo "NETLIFY_BUILD_HOOK secret is not set!" - exit 1 - fi - curl -X POST -d '{}' "$NETLIFY_BUILD_HOOK" diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml index 3d7c12096..8e5c0651e 100644 --- a/.github/workflows/prettier.yml +++ b/.github/workflows/prettier.yml @@ -6,6 +6,10 @@ on: pull_request: branches: [main] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: prettier-check: runs-on: ubuntu-latest @@ -20,8 +24,8 @@ jobs: node-version: "22" cache: "npm" - - name: Install Prettier - run: npm install -g prettier + - name: Install dependencies + run: npm ci - name: Check Prettier formatting run: npm run format:check diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9f584d2a8..153def203 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -8,24 +8,21 @@ on: jobs: publish: - runs-on: ${{ matrix.os }} - - strategy: - matrix: - os: [macos-latest] + runs-on: macos-latest + timeout-minutes: 30 steps: - name: Checkout git repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Node and NPM - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: 18 cache: npm - name: Install dependencies - run: npm install + run: npm ci - name: Build default web bundle run: npm run build @@ -53,18 +50,15 @@ jobs: run: | VERSION=$(node -p "require('./package.json').version") echo "version=${VERSION}" >> $GITHUB_OUTPUT - shell: bash - name: Ensure GitHub Release exists run: | gh release view ${{ github.ref_name }} >/dev/null 2>&1 || gh release create ${{ github.ref_name }} --generate-notes --verify-tag --draft env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - name: Upload static web assets run: | gh release upload ${{ github.ref_name }} release/transformerlab_web.tar.gz release/transformerlab_web_multiuser.tar.gz env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash diff --git a/.github/workflows/pytest-sdk.yml b/.github/workflows/pytest-sdk.yml index b4e7464f3..af9e1948b 100644 --- a/.github/workflows/pytest-sdk.yml +++ b/.github/workflows/pytest-sdk.yml @@ -12,32 +12,32 @@ on: paths: - "lab-sdk/**" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: pytest: runs-on: ubuntu-latest defaults: run: working-directory: lab-sdk - strategy: - fail-fast: false - matrix: - python-version: ["3.10"] steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.10" - name: Set up uv uses: astral-sh/setup-uv@v4 - name: Install project with uv run: | - uv venv --python ${{ matrix.python-version }} + uv venv --python 3.10 uv pip install -e . - name: Run SDK tests (uv) diff --git a/.github/workflows/pytest-server-test-macos.yml b/.github/workflows/pytest-server-test-macos.yml deleted file mode 100644 index 401baa235..000000000 --- a/.github/workflows/pytest-server-test-macos.yml +++ /dev/null @@ -1,75 +0,0 @@ -# This workflow will install Python dependencies and run tests on macOS - -name: Test for Install and Serving (macOS) - -on: - push: - branches: ["main"] - paths: - - "api/**" - - "lab-sdk/**" - pull_request: - branches: ["main"] - paths: - - "api/**" - - "lab-sdk/**" - -jobs: - build: - runs-on: macos-latest - defaults: - run: - working-directory: api - env: - AUTH_CLIENT_ID: ${{ secrets.AUTH_CLIENT_ID }} - AUTH_API_KEY: ${{ secrets.AUTH_API_KEY }} - FRONTEND_URL: http://localhost:8338 - AUTH_SUCCESS_REDIRECT_URL: / - AUTH_ERROR_REDIRECT_URL: / - AUTH_LOGOUT_REDIRECT_URL: / - TRANSFORMERLAB_JWT_SECRET: ${{ secrets.TRANSFORMERLAB_JWT_SECRET }} - TRANSFORMERLAB_REFRESH_SECRET: ${{ secrets.TRANSFORMERLAB_REFRESH_SECRET }} - EMAIL_METHOD: "dev" - strategy: - fail-fast: false - matrix: - python-version: ["3.11"] - - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - pip install uv - ENV_NAME="transformerlab" - TLAB_DIR="$HOME/.transformerlab" - TLAB_CODE_DIR="${TLAB_DIR}/src" - TLAB_STATIC_WEB_DIR="${TLAB_DIR}/webapp" - mkdir -p "${TLAB_DIR}" - mkdir -p "${TLAB_CODE_DIR}" - mkdir -p "${TLAB_STATIC_WEB_DIR}" - # Temporarily remove transformerlab from pyproject.toml to avoid installing unreleased version - # macOS uses BSD sed which requires: sed -i '' or sed -i '.bak' - sed -i '.bak' '/^ "transformerlab==/d' pyproject.toml - ./install.sh install_conda - ./install.sh create_conda_environment - ./install.sh install_dependencies - mv pyproject.toml.bak pyproject.toml || true - uv pip install --system pytest pytest-asyncio jsonschema requests - - name: Install lab-sdk in editable mode - run: | - cd ../lab-sdk - eval "$($HOME/.transformerlab/miniforge3/bin/conda shell.bash hook)" - conda activate $HOME/.transformerlab/envs/transformerlab - uv pip install -e . - - name: Now run the server by calling ./run.sh and wait for it to be ready - run: | - ./run.sh & - python scripts/wait_for_server.py --timeout 120 --interval 2 - - name: Test the API Server and DB Setup - run: | - # Run tests - pytest -m live_server test/server/ diff --git a/.github/workflows/pytest-server-test.yml b/.github/workflows/pytest-server-test.yml index 60f852928..494ff4a06 100644 --- a/.github/workflows/pytest-server-test.yml +++ b/.github/workflows/pytest-server-test.yml @@ -1,6 +1,7 @@ # This workflow will install Python dependencies and run tests +# Covers both Ubuntu and macOS (replaces the former separate macOS workflow) -name: Test for Install and Serving (Ubuntu) +name: Test for Install and Serving on: push: @@ -14,9 +15,14 @@ on: - "api/**" - "lab-sdk/**" +concurrency: + group: ${{ github.workflow }}-${{ matrix.os }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + timeout-minutes: 30 defaults: run: working-directory: api @@ -33,14 +39,14 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.11"] + os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.11" - name: Install dependencies run: | pip install uv diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 09221611e..ba223accd 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -12,9 +12,14 @@ on: paths: - "api/**" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest + timeout-minutes: 20 defaults: run: working-directory: api @@ -27,15 +32,13 @@ jobs: AUTH_LOGOUT_REDIRECT_URL: / strategy: fail-fast: false - matrix: - python-version: ["3.11"] steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.11" - name: Install dependencies run: | pip install uv diff --git a/.github/workflows/python-package-sdk.yml b/.github/workflows/python-package-sdk.yml index 56b566a2e..b8009c2fb 100644 --- a/.github/workflows/python-package-sdk.yml +++ b/.github/workflows/python-package-sdk.yml @@ -1,11 +1,6 @@ # This workflow will upload a Python Package to PyPI when a release is created # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - name: Build and Upload SDK Python Package on: @@ -45,13 +40,11 @@ jobs: EXISTS=$(curl -s https://pypi.org/pypi/$PKG_NAME/json | grep -F "\"$PKG_VERSION\"" || true) if [ -n "$EXISTS" ]; then echo "Error: Version $PKG_VERSION of $PKG_NAME already exists on PyPI." >&2 - sleep 1 exit 1 fi - name: Build release distributions run: | - # NOTE: put your own distribution build steps here. python -m pip install build python -m build @@ -66,36 +59,16 @@ jobs: needs: - build-release-distributions - # Dedicated environments with protections for publishing are strongly recommended. - # For more information, see: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#deployment-protection-rules environment: name: pypi - # url: https://pypi.org/project/transformerlab/${{ github.event.release.name }} steps: - - uses: actions/checkout@v4 - - name: Retrieve release distributions uses: actions/download-artifact@v4 with: name: release-dists path: dist/ - - name: Check if version already exists on PyPI - run: | - cd lab-sdk - PKG_NAME=$(grep -E '^name\s*=\s*' pyproject.toml | head -1 | sed 's/name\s*=\s*"\(.*\)"/\1/') - PKG_VERSION=$(grep -E '^version\s*=\s*' pyproject.toml | head -1 | sed 's/version\s*=\s*"\(.*\)"/\1/') - echo "Package name: $PKG_NAME" - echo "Package version: $PKG_VERSION" - echo "Checking if version exists on PyPI..." - EXISTS=$(curl -s https://pypi.org/pypi/$PKG_NAME/json | grep -F "\"$PKG_VERSION\"" || true) - if [ -n "$EXISTS" ]; then - echo "Error: Version $PKG_VERSION of $PKG_NAME already exists on PyPI." >&2 - sleep 1 - exit 1 - fi - - name: Publish release distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 95d276045..21d9ab33f 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -1,9 +1,15 @@ name: Ruff + on: push: branches: - main pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: ruff: runs-on: ubuntu-latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c8ea922c4..ea64f48cc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,9 +2,14 @@ name: Test on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ matrix.os }}-${{ github.ref }} + cancel-in-progress: true + jobs: test: runs-on: ${{ matrix.os }} + timeout-minutes: 30 strategy: matrix: @@ -12,17 +17,16 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Node.js and NPM - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: 18 cache: npm - name: npm install - run: | - npm install + run: npm ci - name: npm test env: