diff --git a/.github/workflows/test_be.yaml b/.github/workflows/test_be.yaml index 7ea6115430c..676a491320e 100644 --- a/.github/workflows/test_be.yaml +++ b/.github/workflows/test_be.yaml @@ -94,6 +94,8 @@ jobs: uses: styfle/cancel-workflow-action@0.12.1 - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for git diff - name: 🥚 Install Hatch uses: pypa/hatch@install @@ -119,23 +121,33 @@ jobs: python-version: ${{ matrix.python-version }} # Test with base dependencies - - name: Test with base dependencies + - name: Test changed with base dependencies if: ${{ matrix.dependencies == 'core' }} run: | + # On PRs, compare against base branch; on main, compare against HEAD~1 + CHANGED_FROM="${{ github.base_ref && format('origin/{0}', github.base_ref) || 'HEAD~1' }}" hatch run +py=${{ matrix.python-version }} test:test tests/ \ -v \ -k "not test_cli" \ --durations=10 \ + -p packages.pytest_changed --changed-from=$CHANGED_FROM \ --picked=first # Test with optional dependencies - - name: Test with optional dependencies + - name: Test changed with optional dependencies if: ${{ matrix.dependencies == 'core,optional' }} + # Include all tests on 3.13 or main run: | + INCLUDE_UNCHANGED=${{ matrix.python-version == '3.13' }} || ${{ github.ref == 'refs/heads/main' }} + # On PRs, compare against base branch; on main, compare against HEAD~1 + CHANGED_FROM="${{ github.base_ref && format('origin/{0}', github.base_ref) || 'HEAD~1' }}" + hatch run +py=${{ matrix.python-version }} test-optional:test tests/ \ -v \ -k "not test_cli" \ --durations=10 \ + -p packages.pytest_changed --changed-from=$CHANGED_FROM \ + --include-unchanged=$INCLUDE_UNCHANGED \ --picked=first # Test with minimal dependencies using lowest resolution @@ -144,11 +156,14 @@ jobs: - name: Test with minimal dependencies (lowest resolution) if: ${{ matrix.dependencies == 'minimal' }} run: | + # On PRs, compare against base branch; on main, compare against HEAD~1 + CHANGED_FROM="${{ github.base_ref && format('origin/{0}', github.base_ref) || 'HEAD~1' }}" hatch run +py=${{ matrix.python-version }} test:test tests/ \ -v \ -k "not test_cli" \ --durations=10 \ - --picked=first + --picked=first \ + -p packages.pytest_changed --changed-from=$CHANGED_FROM env: UV_RESOLUTION: lowest @@ -177,8 +192,13 @@ jobs: - name: Test with optional dependencies run: | - hatch run +py=3.12 test-optional:test -v tests/ -k "not test_cli" --durations=10 \ - --cov=marimo --cov-report=xml --junitxml=junit.xml -o junit_family=legacy + hatch run +py=3.12 test-optional:test \ + -v \ + tests/ \ + -k "not test_cli" \ + --durations=10 \ + --picked=first \ + --cov=marimo --cov-report=xml --junitxml=junit.xml -o junit_family=legacy # Only upload coverage on `main` so it is not blocking # and only on `3.12`, `ubuntu-latest`, with optional deps since it is mostly duplicate diff --git a/marimo/_data/models.py b/marimo/_data/models.py index 6f5ab9b9467..a19931cf5b1 100644 --- a/marimo/_data/models.py +++ b/marimo/_data/models.py @@ -39,7 +39,7 @@ class DataTableColumn(BaseStruct): sample_values: list[Any] def __post_init__(self) -> None: - # Sometimes like pandas, sqlalchemy or ibis may return column names as objects + # Sometimes libraries (like pandas, sqlalchemy or ibis) may return column names as objects # instead of strings, although their type hints are str # Instead of trying to track this down each time, just convert to string self.name = str(self.name) diff --git a/packages/pytest_changed/README.md b/packages/pytest_changed/README.md new file mode 100644 index 00000000000..4a073ecbd8a --- /dev/null +++ b/packages/pytest_changed/README.md @@ -0,0 +1,70 @@ +# pytest-changed Plugin + +A pytest plugin that uses `ruff analyze graph` to intelligently discover and run only the tests affected by code changes. + +## Overview + +This plugin analyzes your git changes and uses Ruff's dependency graph analysis to determine which tests need to be run. It performs a search through the dependency graph to find all files affected by your changes, then runs only the tests that could be impacted. + +## Usage + +**Run only tests affected by changes from main branch:** + +```bash +pytest -p packages.pytest_changed --changed-from=main +``` + +**Using Hatch:** + +```bash +hatch run +py=3.12 test:test -p packages.pytest_changed --changed-from=main tests/ +``` + +### Compare Against Different References + +```bash +# Compare against HEAD (staged changes) +pytest -p packages.pytest_changed --changed-from=HEAD + +# Compare against origin/main +pytest -p packages.pytest_changed --changed-from=origin/main +``` + +### Preview What Would Be Run + +Use `--collect-only` to see which tests would be selected without running them: + +```bash +pytest -p packages.pytest_changed --changed-from=main --collect-only tests/ +``` + +### Include Unchanged Tests + +Run affected tests plus all other tests: + +```bash +pytest -p packages.pytest_changed --changed-from=main --include-unchanged tests/ +``` + +### Run Specific Test Directories + +The plugin respects pytest's normal path filtering: + +```bash +pytest -p packages.pytest_changed --changed-from=main tests/_runtime/ +``` + +## How It Works + +1. **Find Changes**: Uses `git diff --name-only ` to find Python files that have changed +2. **Build Graph**: Runs `ruff analyze graph --direction dependents` to build a dependency graph +3. **Graph Traversal**: Performs BFS from changed files to find all affected files +4. **Filter Tests**: Identifies which affected files are test files +5. **Run Tests**: Only runs pytest on the affected test files + +## Configuration Options + +### Command Line Options + +- `--changed-from=`: Git reference to compare against (required to activate plugin) +- `--include-unchanged`: Also run tests that haven't changed (optional) diff --git a/packages/pytest_changed/__init__.py b/packages/pytest_changed/__init__.py new file mode 100644 index 00000000000..3255d05807e --- /dev/null +++ b/packages/pytest_changed/__init__.py @@ -0,0 +1,301 @@ +""" +Pytest plugin that uses `ruff analyze graph` to discover and run tests +affected by changes from a git reference. + +Usage: + pytest --changed-from=HEAD + pytest --changed-from=main + pytest --changed-from=origin/main +""" + +import json +import subprocess +from pathlib import Path + +import pytest + +print_ = print + + +def pytest_addoption(parser: pytest.Parser) -> None: + """Add command-line options for the plugin.""" + group = parser.getgroup( + "ruff-graph", "Run tests based on dependency graph" + ) + group.addoption( + "--changed-from", + action="store", + dest="changed_from", + default=None, + help="Git reference to compare against (e.g., HEAD, main, origin/main)", + ) + group.addoption( + "--include-unchanged", + action="store", + dest="include_unchanged", + default=False, + type=lambda x: x.lower() in ("true", "1", "yes"), + help="Also run tests that haven't changed (in addition to affected tests)", + ) + + +def get_changed_files(ref: str, repo_root: Path) -> set[Path]: + """ + Get list of changed Python files from git diff. + + Args: + ref: Git reference to compare against + repo_root: Root of the git repository + + Returns: + Set of absolute paths to changed Python files + """ + try: + result = subprocess.run( + ["git", "diff", "--name-only", ref], + cwd=repo_root, + capture_output=True, + text=True, + check=True, + ) + + changed_files: set[Path] = set() + for line in result.stdout.strip().split("\n"): + if not line: + continue + file_path = repo_root / line + if file_path.suffix == ".py" and file_path.exists(): + changed_files.add(file_path) + + return changed_files + except subprocess.CalledProcessError as e: + error_msg = f"Failed to get git diff from '{ref}': {e}" + if e.stderr: + error_msg += f"\nStderr: {e.stderr}" + error_msg += ( + "\n\nTip: Make sure the reference exists. " + "In CI, you may need to fetch history with 'fetch-depth: 0' " + "in actions/checkout." + ) + pytest.exit(error_msg, returncode=1) + + +def get_dependency_graph( + repo_root: Path, direction: str = "dependents" +) -> dict[str, list[str]]: + """ + Get dependency graph from ruff analyze graph. + + Args: + repo_root: Root of the repository + direction: Either "dependencies" or "dependents" + + Returns: + Dictionary mapping file paths to list of related file paths + """ + try: + result = subprocess.run( + [ + "uvx", + # uv notes `analyze graph` is experimental, + # so we fix the ruff version for now + "ruff@0.13.2", + "analyze", + "graph", + "--detect-string-imports", + "--direction", + direction, + ".", + ], + cwd=repo_root, + capture_output=True, + text=True, + check=True, + ) + + # Parse JSON output + graph: dict[str, list[str]] = json.loads(result.stdout) + + # Convert to absolute paths + absolute_graph = {} + for file_path, dependencies in graph.items(): + abs_path = str(repo_root / file_path) + abs_deps = [str(repo_root / dep) for dep in dependencies] + absolute_graph[abs_path] = abs_deps + + return absolute_graph + except subprocess.CalledProcessError as e: + pytest.exit(f"Failed to run ruff analyze graph: {e}", returncode=1) + except json.JSONDecodeError as e: + pytest.exit(f"Failed to parse ruff output: {e}", returncode=1) + + +def find_affected_files( + changed_files: set[Path], + dependency_graph: dict[str, list[str]], +) -> set[Path]: + """ + Find all files affected by changes using BFS graph traversal. + Handles cycles and deduplication. + + Args: + changed_files: Set of files that have changed + dependency_graph: Map of file -> files that depend on it + + Returns: + Set of all affected file paths (including changed files) + """ + affected: set[Path] = set(changed_files) + visited: set[str] = set() + queue: list[Path] = list(changed_files) + + while queue: + current = queue.pop(0) + current_str = str(current) + + if current_str in visited: + continue + visited.add(current_str) + + # Find all files that depend on this file + dependents = dependency_graph.get(current_str, []) + for dependent in dependents: + dependent_path = Path(dependent) + if dependent_path not in affected: + affected.add(dependent_path) + queue.append(dependent_path) + + return affected + + +def find_test_files(affected_files: set[Path], repo_root: Path) -> set[Path]: + """ + Filter affected files to only include test files. + + Args: + affected_files: Set of all affected files + repo_root: Root of the repository + + Returns: + Set of test file paths + """ + del repo_root + test_files: set[Path] = set() + + for file_path in affected_files: + # Check if it's a test file + if file_path.exists() and ( + file_path.name.startswith("test_") + or file_path.name.endswith("_test.py") + or "tests" in file_path.parts + ): + test_files.add(file_path) + + return test_files + + +def pytest_configure(config: pytest.Config) -> None: + """Configure pytest with our plugin.""" + changed_from: str | None = config.getoption("changed_from") + + if changed_from is None: + return + + # Get the repository root + try: + result = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + check=True, + ) + repo_root = Path(result.stdout.strip()) + except subprocess.CalledProcessError: + pytest.exit("Not in a git repository", returncode=1) + + # Step 1: Find changed files + changed_files = get_changed_files(changed_from, repo_root) + + if not changed_files: + print_(f"No Python files changed from {changed_from}") + config.option.changed_test_files = set() + return + + print_( + f"Found {len(changed_files)} changed Python files from {changed_from}" + ) + for f in sorted(changed_files): + print_(f" - {f.relative_to(repo_root)}") + + # Step 2: Get dependency graph (dependents direction) + print_("\nBuilding dependency graph...") + dependency_graph = get_dependency_graph(repo_root, direction="dependents") + + # Step 3: Find all affected files (BFS with cycle detection) + print_("Finding affected files...") + affected_files = find_affected_files(changed_files, dependency_graph) + print_(f"Found {len(affected_files)} affected files") + + # Step 4: Filter to test files + test_files = find_test_files(affected_files, repo_root) + + if not test_files: + print_(f"\nNo tests affected by changes from {changed_from}") + config.option.changed_test_files = set() + return + + print_(f"\nFound {len(test_files)} affected test files:") + for test_file in sorted(test_files): + print_(f" - {test_file.relative_to(repo_root)}") + + # Store the test files in config for collection hook + config.option.changed_test_files = test_files + + +def pytest_collection_modifyitems( + config: pytest.Config, items: list[pytest.Item] +) -> None: + """Modify test collection to only include affected tests.""" + changed_from: str | None = config.getoption("changed_from") + + if changed_from is None: + return + + changed_test_files: set[Path] = getattr( + config.option, "changed_test_files", set() + ) + + if not changed_test_files: + # No affected tests, skip all + for item in items: + item.add_marker( + pytest.mark.skip( + reason=f"Not affected by changes from {changed_from}" + ) + ) + return + + include_unchanged = config.getoption("include_unchanged") + + # Filter items to only those in affected test files + selected: list[pytest.Item] = [] + deselected: list[pytest.Item] = [] + + for item in items: + test_file = Path(item.fspath) + + if test_file in changed_test_files: + selected.append(item) + elif include_unchanged: + # Run all tests if include_unchanged is set + selected.append(item) + else: + deselected.append(item) + + if deselected: + config.hook.pytest_deselected(items=deselected) + items[:] = selected + + print_(f"\nRunning {len(selected)} tests from affected files") + if deselected: + print_(f"Skipped {len(deselected)} tests from unaffected files") diff --git a/tests/_server/api/endpoints/test_execution.py b/tests/_server/api/endpoints/test_execution.py index 210787a3f50..6e7d19e0e4b 100644 --- a/tests/_server/api/endpoints/test_execution.py +++ b/tests/_server/api/endpoints/test_execution.py @@ -186,6 +186,7 @@ def test_takeover_file_key(client: TestClient) -> None: sys.platform == "win32", reason="Skipping test on Windows due to websocket issues", ) + @pytest.mark.flaky(reruns=5) @with_session(SESSION_ID) def test_app_meta_request(client: TestClient) -> None: response = client.post( @@ -356,6 +357,7 @@ def test_takeover_no_file_key(client: TestClient) -> None: assert response.status_code == 401, response.text @staticmethod + @pytest.mark.flaky(reruns=5) @with_session(SESSION_ID) def test_app_meta_request(client: TestClient) -> None: response = client.post(