add support for pyproject.toml for dependency resolution (#1701)

* add support for pyproject.toml for dependency resolution

* lock pyproject deps automatically

* chore: format

* add a new claude command

* handle the case sensitive paths

* fix new mock path

Author: karakanb

.claude/commands/format-fix.md

@@ -0,0 +1,29 @@
+---
+model: claude-sonnet-4-6
+---
+
+Run `make format` and automatically fix all formatting issues in the codebase.
+
+Follow these steps:
+
+1. First, run the format command to identify issues:
+   ```
+   make format
+   ```
+
+2. Check for any formatting or linting errors in the output.
+
+3. If issues are detected, fix them automatically:
+   - Import ordering issues: Reorganize imports according to Go conventions
+   - Code formatting: Fix indentation, spacing, and line length issues
+   - Style violations: Fix naming conventions and other style issues
+   - Simple lint errors: Apply straightforward fixes
+
+4. After making fixes, re-run `make format` to verify all issues are resolved.
+
+5. If any issues require manual intervention:
+   - Show the specific errors that couldn't be auto-fixed
+   - Provide guidance on how to resolve them
+   - Re-run validation after manual fixes
+
+6. Show a summary of what was fixed and confirm the codebase passes formatting checks.

.gitignore

@@ -154,10 +154,10 @@ venv
 logs/runs
 logs/exports
 .claude
+!.claude/commands/
 bruin
 
 __debug_bin*
-.claude
 test-glossary.yml
 .venv
 logs/queries

Makefile

@@ -20,7 +20,7 @@ endif
 
 JQ_REL_PATH = jq --arg prefix "$$(pwd)" 'walk(if type == "object" and has("path") and (.path | type == "string") then .path |= (if . == $$prefix then "integration-tests" elif startswith($$prefix + "/") then .[($$prefix | length + 1):] elif startswith($$prefix) then .[($$prefix | length):] elif startswith("integration-tests/") then .[16:] else . end) else . end)'
 
-.PHONY: all clean test build build-no-duckdb format pre-commit refresh-integration-expectations integration-test-cloud validate-links
+.PHONY: all clean test build build-no-duckdb format pre-commit refresh-integration-expectations integration-test-cloud validate-links tools-update
 all: clean deps test build
 
 deps: 
@@ -97,6 +97,12 @@ format: lint-python
 	go tool golangci-lint run --timeout 10m60s --build-tags="no_duckdb_arrow" ./...  & \
 	wait
 
+tools-update:
+	go get github.com/daixiang0/gci@latest
+	go get github.com/golangci/golangci-lint/cmd/golangci-lint@latest
+	go get mvdan.cc/gofumpt@latest
+	@go mod tidy
+
 lint-python:
 	uv pip install --system sqlglot
 	@echo "$(OK_COLOR)==> Running Python formatting with ruff...$(NO_COLOR)"

cmd/internal.go

@@ -1464,11 +1464,13 @@ type LockDependenciesResult struct {
 	RequirementsPath string
 	PythonVersion    string
 	InputPath        string
+	DependencyType   python.DependencyType
+	ProjectRoot      string
 }
 
-// FindRequirementsPath finds the requirements.txt path for the given input.
-// It handles both direct requirements.txt files and Python assets.
-func (l *LockDependenciesCommand) FindRequirementsPath(inputPath string) (*LockDependenciesResult, error) {
+// FindDependencyPath finds the dependency configuration for the given input.
+// It handles direct requirements.txt files, pyproject.toml projects, and Python assets.
+func (l *LockDependenciesCommand) FindDependencyPath(inputPath string) (*LockDependenciesResult, error) {
 	// Get absolute path
 	absolutePath, err := filepath.Abs(inputPath)
 	if err != nil {
@@ -1481,53 +1483,67 @@ func (l *LockDependenciesCommand) FindRequirementsPath(inputPath string) (*LockD
 		return nil, errors2.Wrap(err, "failed to find repository")
 	}
 
-	var requirementsTxt string
-
 	// Check if input is a requirements.txt file directly
 	if strings.HasSuffix(inputPath, "requirements.txt") {
-		requirementsTxt = absolutePath
-	} else {
-		// Parse the asset
-		asset, err := l.builder.CreateAssetFromFile(absolutePath, nil)
-		if err != nil {
-			return nil, errors2.Wrap(err, "failed to parse asset")
-		}
+		return &LockDependenciesResult{
+			RequirementsPath: absolutePath,
+			InputPath:        absolutePath,
+			DependencyType:   python.DependencyTypeRequirementsTxt,
+			ProjectRoot:      filepath.Dir(absolutePath),
+		}, nil
+	}
 
-		if asset == nil {
-			return nil, errors2.New("file is not a valid asset")
-		}
+	// Check if input is a pyproject.toml file directly
+	if strings.HasSuffix(inputPath, "pyproject.toml") {
+		return &LockDependenciesResult{
+			InputPath:      absolutePath,
+			DependencyType: python.DependencyTypePyproject,
+			ProjectRoot:    filepath.Dir(absolutePath),
+		}, nil
+	}
 
-		// Check if asset is a Python asset
-		if asset.Type != pipeline.AssetTypePython && !strings.HasSuffix(asset.ExecutableFile.Path, ".py") {
-			return nil, errors2.New("asset is not a Python asset")
-		}
+	// Parse the asset
+	asset, err := l.builder.CreateAssetFromFile(absolutePath, nil)
+	if err != nil {
+		return nil, errors2.Wrap(err, "failed to parse asset")
+	}
 
-		// Find requirements.txt
-		moduleFinder := &python.ModulePathFinder{}
-		requirementsTxt, err = moduleFinder.FindRequirementsTxtInPath(repo.Path, &asset.ExecutableFile)
-		if err != nil {
-			var noReqsError *python.NoRequirementsFoundError
-			if errors.As(err, &noReqsError) {
-				return nil, errors2.New("no requirements.txt found for this asset")
-			}
-			return nil, errors2.Wrap(err, "failed to find requirements.txt")
-		}
+	if asset == nil {
+		return nil, errors2.New("file is not a valid asset")
+	}
+
+	// Check if asset is a Python asset
+	if asset.Type != pipeline.AssetTypePython && !strings.HasSuffix(asset.ExecutableFile.Path, ".py") {
+		return nil, errors2.New("asset is not a Python asset")
+	}
+
+	// Find dependency configuration
+	moduleFinder := &python.ModulePathFinder{}
+	depConfig, err := moduleFinder.FindDependencyConfig(repo.Path, &asset.ExecutableFile)
+	if err != nil {
+		return nil, errors2.Wrap(err, "failed to find dependency configuration")
+	}
+
+	if depConfig.Type == python.DependencyTypeNone {
+		return nil, errors2.New("no dependency configuration found for this asset (no requirements.txt or pyproject.toml)")
 	}
 
 	return &LockDependenciesResult{
-		RequirementsPath: requirementsTxt,
+		RequirementsPath: depConfig.RequirementsTxt,
 		InputPath:        absolutePath,
+		DependencyType:   depConfig.Type,
+		ProjectRoot:      depConfig.ProjectRoot,
 	}, nil
 }
 
 // LockAssetDependencies returns a CLI command that locks Python dependencies for an asset.
-// It finds the asset's requirements file, creates/uses a uv environment, and updates the
-// requirements file with locked versions using uv pip compile.
+// It finds the asset's dependency configuration and locks versions using either
+// uv pip compile (for requirements.txt) or uv lock (for pyproject.toml).
 func LockAssetDependencies() *cli.Command {
 	return &cli.Command{
 		Name:      "lock-asset-dependencies",
 		Usage:     "Lock Python dependencies for a Bruin asset using uv",
-		ArgsUsage: "[path to the asset definition or requirements.txt]",
+		ArgsUsage: "[path to the asset definition, requirements.txt, or pyproject.toml]",
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:        "python-version",
@@ -1560,26 +1576,19 @@ func LockAssetDependencies() *cli.Command {
 				return cli.Exit("", 1)
 			}
 
-			// Use the refactored command to find requirements path
+			// Find dependency configuration
 			cmd := &LockDependenciesCommand{
 				builder: DefaultPipelineBuilder,
 			}
 
-			result, err := cmd.FindRequirementsPath(assetPath)
+			result, err := cmd.FindDependencyPath(assetPath)
 			if err != nil {
 				printErrorJSON(err)
 				return cli.Exit("", 1)
 			}
 
 			result.PythonVersion = pythonVersion
 
-			// Find the git repo for working directory
-			repo, err := git.FindRepoFromPath(result.InputPath)
-			if err != nil {
-				printErrorJSON(errors2.Wrap(err, "failed to find repository"))
-				return cli.Exit("", 1)
-			}
-
 			// Ensure uv is installed
 			uvChecker := &uv.Checker{}
 			uvBinaryPath, err := uvChecker.EnsureUvInstalled(ctx)
@@ -1588,36 +1597,66 @@ func LockAssetDependencies() *cli.Command {
 				return cli.Exit("", 1)
 			}
 
-			// Run uv pip compile to lock dependencies
-			// uv pip compile requirements.txt -o requirements.txt --python-version X.Y --no-header
-			execCmd := exec.CommandContext(ctx, uvBinaryPath, "pip", "compile", result.RequirementsPath, "-o", result.RequirementsPath, "--python-version", pythonVersion, "--quiet", "--no-header") //nolint:gosec
-			execCmd.Dir = repo.Path
+			var execCmd *exec.Cmd
+			switch result.DependencyType { //nolint:exhaustive
+			case python.DependencyTypePyproject:
+				// uv lock --python <version>
+				execCmd = exec.CommandContext(ctx, uvBinaryPath, "lock", "--python", pythonVersion) //nolint:gosec
+				execCmd.Dir = result.ProjectRoot
+
+			case python.DependencyTypeRequirementsTxt:
+				// uv pip compile requirements.txt -o requirements.txt --python-version X.Y --no-header
+				execCmd = exec.CommandContext(ctx, uvBinaryPath, "pip", "compile", result.RequirementsPath, "-o", result.RequirementsPath, "--python-version", pythonVersion, "--quiet", "--no-header") //nolint:gosec
+				execCmd.Dir = result.ProjectRoot
+
+			default:
+				printErrorJSON(errors2.New("no supported dependency configuration found"))
+				return cli.Exit("", 1)
+			}
+
 			execCmd.Stdout = os.Stdout
 			execCmd.Stderr = os.Stderr
 
 			err = execCmd.Run()
 			if err != nil {
-				printErrorJSON(errors2.Wrap(err, "failed to lock dependencies with uv pip compile"))
+				if result.DependencyType == python.DependencyTypePyproject {
+					printErrorJSON(errors2.Wrap(err, "failed to lock dependencies with uv lock"))
+				} else {
+					printErrorJSON(errors2.Wrap(err, "failed to lock dependencies with uv pip compile"))
+				}
 				return cli.Exit("", 1)
 			}
 
 			// Output result
 			switch output {
 			case outputFormatPlain:
-				fmt.Printf("Successfully locked dependencies in %s\n", result.RequirementsPath)
+				if result.DependencyType == python.DependencyTypePyproject {
+					fmt.Printf("Successfully locked dependencies in %s\n", filepath.Join(result.ProjectRoot, "uv.lock"))
+				} else {
+					fmt.Printf("Successfully locked dependencies in %s\n", result.RequirementsPath)
+				}
 			case "json":
 				type jsonResponse struct {
-					RequirementsPath string `json:"requirements_path"`
+					RequirementsPath string `json:"requirements_path,omitempty"`
+					LockFilePath     string `json:"lock_file_path,omitempty"`
 					PythonVersion    string `json:"python_version"`
 					AssetPath        string `json:"asset_path"`
+					DependencyType   string `json:"dependency_type"`
 					Success          bool   `json:"success"`
 				}
 
 				finalOutput := jsonResponse{
-					RequirementsPath: result.RequirementsPath,
-					PythonVersion:    pythonVersion,
-					AssetPath:        result.InputPath,
-					Success:          true,
+					PythonVersion: pythonVersion,
+					AssetPath:     result.InputPath,
+					Success:       true,
+				}
+
+				if result.DependencyType == python.DependencyTypePyproject {
+					finalOutput.DependencyType = "pyproject"
+					finalOutput.LockFilePath = filepath.Join(result.ProjectRoot, "uv.lock")
+				} else {
+					finalOutput.DependencyType = "requirements_txt"
+					finalOutput.RequirementsPath = result.RequirementsPath
 				}
 
 				jsonData, err := json.Marshal(finalOutput)

cmd/internal_test.go

@@ -205,7 +205,7 @@ func normalize(s string) string {
 	return strings.Join(out, "\n")
 }
 
-func TestLockDependenciesCommand_FindRequirementsPath(t *testing.T) {
+func TestLockDependenciesCommand_FindDependencyPath(t *testing.T) {
 	t.Parallel()
 
 	tests := []struct {
@@ -255,41 +255,41 @@ func TestLockDependenciesCommand_FindRequirementsPath(t *testing.T) {
 				builder: DefaultPipelineBuilder,
 			}
 
-			result, err := cmd.FindRequirementsPath(tt.inputPath)
+			result, err := cmd.FindDependencyPath(tt.inputPath)
 
 			if tt.wantErr {
 				if err == nil {
-					t.Errorf("FindRequirementsPath() expected error, got nil")
+					t.Errorf("FindDependencyPath() expected error, got nil")
 					return
 				}
 				if tt.wantErrContain != "" && !strings.Contains(err.Error(), tt.wantErrContain) {
-					t.Errorf("FindRequirementsPath() error = %v, want error containing %q", err, tt.wantErrContain)
+					t.Errorf("FindDependencyPath() error = %v, want error containing %q", err, tt.wantErrContain)
 				}
 				return
 			}
 
 			if err != nil {
-				t.Errorf("FindRequirementsPath() unexpected error: %v", err)
+				t.Errorf("FindDependencyPath() unexpected error: %v", err)
 				return
 			}
 
 			if result == nil {
-				t.Error("FindRequirementsPath() returned nil result")
+				t.Error("FindDependencyPath() returned nil result")
 				return
 			}
 
 			if tt.wantReqSuffix != "" && !strings.HasSuffix(result.RequirementsPath, tt.wantReqSuffix) {
-				t.Errorf("FindRequirementsPath() RequirementsPath = %v, want suffix %v", result.RequirementsPath, tt.wantReqSuffix)
+				t.Errorf("FindDependencyPath() RequirementsPath = %v, want suffix %v", result.RequirementsPath, tt.wantReqSuffix)
 			}
 
 			if result.InputPath == "" {
-				t.Error("FindRequirementsPath() InputPath should not be empty")
+				t.Error("FindDependencyPath() InputPath should not be empty")
 			}
 		})
 	}
 }
 
-func TestLockDependenciesCommand_FindRequirementsPath_PythonAssetWithoutRequirements(t *testing.T) {
+func TestLockDependenciesCommand_FindDependencyPath_PythonAssetWithoutDependencies(t *testing.T) {
 	t.Parallel()
 
 	// Create a temp directory with a Python asset but no requirements.txt
@@ -317,13 +317,13 @@ print("hello")
 		builder: DefaultPipelineBuilder,
 	}
 
-	_, err := cmd.FindRequirementsPath(pyPath)
+	_, err := cmd.FindDependencyPath(pyPath)
 	if err == nil {
-		t.Error("FindRequirementsPath() expected error for Python asset without requirements.txt, got nil")
+		t.Error("FindDependencyPath() expected error for Python asset without dependencies, got nil")
 		return
 	}
 
-	if !strings.Contains(err.Error(), "no requirements.txt found") {
-		t.Errorf("FindRequirementsPath() error = %v, want error containing 'no requirements.txt found'", err)
+	if !strings.Contains(err.Error(), "no dependency configuration found") {
+		t.Errorf("FindDependencyPath() error = %v, want error containing 'no dependency configuration found'", err)
 	}
 }