From 0bf34b0c8c544535fcd61b45e9db04a01218450b Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 2 Oct 2025 15:11:40 +0200 Subject: [PATCH 1/6] Fix processing short pip flags in environment dependencies --- bundle/libraries/local_path.go | 2 +- bundle/libraries/local_path_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go index efbf82eadc..3bf8942723 100644 --- a/bundle/libraries/local_path.go +++ b/bundle/libraries/local_path.go @@ -91,7 +91,7 @@ func IsLocalRequirementsFile(dep string) (string, bool) { } func containsPipFlag(input string) bool { - re := regexp.MustCompile(`--[a-zA-Z0-9-]+`) + re := regexp.MustCompile(`--?[a-zA-Z0-9-]+`) return re.MatchString(input) } diff --git a/bundle/libraries/local_path_test.go b/bundle/libraries/local_path_test.go index a46b7522f5..a054a47b1a 100644 --- a/bundle/libraries/local_path_test.go +++ b/bundle/libraries/local_path_test.go @@ -57,6 +57,7 @@ func TestIsLibraryLocal(t *testing.T) { {path: "s3://mybucket/path/to/package", expected: false}, {path: "dbfs:/mnt/path/to/package", expected: false}, {path: "beautifulsoup4", expected: false}, + {path: "-e some/local/path", expected: false}, // Check the possible version specifiers as in PEP 440 // https://peps.python.org/pep-0440/#public-version-identifiers From 3013c11089e850ce27b2b6829edfe4932be8555c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 2 Oct 2025 15:13:17 +0200 Subject: [PATCH 2/6] changelog --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index b88b27320a..422356f015 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -9,5 +9,6 @@ ### Dependency updates ### Bundles +* Fix processing short pip flags in environment dependencies ([#3708](https://github.com/databricks/cli/pull/3708)) ### API Changes From f62f9e351956ca902e2e341ddcf32cb3f0815c81 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 2 Oct 2025 15:25:34 +0200 Subject: [PATCH 3/6] handle local path --- bundle/config/mutator/normalize_paths.go | 27 ++++++++++++------- bundle/config/mutator/normalize_paths_test.go | 8 ++++++ bundle/libraries/local_path.go | 12 +++------ 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/bundle/config/mutator/normalize_paths.go b/bundle/config/mutator/normalize_paths.go index 992151f65d..f12d6dfd07 100644 --- a/bundle/config/mutator/normalize_paths.go +++ b/bundle/config/mutator/normalize_paths.go @@ -84,19 +84,26 @@ func collectGitSourcePaths(b *bundle.Bundle) []dyn.Path { return jobs } +var flagsWithLocalPath = []string{ + "-r", + "-e", +} + func normalizePath(path string, location dyn.Location, bundleRootPath string) (string, error) { // Handle requirements file paths with -r flag - reqPath, ok := strings.CutPrefix(path, "-r ") - if ok { - // Normalize the path part - reqPath = strings.TrimSpace(reqPath) - normalizedPath, err := normalizePath(reqPath, location, bundleRootPath) - if err != nil { - return "", err - } + for _, flag := range flagsWithLocalPath { + reqPath, ok := strings.CutPrefix(path, flag+" ") + if ok { + // Normalize the path part + reqPath = strings.TrimSpace(reqPath) + normalizedPath, err := normalizePath(reqPath, location, bundleRootPath) + if err != nil { + return "", err + } - // Reconstruct the path with -r flag - return "-r " + normalizedPath, nil + // Reconstruct the path with -r flag + return flag + " " + normalizedPath, nil + } } pathAsUrl, err := url.Parse(path) diff --git a/bundle/config/mutator/normalize_paths_test.go b/bundle/config/mutator/normalize_paths_test.go index ccfc747cfc..75f2723dd4 100644 --- a/bundle/config/mutator/normalize_paths_test.go +++ b/bundle/config/mutator/normalize_paths_test.go @@ -78,6 +78,14 @@ func TestNormalizePath_requirementsFile(t *testing.T) { assert.Equal(t, "-r requirements.txt", value) } +func TestNormalizePath_environmentDependency(t *testing.T) { + tmpDir := t.TempDir() + location := dyn.Location{File: filepath.Join(tmpDir, "resources", "job_1.yml")} + value, err := normalizePath("-e ../file.py", location, tmpDir) + assert.NoError(t, err) + assert.Equal(t, "-e file.py", value) +} + func TestLocationDirectory(t *testing.T) { loc := dyn.Location{File: "file", Line: 1, Column: 2} dir, err := locationDirectory(loc) diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go index 3bf8942723..d7187ce0f7 100644 --- a/bundle/libraries/local_path.go +++ b/bundle/libraries/local_path.go @@ -58,17 +58,11 @@ func IsLibraryLocal(dep string) bool { } } - // If the dependency starts with --, it's a pip flag option which is a valid - // entry for environment dependencies but not a local path - if containsPipFlag(dep) { - return false - } - - // If the dependency is a requirements file, it can either be a local path or a remote path. - // Even though the path to requirements.txt can be local we don't return true in this function anyway + // If the dependency starts with - or --, it's a pip flag option which is a valid + // entry for environment dependencies. Even though the path in the flag can be local we don't return true in this function anyway // and don't treat such path as a local library path. // Instead we handle translation of these paths in a separate code path in TranslatePath mutator. - if strings.HasPrefix(dep, "-r") { + if containsPipFlag(dep) { return false } From 5df9ec5b7e2f219d96505ce9bc89e007278cc3a6 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 2 Oct 2025 16:12:12 +0200 Subject: [PATCH 4/6] local to remote translation for pip flags --- .../environments/dependencies/databricks.yml | 3 +++ .../bundle/environments/dependencies/file.py | 0 .../environments/dependencies/output.txt | 6 ++++++ bundle/config/mutator/normalize_paths.go | 10 +++------ .../paths/job_libraries_paths_visitor.go | 4 ++-- .../config/mutator/paths/translation_mode.go | 4 ++-- bundle/config/mutator/translate_paths.go | 10 +++++---- bundle/libraries/local_path.go | 21 ++++++++++++------- bundle/libraries/local_path_test.go | 4 +++- 9 files changed, 39 insertions(+), 23 deletions(-) create mode 100644 acceptance/bundle/environments/dependencies/file.py diff --git a/acceptance/bundle/environments/dependencies/databricks.yml b/acceptance/bundle/environments/dependencies/databricks.yml index 9974a039a7..3689ad0a7c 100644 --- a/acceptance/bundle/environments/dependencies/databricks.yml +++ b/acceptance/bundle/environments/dependencies/databricks.yml @@ -21,6 +21,9 @@ resources: client: "1" dependencies: - "-r ./requirements.txt" + - "-e ./file.py" + - "-i http://myindexurl.com" + - "--index-url http://myindexurl.com" - "test_package" - "test_package==2.0.1" - "test_package>=2.0.1" diff --git a/acceptance/bundle/environments/dependencies/file.py b/acceptance/bundle/environments/dependencies/file.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acceptance/bundle/environments/dependencies/output.txt b/acceptance/bundle/environments/dependencies/output.txt index 60a1f1933e..bf65e8b460 100644 --- a/acceptance/bundle/environments/dependencies/output.txt +++ b/acceptance/bundle/environments/dependencies/output.txt @@ -23,6 +23,9 @@ Deployment complete! "client": "1", "dependencies": [ "-r /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/requirements.txt", + "-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/file.py", + "-i http://myindexurl.com", + "--index-url http://myindexurl.com", "test_package", "test_package==2.0.1", "test_package>=2.0.1", @@ -79,6 +82,9 @@ Deployment complete! "client": "1", "dependencies": [ "-r /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/requirements.txt", + "-e /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/file.py", + "-i http://myindexurl.com", + "--index-url http://myindexurl.com", "test_package", "test_package==2.0.1", "test_package>=2.0.1", diff --git a/bundle/config/mutator/normalize_paths.go b/bundle/config/mutator/normalize_paths.go index f12d6dfd07..716da0bb0e 100644 --- a/bundle/config/mutator/normalize_paths.go +++ b/bundle/config/mutator/normalize_paths.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator/paths" + "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" ) @@ -84,14 +85,9 @@ func collectGitSourcePaths(b *bundle.Bundle) []dyn.Path { return jobs } -var flagsWithLocalPath = []string{ - "-r", - "-e", -} - func normalizePath(path string, location dyn.Location, bundleRootPath string) (string, error) { - // Handle requirements file paths with -r flag - for _, flag := range flagsWithLocalPath { + // Handle local file paths used inside pip flags + for _, flag := range libraries.PipFlagsWithLocalPaths { reqPath, ok := strings.CutPrefix(path, flag+" ") if ok { // Normalize the path part diff --git a/bundle/config/mutator/paths/job_libraries_paths_visitor.go b/bundle/config/mutator/paths/job_libraries_paths_visitor.go index f4e3c9e6df..ad56b1933d 100644 --- a/bundle/config/mutator/paths/job_libraries_paths_visitor.go +++ b/bundle/config/mutator/paths/job_libraries_paths_visitor.go @@ -62,9 +62,9 @@ func jobLibrariesRewritePatterns() []jobRewritePattern { dyn.Key("dependencies"), dyn.AnyIndex(), ), - TranslateModeEnvironmentRequirements, + TranslateModeEnvironmentPipFlag, func(s string) bool { - _, ok := libraries.IsLocalRequirementsFile(s) + _, _, ok := libraries.IsLocalPathInPipFlag(s) return !ok }, }, diff --git a/bundle/config/mutator/paths/translation_mode.go b/bundle/config/mutator/paths/translation_mode.go index 7382f5a80c..0262524bfb 100644 --- a/bundle/config/mutator/paths/translation_mode.go +++ b/bundle/config/mutator/paths/translation_mode.go @@ -30,6 +30,6 @@ const ( // This allows for disambiguating between paths and PyPI package names. TranslateModeLocalRelativeWithPrefix - // TranslateModeEnvironmentRequirements translates a local requirements file path to be absolute. - TranslateModeEnvironmentRequirements + // TranslateModeEnvironmentPipFlag translates a local file path in a pip flag to be absolute. + TranslateModeEnvironmentPipFlag ) diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 41162b44d3..089d71f938 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -104,8 +104,10 @@ func (t *translateContext) rewritePath( opts translateOptions, ) (string, error) { // If the input is a local requirements file, we need to translate it to an absolute path. - if reqPath, ok := libraries.IsLocalRequirementsFile(input); ok { + var flag string + if reqPath, flagPrefix, ok := libraries.IsLocalPathInPipFlag(input); ok { input = reqPath + flag = flagPrefix } // We assume absolute paths point to a location in the workspace @@ -160,10 +162,10 @@ func (t *translateContext) rewritePath( interp, err = t.translateLocalRelativePath(ctx, input, localPath, localRelPath) case paths.TranslateModeLocalRelativeWithPrefix: interp, err = t.translateLocalRelativeWithPrefixPath(ctx, input, localPath, localRelPath) - case paths.TranslateModeEnvironmentRequirements: + case paths.TranslateModeEnvironmentPipFlag: interp, err = t.translateFilePath(ctx, input, localPath, localRelPath) - // Add the -r flag to the path to indicate it's a requirements file used for environment dependencies. - interp = "-r " + interp + // Add the flag prefix to the path to indicate it's a local file used in pip flags for environment dependencies. + interp = flag + " " + interp default: return "", fmt.Errorf("unsupported translate mode: %d", opts.Mode) } diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go index d7187ce0f7..2463292ae5 100644 --- a/bundle/libraries/local_path.go +++ b/bundle/libraries/local_path.go @@ -74,18 +74,25 @@ func IsLibraryLocal(dep string) bool { return IsLocalPath(dep) } -func IsLocalRequirementsFile(dep string) (string, bool) { - dep, ok := strings.CutPrefix(dep, "-r ") - if !ok { - return "", false +var PipFlagsWithLocalPaths = []string{ + "-r", + "-e", +} + +func IsLocalPathInPipFlag(dep string) (string, string, bool) { + for _, flag := range PipFlagsWithLocalPaths { + dep, ok := strings.CutPrefix(dep, flag+" ") + if ok { + dep = strings.TrimSpace(dep) + return dep, flag, IsLocalPath(dep) + } } - dep = strings.TrimSpace(dep) - return dep, IsLocalPath(dep) + return "", "", false } func containsPipFlag(input string) bool { - re := regexp.MustCompile(`--?[a-zA-Z0-9-]+`) + re := regexp.MustCompile(`^\s*--?[a-zA-Z0-9-]+`) return re.MatchString(input) } diff --git a/bundle/libraries/local_path_test.go b/bundle/libraries/local_path_test.go index a054a47b1a..b8591e11a0 100644 --- a/bundle/libraries/local_path_test.go +++ b/bundle/libraries/local_path_test.go @@ -58,6 +58,8 @@ func TestIsLibraryLocal(t *testing.T) { {path: "dbfs:/mnt/path/to/package", expected: false}, {path: "beautifulsoup4", expected: false}, {path: "-e some/local/path", expected: false}, + {path: "-i http://myindexurl.com", expected: false}, + {path: "--index-url http://myindexurl.com", expected: false}, // Check the possible version specifiers as in PEP 440 // https://peps.python.org/pep-0440/#public-version-identifiers @@ -130,7 +132,7 @@ func TestIsLocalRequirementsFile(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, isLocal := IsLocalRequirementsFile(tt.input) + got, _, isLocal := IsLocalPathInPipFlag(tt.input) require.Equal(t, tt.expected, got) require.Equal(t, tt.isLocal, isLocal) }) From fe2386927131e867a6fc251534c74fce7f298f38 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 2 Oct 2025 16:50:07 +0200 Subject: [PATCH 5/6] fix --- bundle/libraries/local_path.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go index 2463292ae5..33cdcc9900 100644 --- a/bundle/libraries/local_path.go +++ b/bundle/libraries/local_path.go @@ -92,7 +92,7 @@ func IsLocalPathInPipFlag(dep string) (string, string, bool) { } func containsPipFlag(input string) bool { - re := regexp.MustCompile(`^\s*--?[a-zA-Z0-9-]+`) + re := regexp.MustCompile(`--?[a-zA-Z0-9-]+\s`) return re.MatchString(input) } From fa7327c97463619a5f87aa755d644b2b3a1d9866 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 3 Oct 2025 14:47:00 +0200 Subject: [PATCH 6/6] fixes --- NEXT_CHANGELOG.md | 1 + bundle/libraries/local_path.go | 5 ++++- bundle/libraries/local_path_test.go | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 422356f015..64fde5e88e 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -10,5 +10,6 @@ ### Bundles * Fix processing short pip flags in environment dependencies ([#3708](https://github.com/databricks/cli/pull/3708)) +* Add support for referencing local files in -e pip flag for environment dependencies ([#3708](https://github.com/databricks/cli/pull/3708)) ### API Changes diff --git a/bundle/libraries/local_path.go b/bundle/libraries/local_path.go index 33cdcc9900..6b07c70bb8 100644 --- a/bundle/libraries/local_path.go +++ b/bundle/libraries/local_path.go @@ -92,7 +92,10 @@ func IsLocalPathInPipFlag(dep string) (string, string, bool) { } func containsPipFlag(input string) bool { - re := regexp.MustCompile(`--?[a-zA-Z0-9-]+\s`) + // Trailing space means the the flag takes an argument or there's multiple arguments in input + // Alternatively it could be a flag with no argument and no space after it + // For example: -r myfile.txt or --index-url http://myindexurl.com or -i + re := regexp.MustCompile(`--?[a-zA-Z0-9-]+(\s|$)`) return re.MatchString(input) } diff --git a/bundle/libraries/local_path_test.go b/bundle/libraries/local_path_test.go index b8591e11a0..f97e911fc4 100644 --- a/bundle/libraries/local_path_test.go +++ b/bundle/libraries/local_path_test.go @@ -60,6 +60,9 @@ func TestIsLibraryLocal(t *testing.T) { {path: "-e some/local/path", expected: false}, {path: "-i http://myindexurl.com", expected: false}, {path: "--index-url http://myindexurl.com", expected: false}, + {path: "-i", expected: false}, + {path: "--index-url", expected: false}, + {path: "-i -e", expected: false}, // Check the possible version specifiers as in PEP 440 // https://peps.python.org/pep-0440/#public-version-identifiers