Skip to content

Commit 0d0d59a

Browse files
committed
Support reading dependency-groups from pyproject.tomls with no project
(or legacy tool.uv.workspace). This cleaves out a dedicated SourcedDependencyGroups type based on RequiresDist but with only the DependencyGroup handling implemented. This allows `uv pip` to read `dependency-groups` from pyproject.tomls that only have that table defined, per PEP 735, and as implemented by `pip`. However we want our implementation to respect various uv features when they're available: * `tool.uv.sources` * `tool.uv.index` * `tool.uv.dependency-groups.mygroup.requires-python` (#13735) As such we want to opportunistically detect "as much as possible" while doing as little as possible when things are missing. The issue with the old RequiresDist path was that it fundamentally wanted to build the package, and if `[project]` was missing it would try to desperately run setuptools on the pyproject.toml to try to find metadata and make a hash of things. At the same time, the old code also put in a lot of effort to try to pretend that `uv pip` dependency-groups worked like `uv` dependency-groups with defaults and non-only semantics, only to separate them back out again. By explicitly separating them out, we confidently get the expected behaviour. Note that dependency-group support is still included in RequiresDist, as some `uv` paths still use it. It's unclear to me if those paths want this same treatment -- for now I conclude no. Fixes #13138
1 parent 4cc5291 commit 0d0d59a

10 files changed

Lines changed: 360 additions & 86 deletions

File tree

crates/uv-distribution/src/lib.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ pub use error::Error;
44
pub use index::{BuiltWheelIndex, RegistryWheelIndex};
55
pub use metadata::{
66
ArchiveMetadata, BuildRequires, FlatRequiresDist, LoweredRequirement, LoweringError, Metadata,
7-
MetadataError, RequiresDist,
7+
MetadataError, RequiresDist, SourcedDependencyGroups,
88
};
99
pub use reporter::Reporter;
1010
pub use source::prune;
Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
use std::collections::BTreeMap;
2+
use std::path::Path;
3+
4+
use uv_configuration::SourceStrategy;
5+
use uv_distribution_types::{IndexLocations, Requirement};
6+
use uv_normalize::{DEV_DEPENDENCIES, GroupName, PackageName};
7+
use uv_workspace::dependency_groups::FlatDependencyGroups;
8+
use uv_workspace::pyproject::{Sources, ToolUvSources};
9+
use uv_workspace::{DiscoveryOptions, MemberDiscovery, VirtualProject, WorkspaceCache};
10+
11+
use crate::metadata::{GitWorkspaceMember, LoweredRequirement, MetadataError};
12+
13+
#[derive(Debug, Clone)]
14+
pub struct SourcedDependencyGroups {
15+
pub name: Option<PackageName>,
16+
pub dependency_groups: BTreeMap<GroupName, Box<[Requirement]>>,
17+
}
18+
19+
impl SourcedDependencyGroups {
20+
/// Lower by considering `tool.uv` in `pyproject.toml` if present, used for Git and directory
21+
/// dependencies.
22+
pub async fn from_virtual_project(
23+
pyproject_path: &Path,
24+
git_member: Option<&GitWorkspaceMember<'_>>,
25+
locations: &IndexLocations,
26+
source_strategy: SourceStrategy,
27+
cache: &WorkspaceCache,
28+
) -> Result<Self, MetadataError> {
29+
let discovery = DiscoveryOptions {
30+
stop_discovery_at: git_member.map(|git_member| {
31+
git_member
32+
.fetch_root
33+
.parent()
34+
.expect("git checkout has a parent")
35+
.to_path_buf()
36+
}),
37+
members: match source_strategy {
38+
SourceStrategy::Enabled => MemberDiscovery::default(),
39+
SourceStrategy::Disabled => MemberDiscovery::None,
40+
},
41+
};
42+
let project = VirtualProject::discover_defaulted(pyproject_path, &discovery, cache).await?;
43+
44+
// Collect any `tool.uv.index` entries.
45+
let empty = vec![];
46+
let project_indexes = match source_strategy {
47+
SourceStrategy::Enabled => project
48+
.pyproject_toml()
49+
.tool
50+
.as_ref()
51+
.and_then(|tool| tool.uv.as_ref())
52+
.and_then(|uv| uv.index.as_deref())
53+
.unwrap_or(&empty),
54+
SourceStrategy::Disabled => &empty,
55+
};
56+
57+
// Collect any `tool.uv.sources` and `tool.uv.dev_dependencies` from `pyproject.toml`.
58+
let empty = BTreeMap::default();
59+
let project_sources = match source_strategy {
60+
SourceStrategy::Enabled => project
61+
.pyproject_toml()
62+
.tool
63+
.as_ref()
64+
.and_then(|tool| tool.uv.as_ref())
65+
.and_then(|uv| uv.sources.as_ref())
66+
.map(ToolUvSources::inner)
67+
.unwrap_or(&empty),
68+
SourceStrategy::Disabled => &empty,
69+
};
70+
71+
// Collect the dependency groups.
72+
let dependency_groups = {
73+
// First, collect `tool.uv.dev_dependencies`
74+
let dev_dependencies = project
75+
.pyproject_toml()
76+
.tool
77+
.as_ref()
78+
.and_then(|tool| tool.uv.as_ref())
79+
.and_then(|uv| uv.dev_dependencies.as_ref());
80+
81+
// Then, collect `dependency-groups`
82+
let dependency_groups = project
83+
.pyproject_toml()
84+
.dependency_groups
85+
.iter()
86+
.flatten()
87+
.collect::<BTreeMap<_, _>>();
88+
89+
// Flatten the dependency groups.
90+
let mut dependency_groups =
91+
FlatDependencyGroups::from_dependency_groups(&dependency_groups)
92+
.map_err(|err| err.with_dev_dependencies(dev_dependencies))?;
93+
94+
// Add the `dev` group, if `dev-dependencies` is defined.
95+
if let Some(dev_dependencies) = dev_dependencies {
96+
dependency_groups
97+
.entry(DEV_DEPENDENCIES.clone())
98+
.or_insert_with(Vec::new)
99+
.extend(dev_dependencies.clone());
100+
}
101+
102+
dependency_groups
103+
};
104+
105+
// Now that we've resolved the dependency groups, we can validate that each source references
106+
// a valid extra or group, if present.
107+
Self::validate_sources(project_sources, &dependency_groups)?;
108+
109+
// Lower the dependency groups.
110+
let dependency_groups = dependency_groups
111+
.into_iter()
112+
.map(|(name, requirements)| {
113+
let requirements = match source_strategy {
114+
SourceStrategy::Enabled => requirements
115+
.into_iter()
116+
.flat_map(|requirement| {
117+
let requirement_name = requirement.name.clone();
118+
let group = name.clone();
119+
let extra = None;
120+
LoweredRequirement::from_requirement(
121+
requirement,
122+
project.project_name(),
123+
project.root(),
124+
project_sources,
125+
project_indexes,
126+
extra,
127+
Some(&group),
128+
locations,
129+
project.workspace(),
130+
git_member,
131+
)
132+
.map(
133+
move |requirement| match requirement {
134+
Ok(requirement) => Ok(requirement.into_inner()),
135+
Err(err) => Err(MetadataError::GroupLoweringError(
136+
group.clone(),
137+
requirement_name.clone(),
138+
Box::new(err),
139+
)),
140+
},
141+
)
142+
})
143+
.collect::<Result<Box<_>, _>>(),
144+
SourceStrategy::Disabled => {
145+
Ok(requirements.into_iter().map(Requirement::from).collect())
146+
}
147+
}?;
148+
Ok::<(GroupName, Box<_>), MetadataError>((name, requirements))
149+
})
150+
.collect::<Result<BTreeMap<_, _>, _>>()?;
151+
152+
Ok(Self {
153+
name: project.project_name().cloned(),
154+
dependency_groups,
155+
})
156+
}
157+
158+
/// Validate the sources.
159+
///
160+
/// If a source is requested with `group`, ensure that the relevant dependency is
161+
/// present in the relevant `dependency-groups` section.
162+
fn validate_sources(
163+
sources: &BTreeMap<PackageName, Sources>,
164+
dependency_groups: &FlatDependencyGroups,
165+
) -> Result<(), MetadataError> {
166+
for (name, sources) in sources {
167+
for source in sources.iter() {
168+
if let Some(group) = source.group() {
169+
// If the group doesn't exist at all, error.
170+
let Some(dependencies) = dependency_groups.get(group) else {
171+
return Err(MetadataError::MissingSourceGroup(
172+
name.clone(),
173+
group.clone(),
174+
));
175+
};
176+
177+
// If there is no such requirement with the group, error.
178+
if !dependencies
179+
.iter()
180+
.any(|requirement| requirement.name == *name)
181+
{
182+
return Err(MetadataError::IncompleteSourceGroup(
183+
name.clone(),
184+
group.clone(),
185+
));
186+
}
187+
}
188+
}
189+
}
190+
191+
Ok(())
192+
}
193+
}

crates/uv-distribution/src/metadata/mod.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,13 @@ use uv_workspace::dependency_groups::DependencyGroupError;
1212
use uv_workspace::{WorkspaceCache, WorkspaceError};
1313

1414
pub use crate::metadata::build_requires::BuildRequires;
15+
pub use crate::metadata::dependency_groups::SourcedDependencyGroups;
1516
pub use crate::metadata::lowering::LoweredRequirement;
1617
pub use crate::metadata::lowering::LoweringError;
1718
pub use crate::metadata::requires_dist::{FlatRequiresDist, RequiresDist};
1819

1920
mod build_requires;
21+
mod dependency_groups;
2022
mod lowering;
2123
mod requires_dist;
2224

crates/uv-requirements/src/source_tree.rs

Lines changed: 12 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
1-
use std::path::{Path, PathBuf};
1+
use std::borrow::Cow;
2+
use std::path::Path;
23
use std::sync::Arc;
3-
use std::{borrow::Cow, collections::BTreeMap};
44

55
use anyhow::{Context, Result};
66
use futures::TryStreamExt;
77
use futures::stream::FuturesOrdered;
88
use url::Url;
99

10-
use uv_configuration::{DependencyGroups, ExtrasSpecification};
10+
use uv_configuration::ExtrasSpecification;
1111
use uv_distribution::{DistributionDatabase, FlatRequiresDist, Reporter, RequiresDist};
1212
use uv_distribution_types::Requirement;
1313
use uv_distribution_types::{
@@ -37,8 +37,6 @@ pub struct SourceTreeResolution {
3737
pub struct SourceTreeResolver<'a, Context: BuildContext> {
3838
/// The extras to include when resolving requirements.
3939
extras: &'a ExtrasSpecification,
40-
/// The groups to include when resolving requirements.
41-
groups: &'a BTreeMap<PathBuf, DependencyGroups>,
4240
/// The hash policy to enforce.
4341
hasher: &'a HashStrategy,
4442
/// The in-memory index for resolving dependencies.
@@ -51,14 +49,12 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> {
5149
/// Instantiate a new [`SourceTreeResolver`] for a given set of `source_trees`.
5250
pub fn new(
5351
extras: &'a ExtrasSpecification,
54-
groups: &'a BTreeMap<PathBuf, DependencyGroups>,
5552
hasher: &'a HashStrategy,
5653
index: &'a InMemoryIndex,
5754
database: DistributionDatabase<'a, Context>,
5855
) -> Self {
5956
Self {
6057
extras,
61-
groups,
6258
hasher,
6359
index,
6460
database,
@@ -101,46 +97,17 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> {
10197

10298
let mut requirements = Vec::new();
10399

104-
// Resolve any groups associated with this path
105-
let default_groups = DependencyGroups::default();
106-
let groups = self.groups.get(path).unwrap_or(&default_groups);
107-
108100
// Flatten any transitive extras and include dependencies
109101
// (unless something like --only-group was passed)
110-
if groups.prod() {
111-
requirements.extend(
112-
FlatRequiresDist::from_requirements(metadata.requires_dist, &metadata.name)
113-
.into_iter()
114-
.map(|requirement| Requirement {
115-
origin: Some(origin.clone()),
116-
marker: requirement.marker.simplify_extras(&extras),
117-
..requirement
118-
}),
119-
);
120-
}
121-
122-
// Apply dependency-groups
123-
for (group_name, group) in &metadata.dependency_groups {
124-
if groups.contains(group_name) {
125-
requirements.extend(group.iter().cloned().map(|group| Requirement {
126-
origin: Some(RequirementOrigin::Group(
127-
path.to_path_buf(),
128-
metadata.name.clone(),
129-
group_name.clone(),
130-
)),
131-
..group
132-
}));
133-
}
134-
}
135-
// Complain if dependency groups are named that don't appear.
136-
for name in groups.explicit_names() {
137-
if !metadata.dependency_groups.contains_key(name) {
138-
return Err(anyhow::anyhow!(
139-
"The dependency group '{name}' was not found in the project: {}",
140-
path.user_display()
141-
));
142-
}
143-
}
102+
requirements.extend(
103+
FlatRequiresDist::from_requirements(metadata.requires_dist, &metadata.name)
104+
.into_iter()
105+
.map(|requirement| Requirement {
106+
origin: Some(origin.clone()),
107+
marker: requirement.marker.simplify_extras(&extras),
108+
..requirement
109+
}),
110+
);
144111

145112
let requirements = requirements.into_boxed_slice();
146113
let project = metadata.name;

crates/uv-requirements/src/specification.rs

Lines changed: 2 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -290,52 +290,18 @@ impl RequirementsSpecification {
290290
if !groups.is_empty() {
291291
let mut group_specs = BTreeMap::new();
292292
for (path, groups) in groups {
293-
// Conceptually pip `--group` flags just add the group referred to by the file.
294-
// In uv semantics this would be like `--only-group`, however if you do this:
295-
//
296-
// uv pip install -r pyproject.toml --group pyproject.toml:foo
297-
//
298-
// We don't want to discard the package listed by `-r` in the way `--only-group`
299-
// would. So we check to see if any other source wants to add this path, and use
300-
// that to determine if we're doing `--group` or `--only-group` semantics.
301-
//
302-
// Note that it's fine if a file gets referred to multiple times by
303-
// different-looking paths (like `./pyproject.toml` vs `pyproject.toml`). We're
304-
// specifically trying to disambiguate in situations where the `--group` *happens*
305-
// to match with an unrelated argument, and `--only-group` would be overzealous!
306-
let source_exists_without_group = requirement_sources
307-
.iter()
308-
.any(|source| source.source_trees.contains(&path));
309-
let (group, only_group) = if source_exists_without_group {
310-
(groups, Vec::new())
311-
} else {
312-
(Vec::new(), groups)
313-
};
314293
let group_spec = DependencyGroups::from_args(
315294
false,
316295
false,
317296
false,
318-
group,
297+
Vec::new(),
319298
Vec::new(),
320299
false,
321-
only_group,
300+
groups,
322301
false,
323302
);
324-
325-
// If we're doing `--only-group` semantics it's because only `--group` flags referred
326-
// to this file, and so we need to make sure to add it to the list of sources!
327-
if !source_exists_without_group {
328-
let source = Self::from_source(
329-
&RequirementsSource::PyprojectToml(path.clone()),
330-
client_builder,
331-
)
332-
.await?;
333-
requirement_sources.push(source);
334-
}
335-
336303
group_specs.insert(path, group_spec);
337304
}
338-
339305
spec.groups = group_specs;
340306
}
341307

crates/uv-workspace/src/pyproject.rs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -653,6 +653,16 @@ impl<'de> serde::de::Deserialize<'de> for ToolUvSources {
653653
}
654654
}
655655

656+
#[derive(Deserialize, Default, Debug, Clone, PartialEq, Eq)]
657+
#[cfg_attr(test, derive(Serialize))]
658+
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
659+
#[serde(rename_all = "kebab-case")]
660+
pub struct DependencyGroupSettings {
661+
/// Version of python to require when installing this group
662+
#[cfg_attr(feature = "schemars", schemars(with = "Option<String>"))]
663+
pub requires_python: Option<VersionSpecifiers>,
664+
}
665+
656666
#[derive(Deserialize, OptionsMetadata, Default, Debug, Clone, PartialEq, Eq)]
657667
#[cfg_attr(test, derive(Serialize))]
658668
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]

0 commit comments

Comments
 (0)