Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support nesting dependency groups #837

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 46 additions & 1 deletion src/poetry/core/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -347,9 +347,23 @@ def _configure_package_dependencies(
cls._add_package_group_dependencies(
package=package,
group=group,
dependencies=group_config["dependencies"],
dependencies=group_config.get("dependencies", {}),
)

for group_name, group_config in tool_poetry["group"].items():
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a reason to iterate a second time over tool_poetry["group"].items() and not just adding included groups and dependencies in one iteration?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can only include a dependency group into another one if I have an instance of this dependency group.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Makes sense. In that case, it is just unlucky that this is not covered by the tests. I suppose adding a variant of test_create_poetry_with_nested_dependency_groups with swapped dependency groups will cover this case.

if include_groups := group_config.get("include-groups", []):
current_group = package.dependency_group(group_name)
for name in include_groups:
try:
group_to_include = package.dependency_group(name)
except ValueError as e:
raise ValueError(
f"Group '{group_name}' includes group '{name}'"
" which is not defined."
) from e

current_group.include_dependency_group(group_to_include)

if with_groups and "dev-dependencies" in tool_poetry:
cls._add_package_group_dependencies(
package=package,
Expand Down Expand Up @@ -614,6 +628,8 @@ def validate(
' Use "poetry.group.dev.dependencies" instead.'
)

cls._validate_dependency_groups_includes(toml_data, result)

if strict:
# Validate relation between [project] and [tool.poetry]
cls._validate_legacy_vs_project(toml_data, result)
Expand All @@ -622,6 +638,35 @@ def validate(

return result

@classmethod
def _validate_dependency_groups_includes(
cls, toml_data: dict[str, Any], result: dict[str, list[str]]
) -> None:
"""Ensure that dependency groups do not include themselves."""
config = toml_data.setdefault("tool", {}).setdefault("poetry", {})

group_includes = {}
for group_name, group_config in config.get("group", {}).items():
if include_groups := group_config.get("include-groups", []):
group_includes[canonicalize_name(group_name)] = [
canonicalize_name(name) for name in include_groups
]

for group_name in group_includes:
visited = set()
stack = [group_name]
while stack:
group = stack.pop()
visited.add(group)
for include in group_includes.get(group, []):
if include in visited:
result["errors"].append(
f"Cyclic dependency group include in {group_name}:"
f" {group} -> {include}"
)
continue
stack.append(include)

@classmethod
def _validate_legacy_vs_project(
cls, toml_data: dict[str, Any], result: dict[str, list[str]]
Expand Down
20 changes: 18 additions & 2 deletions src/poetry/core/json/schemas/poetry-schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -171,14 +171,30 @@
"^[a-zA-Z-_.0-9]+$": {
"type": "object",
"description": "This represents a single dependency group",
"required": [
"dependencies"
"anyOf": [
{
"required": [
"dependencies"
]
},
{
"required": [
"include-groups"
]
}
],
"properties": {
"optional": {
"type": "boolean",
"description": "Whether the dependency group is optional or not"
},
"include-groups": {
"type": "array",
"description": "List of dependency group names included in this group.",
"items": {
"type": "string"
}
},
"dependencies": {
"type": "object",
"description": "The dependencies of this dependency group",
Expand Down
11 changes: 6 additions & 5 deletions src/poetry/core/packages/dependency.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ def __init__(
if not groups:
groups = [MAIN_GROUP]

self._groups = frozenset(groups)
self.groups = frozenset(groups)
self._allows_prereleases = allows_prereleases
# "_develop" is only required for enriching [project] dependencies
self._develop = False
Expand Down Expand Up @@ -115,10 +115,6 @@ def pretty_constraint(self) -> str:
def pretty_name(self) -> str:
return self._pretty_name

@property
def groups(self) -> frozenset[str]:
return self._groups

@property
def python_versions(self) -> str:
return self._python_versions
Expand Down Expand Up @@ -329,6 +325,11 @@ def with_constraint(self: T, constraint: str | VersionConstraint) -> T:
dependency.constraint = constraint # type: ignore[assignment]
return dependency

def with_groups(self, groups: Iterable[str]) -> Dependency:
dependency = self.clone()
dependency.groups = frozenset(groups)
return dependency

@classmethod
def create_from_pep_508(
cls, name: str, relative_to: Path | None = None
Expand Down
110 changes: 73 additions & 37 deletions src/poetry/core/packages/dependency_group.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
from collections import defaultdict
from typing import TYPE_CHECKING

from packaging.utils import NormalizedName
from packaging.utils import canonicalize_name

from poetry.core.version.markers import parse_marker


Expand All @@ -22,66 +25,87 @@ def __init__(
self._mixed_dynamic = mixed_dynamic
self._dependencies: list[Dependency] = []
self._poetry_dependencies: list[Dependency] = []
self._included_dependency_groups: dict[NormalizedName, DependencyGroup] = {}

@property
def name(self) -> str:
return self._name

@property
def dependencies(self) -> list[Dependency]:
if not self._dependencies:
group_dependencies = self._dependencies
included_group_dependencies = self._resolve_included_dependency_groups()

if not group_dependencies:
# legacy mode
return self._poetry_dependencies
if self._mixed_dynamic and self._poetry_dependencies:
group_dependencies = self._poetry_dependencies
elif self._mixed_dynamic and self._poetry_dependencies:
if all(dep.is_optional() for dep in self._dependencies):
return [
group_dependencies = [
*self._dependencies,
*(d for d in self._poetry_dependencies if not d.is_optional()),
]
if all(not dep.is_optional() for dep in self._dependencies):
return [
elif all(not dep.is_optional() for dep in self._dependencies):
group_dependencies = [
*self._dependencies,
*(d for d in self._poetry_dependencies if d.is_optional()),
]
return self._dependencies

return group_dependencies + included_group_dependencies

@property
def dependencies_for_locking(self) -> list[Dependency]:
if not self._poetry_dependencies:
return self._dependencies
if not self._dependencies:
return self._poetry_dependencies
included_group_dependencies = self._resolve_included_dependency_groups()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might make no difference at the moment, but I think this is wrong because _resolve_included_dependency_groups uses dependencies instead of dependencies_for_locking. I think we need two versions of _resolve_included_dependency_groups (or pass a parameter that tells which dependencies to use).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually I guess I need more information about what's the difference between dependencies and dependencies_for_locking 😅

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess you do not need to know the difference to consider it wrong: It feels clear to me that nesting groups should not violate the distinction between the two.

Nevertheless: dependencies are used for metadata (when building), dependencies_for_locking are used for locking. This distinction is probably not that relevant for groups because we only need dependencies_for_locking and do not use dependencies but that does not mean that we should not care about implementing it correctly.


poetry_dependencies_by_name = defaultdict(list)
for dep in self._poetry_dependencies:
poetry_dependencies_by_name[dep.name].append(dep)

dependencies = []
for dep in self.dependencies:
if dep.name in poetry_dependencies_by_name:
enriched = False
dep_marker = dep.marker
if dep.in_extras:
dep_marker = dep.marker.intersect(
parse_marker(
" or ".join(
f"extra == '{extra}'" for extra in dep.in_extras
if not self._poetry_dependencies:
dependencies = self._dependencies
elif not self._dependencies:
dependencies = self._poetry_dependencies
else:
poetry_dependencies_by_name = defaultdict(list)
for dep in self._poetry_dependencies:
poetry_dependencies_by_name[dep.name].append(dep)

dependencies = []
for dep in self.dependencies:
if dep.name in poetry_dependencies_by_name:
enriched = False
dep_marker = dep.marker
if dep.in_extras:
dep_marker = dep.marker.intersect(
parse_marker(
" or ".join(
f"extra == '{extra}'" for extra in dep.in_extras
)
)
)
)
for poetry_dep in poetry_dependencies_by_name[dep.name]:
marker = dep_marker.intersect(poetry_dep.marker)
if not marker.is_empty():
if marker == dep_marker:
marker = dep.marker
enriched = True
dependencies.append(_enrich_dependency(dep, poetry_dep, marker))
if not enriched:
for poetry_dep in poetry_dependencies_by_name[dep.name]:
marker = dep_marker.intersect(poetry_dep.marker)
if not marker.is_empty():
if marker == dep_marker:
marker = dep.marker
enriched = True
dependencies.append(
_enrich_dependency(dep, poetry_dep, marker)
)
if not enriched:
dependencies.append(dep)
else:
dependencies.append(dep)
else:
dependencies.append(dep)

return dependencies
return dependencies + included_group_dependencies

def _resolve_included_dependency_groups(self) -> list[Dependency]:
"""Resolves and returns the dependencies from included dependency groups.

This method iterates over all included dependency groups and collects
their dependencies, associating them with the current group.
"""
return [
dependency.with_groups([self.name])
for dependency_group in self._included_dependency_groups.values()
for dependency in dependency_group.dependencies
]

def is_optional(self) -> bool:
return self._optional
Expand Down Expand Up @@ -114,6 +138,18 @@ def remove_dependency(self, name: str) -> None:
dependencies.append(dependency)
self._poetry_dependencies = dependencies

def include_dependency_group(self, dependency_group: DependencyGroup) -> None:
group_name = canonicalize_name(dependency_group.name)

if group_name == canonicalize_name(self.name):
raise ValueError("Cannot include the dependency group to itself.")
if group_name in self._included_dependency_groups:
raise ValueError(
f"Dependency group {dependency_group.name} is already included"
)

self._included_dependency_groups[group_name] = dependency_group

def __eq__(self, other: object) -> bool:
if not isinstance(other, DependencyGroup):
return NotImplemented
Expand Down
62 changes: 62 additions & 0 deletions tests/packages/test_dependency_group.py
Original file line number Diff line number Diff line change
Expand Up @@ -491,3 +491,65 @@ def test_dependencies_for_locking_failure(

with pytest.raises(ValueError):
_ = group.dependencies_for_locking


def test_include_dependency_groups() -> None:
group = DependencyGroup(name="group")
group.add_poetry_dependency(
Dependency(name="foo", constraint="*", groups=["group"])
)

group_2 = DependencyGroup(name="group2")
group_2.add_poetry_dependency(
Dependency(name="bar", constraint="*", groups=["group2"])
)

group.include_dependency_group(group_2)

assert [dep.name for dep in group.dependencies] == ["foo", "bar"]
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

suggestion (testing): Test edge case where included group is empty.

Add a test case where group_2 is empty to ensure correct behavior when including an empty group.

Suggested implementation:

# Existing tests above...
def test_include_empty_dependency_group() -> None:
    group_main = DependencyGroup(name="group")
    group_main.add_poetry_dependency(
        Dependency(name="foo", constraint="*", groups=["group"])
    )

    empty_group = DependencyGroup(name="empty")
    # Include empty dependency group
    group_main.include_dependency_group(empty_group)

    # Assert that including an empty group does not affect group_main's dependencies.
    assert [dep.name for dep in group_main.dependencies] == ["foo"]
    assert [dep.name for dep in group_main.dependencies_for_locking] == ["foo"]

assert [dep.name for dep in group.dependencies_for_locking] == ["foo", "bar"]
for dep in group.dependencies:
assert dep.groups == {"group"}

assert [dep.name for dep in group_2.dependencies] == ["bar"]
assert [dep.name for dep in group_2.dependencies_for_locking] == ["bar"]
for dep in group_2.dependencies:
assert dep.groups == {"group2"}


@pytest.mark.parametrize("group_name", ["group_2", "Group-2", "group-2"])
def test_include_dependency_group_raise_if_including_itself(group_name: str) -> None:
group = DependencyGroup(name="group-2")
group.add_poetry_dependency(
Dependency(name="foo", constraint="*", groups=["group"])
)

with pytest.raises(
ValueError, match="Cannot include the dependency group to itself"
):
group.include_dependency_group(DependencyGroup(name=group_name))


@pytest.mark.parametrize("group_name", ["group_2", "Group-2", "group-2"])
def test_include_dependency_group_raise_if_already_included(group_name: str) -> None:
group = DependencyGroup(name="group")
group.add_poetry_dependency(
Dependency(name="foo", constraint="*", groups=["group"])
)

group_2 = DependencyGroup(name="group_2")
group_2.add_poetry_dependency(
Dependency(name="bar", constraint="*", groups=["group2"])
)

group.include_dependency_group(group_2)

group_3 = DependencyGroup(name=group_name)
group_3.add_poetry_dependency(
Dependency(name="bar", constraint="*", groups=["group2"])
)

with pytest.raises(
ValueError, match=f"Dependency group {group_name} is already included"
):
group.include_dependency_group(group_3)
Loading