Skip to content

Commit f71beb6

Browse files
cognifloydcognifloyd
authored
Merge pull request #5874: Tell pants about tests that use globs to test with many pack fixtures
2 parents 6adb0f5 + e159a1c commit f71beb6

File tree

15 files changed

+294
-20
lines changed

15 files changed

+294
-20
lines changed

‎CHANGELOG.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ Added
1414
working on StackStorm, improve our security posture, and improve CI reliability thanks in part
1515
to pants' use of PEX lockfiles. This is not a user-facing addition.
1616
#5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850
17-
#5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864
17+
#5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 #5874
1818
Contributed by @cognifloyd
1919

2020
* Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805

‎pants-plugins/pack_metadata/register.py‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,24 @@
1111
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
14-
from pack_metadata import tailor
15-
from pack_metadata.target_types import PackMetadata, PackMetadataInGitSubmodule
14+
from pack_metadata import tailor, target_types_rules
15+
from pack_metadata.target_types import (
16+
PackMetadata,
17+
PackMetadataInGitSubmodule,
18+
PacksGlob,
19+
)
1620

1721

1822
def rules():
19-
return tailor.rules()
23+
return [
24+
*tailor.rules(),
25+
*target_types_rules.rules(),
26+
]
2027

2128

2229
def target_types():
23-
return [PackMetadata, PackMetadataInGitSubmodule]
30+
return [
31+
PackMetadata,
32+
PackMetadataInGitSubmodule,
33+
PacksGlob,
34+
]

‎pants-plugins/pack_metadata/target_types.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from pants.core.target_types import (
1818
ResourcesGeneratingSourcesField,
1919
ResourcesGeneratorTarget,
20+
GenericTarget,
2021
)
2122

2223

@@ -77,3 +78,18 @@ class PackMetadataInGitSubmodule(PackMetadata):
7778
"has unmatched globs. It prints instructions on how to checkout git "
7879
"submodules."
7980
)
81+
82+
83+
class PacksGlobDependencies(Dependencies):
84+
pass
85+
86+
87+
class PacksGlob(GenericTarget):
88+
alias = "packs_glob"
89+
core_fields = (*COMMON_TARGET_FIELDS, PacksGlobDependencies)
90+
help = (
91+
"Packs glob.\n\n"
92+
"Avoid using this target. It gets automatic dependencies on all "
93+
"subdirectories (packs) except those listed with ! in dependencies. "
94+
"This is unfortunately needed by tests that use a glob to load pack fixtures."
95+
)

‎pants-plugins/pack_metadata/target_types_rules.py‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Copyright 2023 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
import os
15+
from dataclasses import dataclass
16+
17+
from pants.engine.addresses import Address
18+
from pants.engine.fs import GlobMatchErrorBehavior, PathGlobs, Paths
19+
from pants.engine.rules import Get, collect_rules, MultiGet, rule, UnionRule
20+
from pants.engine.target import (
21+
DependenciesRequest,
22+
ExplicitlyProvidedDependencies,
23+
FieldSet,
24+
InferDependenciesRequest,
25+
InferredDependencies,
26+
)
27+
from pants.util.logging import LogLevel
28+
29+
from pack_metadata.target_types import PacksGlobDependencies
30+
31+
32+
@dataclass(frozen=True)
33+
class PacksGlobInferenceFieldSet(FieldSet):
34+
required_fields = (PacksGlobDependencies,)
35+
36+
dependencies: PacksGlobDependencies
37+
38+
39+
class InferPacksGlobDependencies(InferDependenciesRequest):
40+
infer_from = PacksGlobInferenceFieldSet
41+
42+
43+
@rule(
44+
desc="Inferring packs glob dependencies",
45+
level=LogLevel.DEBUG,
46+
)
47+
async def infer_packs_globs_dependencies(
48+
request: InferPacksGlobDependencies,
49+
) -> InferredDependencies:
50+
address = request.field_set.address
51+
52+
pack_build_paths, explicitly_provided_deps = await MultiGet(
53+
Get(
54+
Paths,
55+
PathGlobs(
56+
[os.path.join(address.spec_path, "*", "BUILD")],
57+
glob_match_error_behavior=GlobMatchErrorBehavior.error,
58+
description_of_origin=f"{address}'s packs glob",
59+
),
60+
),
61+
Get(
62+
ExplicitlyProvidedDependencies,
63+
DependenciesRequest(request.field_set.dependencies),
64+
),
65+
)
66+
67+
implicit_packs_deps = {
68+
Address(os.path.dirname(path)) for path in pack_build_paths.files
69+
}
70+
71+
inferred_packs_deps = (
72+
implicit_packs_deps
73+
- explicitly_provided_deps.ignores # FrozenOrderedSet[Address]
74+
- explicitly_provided_deps.includes # FrozenOrderedSet[Address]
75+
)
76+
return InferredDependencies(inferred_packs_deps)
77+
78+
79+
def rules():
80+
return [
81+
*collect_rules(),
82+
UnionRule(InferDependenciesRequest, InferPacksGlobDependencies),
83+
]

‎pants-plugins/pack_metadata/target_types_rules_test.py‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
# Copyright 2023 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
from __future__ import annotations
15+
16+
from textwrap import dedent
17+
18+
from pants.backend.python.target_types import (
19+
PythonSourceTarget,
20+
PythonSourcesGeneratorTarget,
21+
)
22+
from pants.backend.python.target_types_rules import rules as python_target_types_rules
23+
from pants.engine.addresses import Address
24+
from pants.engine.target import InferredDependencies
25+
from pants.testutil.rule_runner import QueryRule, RuleRunner
26+
27+
from .target_types_rules import (
28+
InferPacksGlobDependencies,
29+
PacksGlobInferenceFieldSet,
30+
rules as pack_metadata_target_types_rules,
31+
)
32+
from .target_types import PacksGlob
33+
34+
35+
def test_infer_packs_globs_dependencies() -> None:
36+
rule_runner = RuleRunner(
37+
rules=[
38+
*python_target_types_rules(),
39+
*pack_metadata_target_types_rules(),
40+
QueryRule(InferredDependencies, (InferPacksGlobDependencies,)),
41+
],
42+
target_types=[
43+
PythonSourceTarget,
44+
PythonSourcesGeneratorTarget,
45+
PacksGlob,
46+
],
47+
)
48+
rule_runner.write_files(
49+
{
50+
"packs/BUILD": dedent(
51+
"""\
52+
python_sources(
53+
name="git_submodule",
54+
sources=["./git_submodule/*.py"],
55+
)
56+
57+
packs_glob(
58+
name="all_packs_glob",
59+
dependencies=[
60+
"!./configs", # explicit ignore
61+
"./a", # explicit include
62+
],
63+
)
64+
"""
65+
),
66+
"packs/a/BUILD": "python_sources()",
67+
"packs/a/__init__.py": "",
68+
"packs/a/fixture.py": "",
69+
"packs/b/BUILD": dedent(
70+
"""\
71+
python_sources(
72+
dependencies=["packs/configs/b.yaml"],
73+
)
74+
"""
75+
),
76+
"packs/b/__init__.py": "",
77+
"packs/b/fixture.py": "",
78+
"packs/c/BUILD": "python_sources()",
79+
"packs/c/__init__.py": "",
80+
"packs/c/fixture.py": "",
81+
"packs/d/BUILD": "python_sources()",
82+
"packs/d/__init__.py": "",
83+
"packs/d/fixture.py": "",
84+
# imitate a pack in a git submodule (should NOT have a BUILD file)
85+
"packs/git_submodule/__init__.py": "",
86+
"packs/git_submodule/fixture.py": "",
87+
"packs/configs/BUILD": dedent(
88+
"""\
89+
resources(
90+
sources=["*.yaml"],
91+
)
92+
"""
93+
),
94+
"packs/configs/b.yaml": dedent(
95+
"""\
96+
---
97+
# pack config for pack b
98+
"""
99+
),
100+
}
101+
)
102+
103+
def run_dep_inference(address: Address) -> InferredDependencies:
104+
args = [
105+
"--source-root-patterns=/packs",
106+
]
107+
rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"})
108+
target = rule_runner.get_target(address)
109+
return rule_runner.request(
110+
InferredDependencies,
111+
[InferPacksGlobDependencies(PacksGlobInferenceFieldSet.create(target))],
112+
)
113+
114+
assert run_dep_inference(
115+
Address("packs", target_name="all_packs_glob")
116+
) == InferredDependencies(
117+
[
118+
# should not have packs/a (explicit dep does not need to be inferred)
119+
# should not have packs/configs (explicitly ignored)
120+
# should not have packs/git_submodule (no BUILD file = no targets to add)
121+
Address("packs/b"),
122+
Address("packs/c"),
123+
Address("packs/d"),
124+
],
125+
)

‎st2api/tests/unit/controllers/v1/test_pack_config_schema.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,12 @@
1717

1818
from st2tests.api import FunctionalTest
1919

20-
from st2tests.fixturesloader import get_fixtures_packs_base_path
20+
# import this so that pants can infer dependencies for the glob below
21+
from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH
2122

2223
__all__ = ["PackConfigSchemasControllerTestCase"]
2324

24-
PACKS_PATH = get_fixtures_packs_base_path()
25-
CONFIG_SCHEMA_COUNT = len(glob.glob("%s/*/config.schema.yaml" % (PACKS_PATH)))
25+
CONFIG_SCHEMA_COUNT = len(glob.glob(f"{PACKS_PATH}/*/config.schema.yaml"))
2626
assert CONFIG_SCHEMA_COUNT > 1
2727

2828

‎st2api/tests/unit/controllers/v1/test_pack_configs.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,13 @@
1919

2020
from st2tests.api import FunctionalTest
2121
from st2api.controllers.v1.pack_configs import PackConfigsController
22-
from st2tests.fixturesloader import get_fixtures_packs_base_path
22+
23+
# import this so that pants can infer dependencies for the glob below
24+
from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH
2325

2426
__all__ = ["PackConfigsControllerTestCase"]
2527

26-
PACKS_PATH = get_fixtures_packs_base_path()
27-
CONFIGS_COUNT = len(glob.glob("%s/configs/*.yaml" % (PACKS_PATH)))
28+
CONFIGS_COUNT = len(glob.glob(f"{PACKS_PATH}/configs/*.yaml"))
2829
assert CONFIGS_COUNT > 1
2930

3031

‎st2common/tests/integration/test_register_content_script.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@
2222
from st2tests.base import IntegrationTestCase
2323
from st2common.util.shell import run_command
2424
from st2tests import config as test_config
25-
from st2tests.fixturesloader import get_fixtures_packs_base_path
2625

2726
# import this so that pants can infer dependencies for the glob below
27+
from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH
2828
from st2tests.fixtures.packs.dummy_pack_1.fixture import PACK_PATH as DUMMY_PACK_1_PATH
2929
from st2tests.fixtures.packs.dummy_pack_4.fixture import PACK_PATH as DUMMY_PACK_4_PATH
3030
from st2tests.fixtures.packs.runners.fixture import FIXTURE_PATH as RUNNER_DIRS
@@ -38,8 +38,7 @@
3838
BASE_CMD_ARGS = [sys.executable, SCRIPT_PATH, "--config-file=conf/st2.tests.conf", "-v"]
3939
BASE_REGISTER_ACTIONS_CMD_ARGS = BASE_CMD_ARGS + ["--register-actions"]
4040

41-
PACKS_PATH = get_fixtures_packs_base_path()
42-
PACKS_COUNT = len(glob.glob("%s/*/pack.yaml" % (PACKS_PATH)))
41+
PACKS_COUNT = len(glob.glob(f"{PACKS_PATH}/*/pack.yaml"))
4342
assert PACKS_COUNT >= 2
4443

4544

‎st2common/tests/unit/test_policies_registrar.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
from st2common.persistence.policy import Policy
2828
from st2common.persistence.policy import PolicyType
2929
from st2tests.base import CleanDbTestCase
30-
from st2tests.fixturesloader import get_fixtures_packs_base_path
30+
from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH
3131
from st2tests.fixtures.packs.dummy_pack_1.fixture import (
3232
PACK_NAME as DUMMY_PACK_1,
3333
PACK_PATH as DUMMY_PACK_1_PATH,
@@ -63,8 +63,7 @@ def test_register_all_policies(self):
6363
policies_dbs = Policy.get_all()
6464
self.assertEqual(len(policies_dbs), 0)
6565

66-
packs_base_path = get_fixtures_packs_base_path()
67-
count = policies_registrar.register_policies(packs_base_paths=[packs_base_path])
66+
count = policies_registrar.register_policies(packs_base_paths=[PACKS_PATH])
6867

6968
# Verify PolicyDB objects have been created
7069
policies_dbs = Policy.get_all()

‎st2common/tests/unit/test_triggers_registrar.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
from st2common.persistence.trigger import Trigger
2020
from st2common.persistence.trigger import TriggerType
2121
from st2tests.base import CleanDbTestCase
22-
from st2tests.fixturesloader import get_fixtures_packs_base_path
22+
from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH
2323
from st2tests.fixtures.packs.dummy_pack_1.fixture import (
2424
PACK_NAME as DUMMY_PACK_1,
2525
PACK_PATH as DUMMY_PACK_1_PATH,
@@ -33,8 +33,7 @@ def test_register_all_triggers(self):
3333
trigger_type_dbs = TriggerType.get_all()
3434
self.assertEqual(len(trigger_type_dbs), 0)
3535

36-
packs_base_path = get_fixtures_packs_base_path()
37-
count = triggers_registrar.register_triggers(packs_base_paths=[packs_base_path])
36+
count = triggers_registrar.register_triggers(packs_base_paths=[PACKS_PATH])
3837
self.assertEqual(count, 2)
3938

4039
# Verify TriggerTypeDB and corresponding TriggerDB objects have been created

0 commit comments

Comments
 (0)