Skip to content

Commit ec2d399

Browse files
cognifloydcognifloyd
authored
Merge pull request #5789 from StackStorm/pants-reqs
Inform pants about 3rd party dependencies and constraints
2 parents d274278 + fefc94d commit ec2d399

File tree

7 files changed

+261
-5
lines changed

7 files changed

+261
-5
lines changed

‎BUILD‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,37 @@
11
python_requirements(
2-
name="root",
2+
name="reqs",
3+
source="requirements-pants.txt",
4+
# module_mapping can be removed once pants is released with
5+
# https://github.com/pantsbuild/pants/pull/17390
6+
module_mapping={
7+
"python-editor": ["editor"],
8+
"python-json-logger": ["pythonjsonlogger"],
9+
"python-statsd": ["statsd"],
10+
"sseclient-py": ["sseclient"],
11+
"oslo.config": ["oslo_config"],
12+
"RandomWords": ["random_words"],
13+
},
14+
overrides={
15+
# flex and stevedore uses pkg_resources w/o declaring the dep
16+
("flex", "stevedore"): {
17+
"dependencies": [
18+
"//:reqs#setuptools",
19+
]
20+
},
21+
# do not use the prance[flex] extra as that pulls in an old version of flex
22+
"prance": {
23+
"dependencies": [
24+
"//:reqs#flex",
25+
]
26+
},
27+
# tooz needs one or more backends (tooz is used by the st2 coordination backend)
28+
"tooz": {
29+
"dependencies": [
30+
"//:reqs#redis",
31+
"//:reqs#zake",
32+
]
33+
},
34+
},
335
)
436

537
python_test_utils(

‎CHANGELOG.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ Added
1010
* Continue introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)
1111
working on StackStorm, improve our security posture, and improve CI reliability thanks in part
1212
to pants' use of PEX lockfiles. This is not a user-facing addition.
13-
#5778
13+
#5778 #5789
1414
Contributed by @cognifloyd
1515

1616

‎contrib/core/BUILD‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,7 @@ python_sources()
33
python_requirements(
44
name="reqs",
55
source="requirements-tests.txt",
6+
# module_mapping can be removed once pants is released with
7+
# https://github.com/pantsbuild/pants/pull/17390
8+
module_mapping={"mail-parser": ["mailparser"]},
69
)

‎contrib/runners/winrm_runner/BUILD‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
python_requirement(
2+
name="winrm",
3+
requirements=["pywinrm"],
4+
# modules can be removed once pants is released with
5+
# https://github.com/pantsbuild/pants/pull/17390
6+
modules=["winrm"],
7+
)

‎lockfiles/st2-constraints.txt‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
# Add/remove version constraints for transitive dependencies in this file
2+
# (transitive dependencies are dependencies of our direct dependencies).
3+
#
4+
# Direct dependencies should be recorded in `requirements-pants.txt`, not here.
5+
6+
# please document each version constraint as follows:
7+
#
8+
# REQUIRED BY: <package>, <package>, ...
9+
# REASON: <why do we need to constrain this transitive dep?>
10+
# NOTE: <status of this constraint / when can we remove it?>
11+
# DROPS RESOLVED VERSION: <which version pip resolved without this constraint>
12+
#<package><version constraint>
13+
14+
# ############################################ #
15+
# pinned transitive deps from requirements.txt #
16+
# ############################################ #
17+
18+
# REQUIRED BY: jinja2
19+
# REASON: Fix MarkupSafe to < 2.1.0 as 2.1.0 removes soft_unicode >=0.23 was from jinja2
20+
# NOTE: try to remove constraint later.
21+
# DROPS RESOLVED VERSION: unknown
22+
MarkupSafe<2.1.0,>=0.23
23+
24+
# REQUIRED BY: kombu
25+
# REASON: unknown -- this looks like a lockfile-style pin
26+
# NOTE: try to remove constraint later.
27+
# DROPS RESOLVED VERSION: 5.1.1
28+
amqp==5.0.6
29+
30+
# REQUIRED BY: cryptography, paramiko, passlib
31+
# REASON: unknown -- this looks like a lockfile-style pin
32+
# NOTE: try to remove constraint later.
33+
# DROPS RESOLVED VERSION: 4.0.1
34+
bcrypt==3.2.0
35+
36+
# REQUIRED BY: bcrypt, cryptography, pynacl, zstandard
37+
# REASON: unknown
38+
# NOTE: try to remove constraint later.
39+
# DROPS RESOLVED VERSION: 1.15.1
40+
cffi<1.15.0
41+
42+
# REQUIRED BY: orquesta, prance, requests
43+
# REASON: requests 2.23 requires chardet < 3.1.0
44+
# NOTE: orquesta already constrains this, so this is just documentation.
45+
# DROPS RESOLVED VERSION: 3.0.4
46+
#chardet<3.1.0
47+
48+
# REQUIRED BY: jsonpath-rw, networkx
49+
# REASON:
50+
# networkx requires decorator>=4.3,<5 which should resolve to version 4.4.2
51+
# but the wheel on pypi does not say it supports python3.8, so pip gets
52+
# confused. For now, pin decorator to work around pip's confusion.
53+
# NOTE: Since pants/pex use a newer version of pip, this is not an issue.
54+
# DROPS RESOLVED VERSION: 4.4.2
55+
#decorator==4.4.2
56+
57+
# REQUIRED BY: eventlet, pymongo
58+
# REASON: 2.0 version breaks pymongo work with hosts
59+
# NOTE: try to remove this later
60+
# DROPS RESOLVED VERSION: 1.16
61+
dnspython>=1.16.0,<2.0.0
62+
63+
# REQUIRED BY: eventlet
64+
# REASON: unknown -- this looks like a lockfile-style pin
65+
# NOTE: We are having a hard time upgrading eventlet, so this pin is commented
66+
# out to see if that will help. If any tests fail, uncomment this.
67+
# DROPS RESOLVED VERSION: 1.1.3.post0
68+
#greenlet==1.0.0
69+
70+
# REQUIRED BY: argcomplete, click, debtcollector, kombu, pluggy, prettytable,
71+
# pytest, virtualenv
72+
# REASON: unknown
73+
# NOTE: This pinned version (3.10.1) actually conflicts with other requirements.
74+
# So, it is commented out. If there are issues with newer versions,
75+
# update this with a range of valid versions.
76+
# DROPS RESOLVED VERSION: 4.8.3
77+
#importlib-metadata==3.10.1
78+
79+
# REQUIRED BY: tooz
80+
# REASON: unknown
81+
# NOTE: try to remove constraint later.
82+
# DROPS RESOLVED VERSION: 4.13
83+
oslo.utils<5.0,>=4.0.0
84+
85+
# REQUIRED BY: tooz
86+
# REASON: unknown
87+
# NOTE: try to remove constraint later.
88+
# DROPS RESOLVED VERSION: 8.1
89+
tenacity>=3.2.1,<7.0.0
90+
91+
# REQUIRED BY: st2-auth-backend-flat-file
92+
# REASON: unknown -- this looks like a lockfile-style pin
93+
# NOTE: st2-auth-backend-flat-file has a version range >=1.7.1,<1.8.0
94+
# If we need to narrow that range, we should do so in:
95+
# https://github.com/StackStorm/st2-auth-backend-flat-file/blob/master/requirements.txt
96+
# DROPS RESOLVED VERSION: 1.7.4
97+
#passlib==1.7.4
98+
99+
# pyOpenSSL required by: pymongo[ocsp], redis[ocsp], urllib3[secure]
100+
# but we don't use any of those, so skip copying from fixed-requirements.txt
101+
102+
# REQUIRED BY: httplib2, oslo.utils, packaging
103+
# REASON: unknown -- It looks like <3 was only needed for python2 compatibility.
104+
# NOTE: this is still here, commented, until we can validate that all test are
105+
# passing without it.
106+
# DROPS RESOLVED VERSION: 3.0.7
107+
#pyparsing<3
108+
109+
# REQUIRED BY: async-timeout, gitpython, importlib-metadata, redis
110+
# REASON: importlib-metadata requires typing-extensions but v4.2.0 requires py3.7+
111+
# NOTE: try to remove constraint later.
112+
# DROPS RESOLVED VERSION: 4.1.1
113+
typing-extensions<4.2

‎pants.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,9 @@ pants_ignore.add = [
3535
"st2common/tests/fixtures/requirements-used-for-tests.txt",
3636
"/fixed-requirements.txt",
3737
"/test-requirements.txt",
38-
# keep requirements.txt for now. We might ignore it if we need an alternate interrim
39-
# file that is decoupled from our legacy requirements files generation.
40-
# "/requirements.txt",
38+
# ignore requirements.txt for now, preferring interim files that are decoupled from
39+
# legacy requirements files generation: requirements-pants.txt & lockfiles/st2-constraints.txt
40+
"/requirements.txt",
4141
]
4242

4343
[source]

‎requirements-pants.txt‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
# Add/remove direct 3rd party dependencies here, with version constraints if necessary.
2+
#
3+
# Please do not add transitive dependencies in this file (ie dependencies of our dependencies).
4+
# Use `lockfiles/st2-constraints.txt` to constrain the version of these transitive dependencies.
5+
#
6+
# Please keep this list alphabetical, with tooz backends in a separate list.
7+
8+
apscheduler
9+
argcomplete
10+
ciso8601
11+
cryptography
12+
# eventlet 0.31+ and gunicorn 20.1.0 are not compatible
13+
eventlet<0.31
14+
# flex parses the openapi 2 spec in our router
15+
flex
16+
# gitpython & gitdb are used for pack management
17+
gitdb
18+
gitpython
19+
# st2common/tests/integration/test_util_green.py requires greenlet (as does eventlet)
20+
greenlet
21+
gunicorn
22+
jinja2
23+
jsonpath-rw
24+
jsonschema
25+
kombu
26+
lockfile
27+
mock
28+
mongoengine
29+
# Note: networkx v2.6 dropped support for Python3.6
30+
# networkx version is constrained in orquesta.
31+
networkx
32+
orjson
33+
orquesta @ git+https://github.com/StackStorm/[email protected]
34+
# NOTE: Recent version substantially affect the performance and add big import time overhead
35+
# See https://github.com/StackStorm/st2/issues/4160#issuecomment-394386433 for details
36+
oslo.config>=1.12.1,<1.13
37+
paramiko
38+
# prance is used by st2-validate-api-spec to validate the openapi spec
39+
# prance needs flex, but do not use the extra as that gets an old version.
40+
prance
41+
prettytable
42+
# For st2client: prompt-toolkit v2+ does not have prompt_toolkit.token.Token
43+
prompt-toolkit<2
44+
psutil
45+
pymongo
46+
# pyrabbit used in an integration test
47+
pyrabbit
48+
pytest
49+
python-dateutil
50+
python-editor
51+
# pythonjsonlogger referenced in st2actions/conf/logging.conf
52+
python-json-logger
53+
python-statsd
54+
pytz
55+
PyYAML
56+
# RandomWords used in some tests
57+
RandomWords
58+
requests[security]
59+
retrying
60+
routes
61+
semver
62+
# setuptools provides pkg_resources
63+
setuptools
64+
simplejson
65+
six
66+
# NOTE: we use sseclient-py instead of sseclient because sseclient
67+
# has various issues which sometimes hang the connection for a long time, etc.
68+
sseclient-py
69+
# bandit doesn't work w/ stevedore 3+
70+
stevedore<3
71+
# For backward compatibility reasons, flat file backend is installed by default
72+
st2-auth-backend-flat-file @ git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master
73+
st2-auth-ldap @ git+https://github.com/StackStorm/st2-auth-ldap.git@master
74+
st2-rbac-backend @ git+https://github.com/StackStorm/st2-rbac-backend.git@master
75+
# tabulate used by tools/log_watcher.py
76+
tabulate
77+
tooz
78+
udatetime
79+
ujson
80+
unittest2
81+
virtualenv
82+
webob
83+
webtest
84+
# zstandard is used for micro benchmarks
85+
zstandard
86+
87+
# tooz backends
88+
redis
89+
zake
90+
91+
# was in fixed-requirements.txt, but not in requirements-pants.txt
92+
# keyczar is used by a python2-only test.
93+
#python-keyczar
94+
95+
###########
96+
97+
# not needed with switch to pytest
98+
#nose
99+
#nose-timer
100+
#nose-parallel
101+
#rednose

0 commit comments

Comments
 (0)