Inform pants about 3rd party dependencies and constraints

BUILD

@@ -1,5 +1,37 @@
 python_requirements(
-    name="root",
+    name="reqs",
+    source="requirements-pants.txt",
+    # module_mapping can be removed once pants is released with
+    # https://github.com/pantsbuild/pants/pull/17390
+    module_mapping={
+        "python-editor": ["editor"],
+        "python-json-logger": ["pythonjsonlogger"],
+        "python-statsd": ["statsd"],
+        "sseclient-py": ["sseclient"],
+        "oslo.config": ["oslo_config"],
+        "RandomWords": ["random_words"],
+    },
+    overrides={
+        # flex and stevedore uses pkg_resources w/o declaring the dep
+        ("flex", "stevedore"): {
+            "dependencies": [
+                "//:reqs#setuptools",
+            ]
+        },
+        # do not use the prance[flex] extra as that pulls in an old version of flex
+        "prance": {
+            "dependencies": [
+                "//:reqs#flex",
+            ]
+        },
+        # tooz needs one or more backends (tooz is used by the st2 coordination backend)
+        "tooz": {
+            "dependencies": [
+                "//:reqs#redis",
+                "//:reqs#zake",
+            ]
+        },
+    },
 )
 
 python_test_utils(

CHANGELOG.rst

@@ -10,7 +10,7 @@ Added
 * Continue introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
-  #5778
+  #5778 #5789
   Contributed by @cognifloyd
 
 

contrib/core/BUILD

@@ -3,4 +3,7 @@ python_sources()
 python_requirements(
     name="reqs",
     source="requirements-tests.txt",
+    # module_mapping can be removed once pants is released with
+    # https://github.com/pantsbuild/pants/pull/17390
+    module_mapping={"mail-parser": ["mailparser"]},
 )

contrib/runners/winrm_runner/BUILD

@@ -0,0 +1,7 @@
+python_requirement(
+    name="winrm",
+    requirements=["pywinrm"],
+    # modules can be removed once pants is released with
+    # https://github.com/pantsbuild/pants/pull/17390
+    modules=["winrm"],
+)

lockfiles/st2-constraints.txt

@@ -0,0 +1,113 @@
+# Add/remove version constraints for transitive dependencies in this file
+# (transitive dependencies are dependencies of our direct dependencies).
+#
+# Direct dependencies should be recorded in `requirements-pants.txt`, not here.
+
+# please document each version constraint as follows:
+#
+# REQUIRED BY: <package>, <package>, ...
+# REASON: <why do we need to constrain this transitive dep?>
+# NOTE: <status of this constraint / when can we remove it?>
+# DROPS RESOLVED VERSION: <which version pip resolved without this constraint>
+#<package><version constraint>
+
+# ############################################ #
+# pinned transitive deps from requirements.txt #
+# ############################################ #
+
+# REQUIRED BY: jinja2
+# REASON: Fix MarkupSafe to < 2.1.0 as 2.1.0 removes soft_unicode >=0.23 was from jinja2
+# NOTE: try to remove constraint later.
+# DROPS RESOLVED VERSION: unknown
+MarkupSafe<2.1.0,>=0.23
+
+# REQUIRED BY: kombu
+# REASON: unknown -- this looks like a lockfile-style pin
+# NOTE: try to remove constraint later.
+# DROPS RESOLVED VERSION: 5.1.1
+amqp==5.0.6
+
+# REQUIRED BY: cryptography, paramiko, passlib
+# REASON: unknown -- this looks like a lockfile-style pin
+# NOTE: try to remove constraint later.
+# DROPS RESOLVED VERSION: 4.0.1
+bcrypt==3.2.0
+
+# REQUIRED BY: bcrypt, cryptography, pynacl, zstandard
+# REASON: unknown
+# NOTE: try to remove constraint later.
+# DROPS RESOLVED VERSION: 1.15.1
+cffi<1.15.0
+
+# REQUIRED BY: orquesta, prance, requests
+# REASON: requests 2.23 requires chardet < 3.1.0
+# NOTE: orquesta already constrains this, so this is just documentation.
+# DROPS RESOLVED VERSION: 3.0.4
+#chardet<3.1.0
+
+# REQUIRED BY: jsonpath-rw, networkx
+# REASON:
+#     networkx requires decorator>=4.3,<5 which should resolve to version 4.4.2
+#     but the wheel on pypi does not say it supports python3.8, so pip gets
+#     confused. For now, pin decorator to work around pip's confusion.
+# NOTE: Since pants/pex use a newer version of pip, this is not an issue.
+# DROPS RESOLVED VERSION: 4.4.2
+#decorator==4.4.2
+
+# REQUIRED BY: eventlet, pymongo
+# REASON: 2.0 version breaks pymongo work with hosts
+# NOTE: try to remove this later
+# DROPS RESOLVED VERSION: 1.16
+dnspython>=1.16.0,<2.0.0
+
+# REQUIRED BY: eventlet
+# REASON: unknown -- this looks like a lockfile-style pin
+# NOTE: We are having a hard time upgrading eventlet, so this pin is commented
+#       out to see if that will help. If any tests fail, uncomment this.
+# DROPS RESOLVED VERSION: 1.1.3.post0
+#greenlet==1.0.0
+
+# REQUIRED BY: argcomplete, click, debtcollector, kombu, pluggy, prettytable,
+#             pytest, virtualenv
+# REASON: unknown
+# NOTE: This pinned version (3.10.1) actually conflicts with other requirements.
+#       So, it is commented out. If there are issues with newer versions,
+#       update this with a range of valid versions.
+# DROPS RESOLVED VERSION: 4.8.3
+#importlib-metadata==3.10.1
+
+# REQUIRED BY: tooz
+# REASON: unknown
+# NOTE: try to remove constraint later.
+# DROPS RESOLVED VERSION: 4.13
+oslo.utils<5.0,>=4.0.0
+
+# REQUIRED BY: tooz
+# REASON: unknown
+# NOTE: try to remove constraint later.
+# DROPS RESOLVED VERSION: 8.1
+tenacity>=3.2.1,<7.0.0
+
+# REQUIRED BY: st2-auth-backend-flat-file
+# REASON: unknown -- this looks like a lockfile-style pin
+# NOTE: st2-auth-backend-flat-file has a version range >=1.7.1,<1.8.0
+#       If we need to narrow that range, we should do so in:
+#       https://github.com/StackStorm/st2-auth-backend-flat-file/blob/master/requirements.txt
+# DROPS RESOLVED VERSION: 1.7.4
+#passlib==1.7.4
+
+# pyOpenSSL required by: pymongo[ocsp], redis[ocsp], urllib3[secure]
+# but we don't use any of those, so skip copying from fixed-requirements.txt
+
+# REQUIRED BY: httplib2, oslo.utils, packaging
+# REASON: unknown -- It looks like <3 was only needed for python2 compatibility.
+# NOTE: this is still here, commented, until we can validate that all test are
+#       passing without it.
+# DROPS RESOLVED VERSION: 3.0.7
+#pyparsing<3
+
+# REQUIRED BY: async-timeout, gitpython, importlib-metadata, redis
+# REASON: importlib-metadata requires typing-extensions but v4.2.0 requires py3.7+
+# NOTE: try to remove constraint later.
+# DROPS RESOLVED VERSION: 4.1.1
+typing-extensions<4.2

pants.toml

@@ -35,9 +35,9 @@ pants_ignore.add = [
   "st2common/tests/fixtures/requirements-used-for-tests.txt",
   "/fixed-requirements.txt",
   "/test-requirements.txt",
-  # keep requirements.txt for now. We might ignore it if we need an alternate interrim
-  # file that is decoupled from our legacy requirements files generation.
-  # "/requirements.txt",
+  # ignore requirements.txt for now, preferring interim files that are decoupled from
+  # legacy requirements files generation: requirements-pants.txt & lockfiles/st2-constraints.txt
+  "/requirements.txt",
 ]
 
 [source]

requirements-pants.txt

@@ -0,0 +1,101 @@
+# Add/remove direct 3rd party dependencies here, with version constraints if necessary.
+#
+# Please do not add transitive dependencies in this file (ie dependencies of our dependencies).
+# Use `lockfiles/st2-constraints.txt` to constrain the version of these transitive dependencies.
+#
+# Please keep this list alphabetical, with tooz backends in a separate list.
+
+apscheduler
+argcomplete
+ciso8601
+cryptography
+# eventlet 0.31+ and gunicorn 20.1.0 are not compatible
+eventlet<0.31
+# flex parses the openapi 2 spec in our router
+flex
+# gitpython & gitdb are used for pack management
+gitdb
+gitpython
+# st2common/tests/integration/test_util_green.py requires greenlet (as does eventlet)
+greenlet
+gunicorn
+jinja2
+jsonpath-rw
+jsonschema
+kombu
+lockfile
+mock
+mongoengine
+# Note: networkx v2.6 dropped support for Python3.6
+# networkx version is constrained in orquesta.
+networkx
+orjson
+orquesta @ git+https://github.com/StackStorm/[email protected]
+# NOTE: Recent version substantially affect the performance and add big import time overhead
+# See https://github.com/StackStorm/st2/issues/4160#issuecomment-394386433 for details
+oslo.config>=1.12.1,<1.13
+paramiko
+# prance is used by st2-validate-api-spec to validate the openapi spec
+# prance needs flex, but do not use the extra as that gets an old version.
+prance
+prettytable
+# For st2client: prompt-toolkit v2+ does not have prompt_toolkit.token.Token
+prompt-toolkit<2
+psutil
+pymongo
+# pyrabbit used in an integration test
+pyrabbit
+pytest
+python-dateutil
+python-editor
+# pythonjsonlogger referenced in st2actions/conf/logging.conf
+python-json-logger
+python-statsd
+pytz
+PyYAML
+# RandomWords used in some tests
+RandomWords
+requests[security]
+retrying
+routes
+semver
+# setuptools provides pkg_resources
+setuptools
+simplejson
+six
+# NOTE: we use sseclient-py instead of sseclient because sseclient
+# has various issues which sometimes hang the connection for a long time, etc.
+sseclient-py
+# bandit doesn't work w/ stevedore 3+
+stevedore<3
+# For backward compatibility reasons, flat file backend is installed by default
+st2-auth-backend-flat-file @ git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master
+st2-auth-ldap @ git+https://github.com/StackStorm/st2-auth-ldap.git@master
+st2-rbac-backend @ git+https://github.com/StackStorm/st2-rbac-backend.git@master
+# tabulate used by tools/log_watcher.py
+tabulate
+tooz
+udatetime
+ujson
+unittest2
+virtualenv
+webob
+webtest
+# zstandard is used for micro benchmarks
+zstandard
+
+# tooz backends
+redis
+zake
+
+# was in fixed-requirements.txt, but not in requirements-pants.txt
+# keyczar is used by a python2-only test.
+#python-keyczar
+
+###########
+
+# not needed with switch to pytest
+#nose
+#nose-timer
+#nose-parallel
+#rednose