Update fixate-requirements.py to cope with PIP 20.3.3 (#5123)

* Update fixate-requirements.py to cope with PIP 20.3.3

* Update simplejson as earlier not python 3 supported

* Update rednose so py2 versions do not get attempted

* Pin min RandomWords version if use newer pip for build

* Set min for python-json-logger

* ipaddr is python2 only module, picked up by new pip when build. Update to use ipaddress built-in python3 module

* Update min versions to what used in a previous CI build for the new version files

* Update a few more requirements where pip considers multiple versions

* Update more versions

* Pin some requirements that have multiple versions, and are getting pulled in by other dependencies

* Pin some requirements that have multiple versions, and are getting pulled in by other dependencies

* Pin some requirements that have multiple versions, and are getting pulled in by other dependencies

* Pin some requirements that have multiple versions, and are getting pulled in by other dependencies

* Pin some requirements that have multiple versions, and are getting pulled in by other dependencies

* Update pip version

* Install libyaml-dev for PyYAML

* Set min pyparsing

* Put import of internal_req into try/except

* Modify fixate-requirements to work with older and new pips

* vine requirement clashing with amqp

* Remove vine completely to prevent pip clash with amqp

* Remove explicit vine

* If using legacy resolver then have to either not specify versions of inherited dependencies, or have exact version - else it conflicts

* Specifying urllib3 conflicts with requests[security]

* Specifying urllib3 conflicts with requests[security]

* Update version on st2tests

* Update isort

* Update to use virtualenv 20.2.2

* Re-instate setuptools upgrade but with version used by virtualenv

* Update to newest virtualenv to match pip 20.3.3

* Fix lint

* Update pip in run-pack-tests

* Apply suggestions from code review

Co-authored-by: Jacob Floyd <[email protected]>

* Black formatting

* Remove trailing space for lint

* Update CHANGELOG.rst

* Update on requirements

* Update CHANGELOG.rst to force re-run of checks

Co-authored-by: blag <[email protected]>
Co-authored-by: Jacob Floyd <[email protected]>

Author: 3 people

.github/workflows/ci.yaml

@@ -123,10 +123,10 @@ jobs:
             ${{ runner.os }}-${{ matrix.python }}-
       - name: Install apt depedencies
         run: |
-          # install dev dependencies for Python LDAP module
+          # install dev dependencies for Python YAML and LDAP packages
           # https://github.com/StackStorm/st2-auth-ldap
           sudo apt-get -y update
-          sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev ldap-utils
+          sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
       - name: Install virtualenv
         run: |
           # Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we

CHANGELOG.rst

@@ -40,6 +40,9 @@ Changed
 * Import ABC from collections.abc for Python 3.10 compatibility. (#5007)
   Contributed by @tirkarthi
 
+* Updated to use virtualenv 20.4.0/PIP20.3.3 and fixate-requirements to work with PIP 20.3.3 #512
+  Contributed by Amanda McGuinness (@amanda11 Ammeon Solutions)
+
 3.4.0 - March 02, 2021
 ----------------------
 

Makefile

@@ -55,7 +55,8 @@ REQUIREMENTS := test-requirements.txt requirements.txt
 
 # Pin common pip version here across all the targets
 # Note! Periodic maintenance pip upgrades are required to be up-to-date with the latest pip security fixes and updates
-PIP_VERSION ?= 20.0.2
+PIP_VERSION ?= 20.3.3
+SETUPTOOLS_VERSION ?= 51.3.3
 PIP_OPTIONS := $(ST2_PIP_OPTIONS)
 
 ifndef PYLINT_CONCURRENCY
@@ -461,11 +462,8 @@ flake8: requirements .flake8
 	touch $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate
 	chmod +x $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate
 
-	# NOTE We need to upgrade setuptools to avoid bug with dependency resolving in old versions
-	# Setuptools 42 added support for python_requires, which is used by the configparser package,
-	# which is required by the importlib-metadata package
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip==$(PIP_VERSION)"
-	$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "setuptools==44.1.0"
+	$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "setuptools==$(SETUPTOOLS_VERSION)"
 
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/activate; cd st2client ; ../$(VIRTUALENV_ST2CLIENT_DIR)/bin/python setup.py install ; cd ..
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/st2 --version
@@ -611,9 +609,7 @@ requirements: virtualenv .requirements .sdist-requirements install-runners insta
 	#       only have to update it one place when we change the version
 	$(VIRTUALENV_DIR)/bin/pip install --upgrade $(shell grep "^virtualenv" fixed-requirements.txt)
 
-	# setuptools >= 41.0.1 is required for packs.install in dev envs
-	# setuptools >= 42     is required so setup.py install respects dependencies' python_requires
-	$(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==44.1.0"
+	$(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==$(SETUPTOOLS_VERSION)"  # workaround for pbr issue
 	$(VIRTUALENV_DIR)/bin/pip install --upgrade "pbr==5.4.3"  # workaround for pbr issue
 
 	# Fix for Travis CI race

fixed-requirements.txt

@@ -46,9 +46,9 @@ six==1.13.0
 sseclient-py==1.7
 stevedore==1.30.1
 tooz==2.8.0
-# Note: virtualenv embeds the pip wheel, so pinning virtualenv also pins pip
-# virtualenv==20.0.18 has pip==20.0.2
-virtualenv==20.0.18
+# Note: virtualenv embeds wheels for pip, wheel, and setuptools. So pinning virtualenv pins those as well.
+# virtualenv==20.4.0 (<21) has pip==20.3.3 wheel==0.36.2 setuptools==51.3.3
+virtualenv==20.4.0
 webob==1.8.5
 zake==0.2.2
 # test requirements below

requirements.txt

@@ -12,7 +12,7 @@ argcomplete
 bcrypt==3.1.7
 chardet<3.1.0
 cryptography==3.3.2
-dnspython<2.0.0,>=1.16.0
+dnspython>=1.16.0,<2.0.0
 eventlet==0.25.1
 flex==6.14.0
 git+https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper
@@ -23,7 +23,6 @@ git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backe
 gitpython==2.1.15
 greenlet==0.4.15
 gunicorn==19.9.0
-ipaddr
 jinja2==2.10.3
 jsonpath-rw==1.4.0
 jsonschema==2.6.0
@@ -37,14 +36,14 @@ nose
 nose-parallel==0.3.1
 nose-timer==0.7.5
 orjson==3.5.0
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
 paramiko==2.7.1
 passlib==1.7.1
 prettytable
 prompt-toolkit==1.0.15
 psutil==5.6.6
-pyinotify==0.9.6 ; platform_system == "Linux"
+pyinotify==0.9.6; platform_system=="Linux"
 pymongo==3.10.0
 pyrabbit
 python-dateutil==2.8.0
@@ -69,4 +68,4 @@ unittest2
 webob==1.8.5
 webtest
 zake==0.2.2
-zipp<=1.0.0,>=0.5
+zipp>=0.5,<=1.0.0

scripts/fixate-requirements.py

@@ -70,6 +70,12 @@
         print("Using pip: %s" % (str(pip_version)))
         sys.exit(1)
 
+try:
+    from pip._internal.req.constructors import parse_req_from_line
+except ImportError:
+    # Do not error, as will only use on pip >= 20
+    pass
+
 
 def parse_args():
     parser = argparse.ArgumentParser(
@@ -138,19 +144,34 @@ def merge_source_requirements(sources):
     merged_requirements = []
     for infile_path in (locate_file(p, must_exist=True) for p in sources):
         for req in load_requirements(infile_path):
-            # Requirements starting with project name "project ..."
-            if req.req:
-                # Skip already added project name
-                if req.name in projects:
-                    continue
-                projects.add(req.name)
-                merged_requirements.append(req)
-
-            # Requirements lines like "vcs+proto://url"
-            elif req.link:
-                merged_requirements.append(req)
+            if hasattr(req, "requirement"):
+                # Requirements starting with project name "project ..."
+                parsedreq = parse_req_from_line(req.requirement, req.line_source)
+                if parsedreq.requirement:
+                    # Skip already added project name
+                    if parsedreq.requirement.name in projects:
+                        continue
+                    projects.add(parsedreq.requirement.name)
+                    merged_requirements.append(req)
+
+                # Requirements lines like "vcs+proto://url"
+                elif parsedreq.link:
+                    merged_requirements.append(req)
+                else:
+                    raise RuntimeError("Unexpected requirement {0}".format(req))
             else:
-                raise RuntimeError("Unexpected requirement {0}".format(req))
+                if req.req:
+                    # Skip already added project name
+                    if req.name in projects:
+                        continue
+                    projects.add(req.name)
+                    merged_requirements.append(req)
+
+                # Requirements lines like "vcs+proto://url"
+                elif req.link:
+                    merged_requirements.append(req)
+                else:
+                    raise RuntimeError("Unexpected requirement {0}".format(req))
 
     return merged_requirements
 
@@ -169,10 +190,17 @@ def write_requirements(
     # Make sure there are no duplicate / conflicting definitions
     fixedreq_hash = {}
     for req in fixed:
-        project_name = req.name
+        if hasattr(req, "requirement"):
+            parsedreq = parse_req_from_line(req.requirement, req.line_source)
+            project_name = parsedreq.requirement.name
 
-        if not req.req:
-            continue
+            if not req.requirement:
+                continue
+        else:
+            project_name = req.name
+
+            if not req.req:
+                continue
 
         if project_name in fixedreq_hash:
             raise ValueError(
@@ -184,22 +212,38 @@ def write_requirements(
     lines_to_write = []
     links = set()
     for req in requirements:
-        if req.name in skip:
+        if hasattr(req, "requirement"):
+            parsedreq = parse_req_from_line(req.requirement, req.line_source)
+            project_name = parsedreq.requirement.name
+            linkreq = parsedreq
+        else:
+            project_name = req.name
+            linkreq = req
+        if project_name in skip:
             continue
 
         # we don't have any idea how to process links, so just add them
-        if req.link and req.link not in links:
-            links.add(req.link)
-            rline = str(req.link)
+        if linkreq.link and linkreq.link not in links:
+            links.add(linkreq.link)
+            rline = str(linkreq.link)
 
-            if req.editable:
+            if (hasattr(req, "is_editable") and req.is_editable) or (
+                hasattr(req, "editable") and req.editable
+            ):
                 rline = "-e %s" % (rline)
-        elif req.req:
+        elif hasattr(req, "requirement") and req.requirement:
+            project = parsedreq.requirement.name
+            req_obj = fixedreq_hash.get(project, req)
+
+            rline = str(req_obj.requirement)
+
+            # Markers are included in req_obj.requirement, so no
+            # special processing required
+        elif hasattr(req, "req") and req.req:
             project = req.name
             req_obj = fixedreq_hash.get(project, req)
 
             rline = str(req_obj.req)
-
             # Also write out environment markers
             if req_obj.markers:
                 rline += " ; {}".format(str(req_obj.markers))

st2actions/requirements.txt

@@ -13,9 +13,9 @@ gitpython==2.1.15
 jinja2==2.10.3
 kombu==4.6.6
 lockfile==0.12.2
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
-pyinotify==0.9.6 ; platform_system == "Linux"
+pyinotify==0.9.6; platform_system=="Linux"
 python-dateutil==2.8.0
 python-json-logger
 pyyaml==5.1.2

st2api/requirements.txt

@@ -10,7 +10,7 @@ gunicorn==19.9.0
 jsonschema==2.6.0
 kombu==4.6.6
 mongoengine==0.18.2
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
 pymongo==3.10.0
 simplejson

st2auth/requirements.txt

@@ -10,7 +10,7 @@ eventlet==0.25.1
 git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file
 git+https://github.com/StackStorm/st2-auth-ldap.git@master#egg=st2-auth-ldap
 gunicorn==19.9.0
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 passlib==1.7.1
 pymongo==3.10.0
 six==1.13.0

st2client/requirements.txt

@@ -20,4 +20,4 @@ pyyaml==5.1.2
 requests[security]==2.23.0
 six==1.13.0
 sseclient-py==1.7
-zipp<=1.0.0,>=0.5
+zipp>=0.5,<=1.0.0