Move to "pip-tools"/"pip compile" format (requirements.in and .txt) for tools like renovate

README.md

@@ -23,7 +23,7 @@ Done. You can now `cd ducker` and call `./ducker` to bootstrap and run it.
 
 $ cd ducker
 $ ./ducker
-Running unclean installation from requirements.txt
+Running unclean installation from requirements.in
 Ensuring unclean install ...
 Please initiate a query.
 Ducker (? for help) q
@@ -56,7 +56,7 @@ either detect the newest Python or select the best python of your choice.
 
 Two disable the automatic detection of the newest version and provide a
 list of acceptable Python versions (tried in the order you list them)
-add the following line to your requirements.txt file:
+add the following line to your requirements.in file:
 
 ```
 # appenv-python-preference: 3.6,3.9,3.8

bootstrap-dev.sh

@@ -7,4 +7,4 @@ PYTHON=$(basename $PYTHONABS)
 
 rm -rf .Python bin lib include
 $PYTHONABS -m venv .
-bin/pip install --upgrade -r requirements.txt
+bin/pip install --upgrade -r requirements.in

example/requirements.in

@@ -0,0 +1 @@
+ducker

example/requirements.txt

@@ -1 +1,3 @@
-ducker
+# appenv-requirements-hash: 23726ae5a1e34c3fd5735728c69f2c436b6801d9ac7df6e3d0d5eb7d03fc2a0d
+ducker==2.0.1
+setuptools==65.5.0

src/appenv.py

@@ -7,7 +7,7 @@
 #   - the appenv file is placed in a repo with the name of the application
 #   - the name of the application/file is an entrypoint XXX
 #   - python3.X+ with ensurepip
-#   - a requirements.txt file next to the appenv file
+#   - a requirements.in file next to the appenv file
 
 # TODO
 #
@@ -144,8 +144,8 @@ def ensure_venv(target):
 
 def parse_preferences():
     preferences = None
-    if os.path.exists('requirements.txt'):
-        with open('requirements.txt') as f:
+    if os.path.exists('requirements.in'):
+        with open('requirements.in') as f:
             for line in f:
                 # Expected format:
                 # # appenv-python-preference: 3.1,3.9,3.4
@@ -165,7 +165,7 @@ def ensure_minimal_python():
         # We have no preferences defined, use the current python.
         print("Updating lockfile with with {}.".format(current_python))
         print("If you want to use a different version, set it via")
-        print(" `# appenv-python-preference:` in requirements.txt.")
+        print(" `# appenv-python-preference:` in requirements.in.")
         return
 
     preferences.sort(key=lambda s: [int(u) for u in s.split('.')])
@@ -192,7 +192,7 @@ def ensure_minimal_python():
         os.execv(python, argv)
     else:
         print("Could not find the minimal preferred Python version.")
-        print("To ensure a working requirements.lock on all Python versions")
+        print("To ensure a working requirements.txt on all Python versions")
         print("make Python {} available on this system.".format(
             preferences[0]))
         sys.exit(66)
@@ -213,8 +213,7 @@ def ensure_best_python(base):
         if sys.version_info >= (3, 12):
             print("You are using a Python version >= 3.12.")
             print(
-                "Please specify a Python version in the requirements.txt file."
-            )
+                "Please specify a Python version in the requirements.in file.")
             print("Lockfiles created with a Python version lower than 3.12")
             print("may create a broken venv with a Python version >= 3.12.")
         # use newest Python available if nothing else is requested
@@ -373,39 +372,39 @@ def run(self, command, argv):
         os.execv(cmd, argv)
 
     def _assert_requirements_lock(self):
-        if not os.path.exists('requirements.lock'):
-            print('No requirements.lock found. Generate it using'
+        if not os.path.exists('requirements.txt'):
+            print('No requirements.txt found. Generate it using'
                   ' ./appenv update-lockfile')
             sys.exit(67)
 
-        with open('requirements.lock') as f:
+        with open('requirements.txt') as f:
             locked_hash = None
             for line in f:
                 if line.startswith("# appenv-requirements-hash: "):
                     locked_hash = line.split(':')[1].strip()
                     break
             if locked_hash != self._hash_requirements():
-                print('requirements.txt seems out of date (hash mismatch). '
+                print('requirements.in seems out of date (hash mismatch). '
                       'Regenerate using ./appenv update-lockfile')
                 sys.exit(67)
 
     def _hash_requirements(self):
-        with open('requirements.txt', 'rb') as f:
+        with open('requirements.in', 'rb') as f:
             hash_content = f.read()
         return hashlib.new("sha256", hash_content).hexdigest()
 
     def prepare(self, args=None, remaining=None):
-        # copy used requirements.txt into the target directory so we can use
+        # copy used requirements.in into the target directory so we can use
         # that to check later
         # - when to clean up old versions? keep like one or two old revisions?
-        # - enumerate the revisions and just copy the requirements.txt, check
+        # - enumerate the revisions and just copy the requirements.in, check
         #   for ones that are clean or rebuild if necessary
         os.chdir(self.base)
 
         self._assert_requirements_lock()
 
         hash_content = []
-        with open("requirements.lock", "rb") as f:
+        with open("requirements.txt", "rb") as f:
             requirements = f.read()
         hash_content.append(os.fsencode(os.path.realpath(sys.executable)))
         hash_content.append(requirements)
@@ -443,13 +442,13 @@ def prepare(self, args=None, remaining=None):
         if not os.path.exists(env_dir):
             ensure_venv(env_dir)
 
-            with open(os.path.join(env_dir, "requirements.lock"), "wb") as f:
+            with open(os.path.join(env_dir, "requirements.txt"), "wb") as f:
                 f.write(requirements)
 
             print("Installing ...")
             pip(env_dir, [
                 "install", "--no-deps", "-r",
-                "{env_dir}/requirements.lock".format(env_dir=env_dir)])
+                "{env_dir}/requirements.txt".format(env_dir=env_dir)])
             pip(env_dir, ["check"])
 
             with open(os.path.join(env_dir, "appenv.ready"), "w") as f:
@@ -495,7 +494,7 @@ def init(self, args=None, remaining=None):
         if os.path.exists(command):
             os.unlink(command)
         os.symlink('appenv', command)
-        with open("requirements.txt", "w") as requirements_txt:
+        with open("requirements.in", "w") as requirements_txt:
             requirements_txt.write(dependency + "\n")
         print()
         print("Done. You can now `cd {}` and call"
@@ -523,7 +522,7 @@ def update_lockfile(self, args=None, remaining=None):
             cmd(["rm", "-rf", tmpdir])
         ensure_venv(tmpdir)
         print("Installing packages ...")
-        pip(tmpdir, ["install", "-r", "requirements.txt"])
+        pip(tmpdir, ["install", "-r", "requirements.in"])
 
         extra_specs = []
         result = pip(
@@ -538,7 +537,7 @@ def update_lockfile(self, args=None, remaining=None):
             parsed_requirement = parse_requirement_string(line)
             pinned_versions[parsed_requirement.name] = parsed_requirement
         requested_versions = {}
-        with open('requirements.txt') as f:
+        with open('requirements.in') as f:
             for line in f.readlines():
                 if line.strip().startswith('-e '):
                     extra_specs.append(line.strip())
@@ -568,7 +567,7 @@ def update_lockfile(self, args=None, remaining=None):
         lines = [str(spec) for spec in final_versions.values()]
         lines.extend(extra_specs)
         lines.sort()
-        with open(os.path.join(self.base, "requirements.lock"), "w") as f:
+        with open(os.path.join(self.base, "requirements.txt"), "w") as f:
             f.write('# appenv-requirements-hash: {}\n'.format(
                 self._hash_requirements()))
             f.write('\n'.join(lines))

tests/test_init.py

@@ -20,7 +20,7 @@ def test_init(workdir, monkeypatch):
         original_appenv = f.read()
     assert ducker_appenv == original_appenv
 
-    with open(os.path.join(workdir, "ducker", "requirements.txt")) as f:
+    with open(os.path.join(workdir, "ducker", "requirements.in")) as f:
         requirements = f.read()
 
     assert requirements == "ducker\n"
@@ -37,7 +37,7 @@ def test_init(workdir, monkeypatch):
         original_appenv = f.read()
     assert ducker_appenv == original_appenv
 
-    with open(os.path.join(workdir, "ducker", "requirements.txt")) as f:
+    with open(os.path.join(workdir, "ducker", "requirements.in")) as f:
         requirements = f.read()
 
     assert requirements == "ducker\n"
@@ -55,7 +55,7 @@ def test_init_explicit_target(workdir, monkeypatch):
     with open(appenv.__file__) as f:
         original_appenv = f.read()
 
-    with open(os.path.join(workdir, "baz", "requirements.txt")) as f:
+    with open(os.path.join(workdir, "baz", "requirements.in")) as f:
         requirements = f.read()
 
     assert requirements == "ducker\n"
@@ -74,7 +74,7 @@ def test_init_explicit_package_and_target(workdir, monkeypatch):
     with open(appenv.__file__) as f:
         original_appenv = f.read()
 
-    with open(os.path.join(workdir, "baz", "requirements.txt")) as f:
+    with open(os.path.join(workdir, "baz", "requirements.in")) as f:
         requirements = f.read()
 
     assert requirements == "bar\n"

tests/test_run.py

@@ -54,7 +54,7 @@ def test_bootstrap_and_run_python_with_lockfile(workdir, monkeypatch):
 
 
 def test_bootstrap_and_run_without_lockfile(workdir, monkeypatch):
-    """It raises as error if no requirements.lock is present."""
+    """It raises as error if no requirements.txt is present."""
     monkeypatch.setattr("sys.stdin", io.StringIO("ducker\nducker==2.0.1\n\n"))
 
     env = appenv.AppEnv(os.path.join(workdir, 'ducker'), os.getcwd())
@@ -70,9 +70,8 @@ def test_bootstrap_and_run_without_lockfile(workdir, monkeypatch):
 
     with pytest.raises(subprocess.CalledProcessError) as err:
         subprocess.check_output(["./ducker", "--help"])
-    assert err.value.output == (
-        b"No requirements.lock found. Generate it using"
-        b" ./appenv update-lockfile\n")
+    assert err.value.output == (b"No requirements.txt found. Generate it using"
+                                b" ./appenv update-lockfile\n")
 
 
 def test_bootstrap_and_run_with_outdated_lockfile(workdir, monkeypatch):
@@ -93,14 +92,14 @@ def test_bootstrap_and_run_with_outdated_lockfile(workdir, monkeypatch):
         './appenv python -c "print(1)"', shell=True)
     assert output == b"1\n"
 
-    with open("requirements.txt", 'w') as f:
+    with open("requirements.in", 'w') as f:
         f.write('ducker==2.0.1')
 
     s = subprocess.Popen(
         './appenv python -c "print(1)"', shell=True, stdout=subprocess.PIPE)
     stdout, stderr = s.communicate()
     assert stdout == b"""\
-requirements.txt seems out of date (hash mismatch). Regenerate using ./appenv update-lockfile
+requirements.in seems out of date (hash mismatch). Regenerate using ./appenv update-lockfile
 """  # noqa
 
     subprocess.check_call('./appenv update-lockfile', shell=True)

tests/test_update_lockfile.py

@@ -13,7 +13,7 @@ def test_init_and_create_lockfile(workdir, monkeypatch):
     env = appenv.AppEnv(os.path.join(workdir, 'ducker'), os.getcwd())
     env.init()
 
-    lockfile = os.path.join(workdir, "ducker", "requirements.lock")
+    lockfile = os.path.join(workdir, "ducker", "requirements.txt")
     assert not os.path.exists(lockfile)
 
     env.update_lockfile()
@@ -36,8 +36,8 @@ def test_update_lockfile_minimal_python(workdir, monkeypatch):
     env = appenv.AppEnv(os.path.join(workdir, 'ppytest'), os.getcwd())
     env.init()
 
-    lockfile = os.path.join(workdir, "ppytest", "requirements.lock")
-    requirements_file = os.path.join(workdir, "ppytest", "requirements.txt")
+    lockfile = os.path.join(workdir, "ppytest", "requirements.txt")
+    requirements_file = os.path.join(workdir, "ppytest", "requirements.in")
 
     with open(requirements_file, "r+") as f:
         lines = f.readlines()
@@ -68,7 +68,7 @@ def test_update_lockfile_missing_minimal_python(workdir, monkeypatch):
     env = appenv.AppEnv(os.path.join(workdir, 'ppytest'), os.getcwd())
     env.init()
 
-    requirements_file = os.path.join(workdir, "ppytest", "requirements.txt")
+    requirements_file = os.path.join(workdir, "ppytest", "requirements.in")
 
     with open(requirements_file, "r+") as f:
         lines = f.readlines()