Merge pull request #165 from Erotemic/feat/relative-path-walk-up

add: support for new features in 3.12 pathlib

Author: Erotemic

.github/workflows/tests.yml

@@ -195,7 +195,7 @@ jobs:
           arch: auto
         - python-version: '3.7'
           install-extras: tests,optional
-          os: ubuntu-latest
+          os: ubuntu-22.04
           arch: auto
         - python-version: '3.8'
           install-extras: tests,optional
@@ -442,7 +442,7 @@ jobs:
         $GPG_EXECUTABLE --list-keys  || echo "first invocation of gpg creates directories and returns 1"
         $GPG_EXECUTABLE --list-keys
         VERSION=$(python -c "import setup; print(setup.VERSION)")
-        pip install twine
+        pip install twine packaging -U
         pip install urllib3 requests[security] twine
         GPG_KEYID=$(cat dev/public_gpg_key)
         echo "GPG_KEYID = '$GPG_KEYID'"
@@ -517,7 +517,7 @@ jobs:
         $GPG_EXECUTABLE --list-keys  || echo "first invocation of gpg creates directories and returns 1"
         $GPG_EXECUTABLE --list-keys
         VERSION=$(python -c "import setup; print(setup.VERSION)")
-        pip install twine
+        pip install twine packaging -U
         pip install urllib3 requests[security] twine
         GPG_KEYID=$(cat dev/public_gpg_key)
         echo "GPG_KEYID = '$GPG_KEYID'"

CHANGELOG.md

@@ -6,6 +6,12 @@ This project (loosely) adheres to [Semantic Versioning](https://semver.org/spec/
 
 ## Version 1.3.8 - Unreleased
 
+### Added
+* Added: `ub.Path.relative_to` now backports the `walk_up` feature from Python 3.12 
+
+### Changed
+* `ub.Path.walk` now supports the same signature as the new version in Python 3.12
+
 
 ## Version 1.3.7 - Released 2024-12-06
 

docs/source/conf.py

@@ -136,7 +136,7 @@ def visit_Assign(self, node):
     return visitor.version
 
 project = 'ubelt'
-copyright = '2024, Jon Crall'
+copyright = '2025, Jon Crall'
 author = 'Jon Crall'
 modname = 'ubelt'
 

tests/test_pathlib.py

@@ -540,3 +540,40 @@ def test_copy_dir_to_existing_dir_withconflict():
 
 def _comparable_walk(p):
     return sorted([(tuple(sorted(f)), tuple(sorted(d))) for (r, f, d) in (p).walk()])
+
+
+def test_walk_compat_312():
+    """
+    Test that our version works the same as the 3.12 version
+    """
+    import sys
+    if sys.version_info[0:2] < (3, 12):
+        import pytest
+        pytest.skip('only test on 3.12')
+
+    import ubelt as ub
+    import pathlib
+
+    ours = ub.Path.appdir('ubelt/tests/ls')
+    theirs = pathlib.Path(ours)
+
+    (ours / 'dir1').ensuredir()
+    (ours / 'dir2').ensuredir()
+    (ours / 'file1').touch()
+    (ours / 'file2').touch()
+    (ours / 'dir1/file3').touch()
+    (ours / 'dir2/file4').touch()
+
+    subdirs1 = list(ours.walk())
+    assert len(subdirs1) == 3
+
+    subdirs2 = list(theirs.walk())
+    assert subdirs1 == subdirs2
+
+
+def test_walk_bad_kwargs():
+    import ubelt as ub
+    import pytest
+    self = ub.Path('foo')
+    with pytest.raises(TypeError):
+        list(self.walk(does_not_exist=True))

ubelt/util_path.py

@@ -48,6 +48,9 @@
 
 WIN32 = sys.platform.startswith('win32')
 
+PYTHON_LT_3_8 = sys.version_info[0:2] < (3, 8)
+PYTHON_GE_3_12 = sys.version_info[0:2] >= (3, 12)
+
 
 def augpath(path, suffix='', prefix='', ext=None, tail='', base=None,
             dpath=None, relative=None, multidot=False):
@@ -1228,7 +1231,57 @@ def touch(self, mode=0o0666, exist_ok=True):
         super().touch(mode=mode, exist_ok=exist_ok)
         return self
 
-    def walk(self, topdown=True, onerror=None, followlinks=False):
+    def relative_to(self, *other, **kwargs):
+        """
+        Return the relative path to another path identified by the passed
+        arguments.  If the operation is not possible (because this is not a
+        subpath of the other path), raise ValueError.
+
+        Includes a Backport of :meth:`pathlib.Path.relative_to` with
+        ``walk_up=True`` that's not available pre 3.12.
+
+        Args:
+            other (Path | str): the base path
+
+            walk_up (bool):
+                controls whether `..` may be used to resolve the path.
+
+        Returns:
+            Path: the new relative path
+
+        References:
+            https://stackoverflow.com/questions/38083555/using-pathlibs-relative-to-for-directories-on-the-same-level
+            https://github.com/p2p-ld/numpydantic/blob/66fffc49f87bfaaa2f4d05bf1730c343b10c9cc6/src/numpydantic/serialization.py#L107-L142
+
+        Example:
+            >>> import ubelt as ub
+            >>> import pytest
+            >>> self = ub.Path('foo/bar')
+            >>> other = ub.Path('foo/bar/baz')
+            >>> result = self.relative_to(other, walk_up=True)
+            >>> assert result == ub.Path('..')
+            >>> with pytest.raises(ValueError):
+            >>>     self.relative_to(other)
+            >>> with pytest.raises(ValueError):
+            >>>     self.relative_to(other, walk_up=False)
+            >>> with pytest.raises(TypeError):
+            >>>     self.relative_to(other, not_a_kwarg=False)
+        """
+        if PYTHON_GE_3_12:
+            return super().relative_to(*other, **kwargs)
+        else:
+            # Test to see if we need the backport
+            walk_up = kwargs.pop('walk_up', False)
+            if len(kwargs):
+                bad_key = list(kwargs)[0]
+                raise TypeError(f'{self.__class__.__name__}.relative_to() got an unexpected keyword argument {bad_key!r}')
+            if not walk_up:
+                return super().relative_to(*other, **kwargs)
+            else:
+                # Use the backport
+                return _relative_path_backport(self, other, walk_up=walk_up)
+
+    def walk(self, topdown=True, onerror=None, followlinks=False, **kwargs):
         """
         A variant of :func:`os.walk` for pathlib
 
@@ -1244,6 +1297,11 @@ def walk(self, topdown=True, onerror=None, followlinks=False):
             followlinks (bool):
                 if True recurse into symbolic directory links
 
+            **kwargs:
+                Accepts aliases the 3.12 version of the above names: top_down,
+                on_error, follow_symlinks. In the future we may switch the 3.12
+                variants to be the primary arguments.
+
         Yields:
             Tuple['Path', List[str], List[str]]:
                 the root path, directory names, and file names
@@ -1274,11 +1332,28 @@ def walk(self, topdown=True, onerror=None, followlinks=False):
             >>>     if 'CVS' in dirs:
             >>>         dirs.remove('CVS')  # don't visit CVS directories
         """
-        cls = self.__class__
-        walker = os.walk(self, topdown=topdown, onerror=onerror,
-                         followlinks=followlinks)
-        for root, dnames, fnames in walker:
-            yield (cls(root), dnames, fnames)
+        # Add kwargs to support ubelt original kwargs as well as pathlib kwargs
+        top_down = kwargs.pop('top_down', topdown)
+        on_error = kwargs.pop('on_error', onerror)
+        follow_symlinks = kwargs.pop('follow_symlinks', followlinks)
+
+        if len(kwargs):
+            bad_key = list(kwargs)[0]
+            raise TypeError(f'{self.__class__.__name__}.relative_to() got an unexpected keyword argument {bad_key!r}')
+
+        if PYTHON_GE_3_12:
+            # Use the parent implementation if available
+            yield from super().walk(
+                top_down=top_down, on_error=on_error,
+                follow_symlinks=follow_symlinks)
+        else:
+            # TODO: backport the 3.12 implementation, which is more efficient
+            # Our original implementation
+            cls = self.__class__
+            walker = os.walk(self, topdown=top_down, onerror=on_error,
+                             followlinks=follow_symlinks)
+            for root, dnames, fnames in walker:
+                yield (cls(root), dnames, fnames)
 
     def __add__(self, other):
         """
@@ -1565,7 +1640,7 @@ def copy(self, dst, follow_file_symlinks=False, follow_dir_symlinks=False,
             _patch_win32_stats_on_pypy()
 
         if self.is_dir():
-            if sys.version_info[0:2] < (3, 8):  # nocover
+            if PYTHON_LT_3_8:  # nocover
                 copytree = _compat_copytree
             else:
                 copytree = shutil.copytree
@@ -1894,7 +1969,35 @@ def _patch_win32_stats_on_pypy():
         stat.IO_REPARSE_TAG_SYMLINK = 0xa000000c      # windows
 
 
-if sys.version_info[0:2] < (3, 8):  # nocover
+def _relative_path_backport(self, other, walk_up=False):
+    if not isinstance(other, _PathBase):
+        other = type(self)(*other)
+        # other = self.with_segments(other)
+    # anchor0, parts0 = self._stack
+    # anchor1, parts1 = other._stack
+    self_parts = self.parts
+    other_parts = other.parts
+    anchor0, parts0 = self_parts[0], list(reversed(self_parts[1:]))
+    anchor1, parts1 = other_parts[0], list(reversed(other_parts[1:]))
+    if anchor0 != anchor1:
+        raise ValueError(f"{self._raw_path!r} and {other._raw_path!r} have different anchors")
+    while parts0 and parts1 and parts0[-1] == parts1[-1]:
+        parts0.pop()
+        parts1.pop()
+    for part in parts1:
+        if not part or part == '.':
+            pass
+        elif not walk_up:
+            raise ValueError(f"{self._raw_path!r} is not in the subpath of {other._raw_path!r}")
+        elif part == '..':
+            raise ValueError(f"'..' segment in {other._raw_path!r} cannot be walked")
+        else:
+            parts0.append('..')
+    # return self.with_segments('', *reversed(parts0))
+    return type(self)('', *reversed(parts0))
+
+
+if PYTHON_LT_3_8:  # nocover
 
     # Vendor in a nearly modern copytree for Python 3.6 and 3.7
     def _compat_copytree(src, dst, symlinks=False, ignore=None,