autour_de_minuit/blender-asset-tracer
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

Improved path handling

Browse Source 
This commit fixes a bunch of issues at the same time, as they are all
related to path handling:

- `pathlib.Path.resolve()` or `.absolute()` are replaced by
  `bpathlib.make_absolute()`. The latter does NOT follow symlinks and does
  NOT network mounts from a drive letter to UNC notation. This also has
  advantages on non-Windows sytems, as it allows BAT-packing a directory
  structure with symlinked files (such as a Shaman checkout).
- Better handling of drive letters, and of paths that cross drive
  boundaries.
- Better testing of Windows-specific cases when running the tests on
  Windows, and of POSIX-specific cases on other platforms.

Thanks to @wisaac for starting this patch in D6676.

Thanks to @jbakker for pointing out the drive letter issue. This fixes
T70655.
This commit is contained in:
Sybren A. Stüvel
2020-03-17 17:15:19 +01:00
parent 3fa4389658
commit e4bf2e8e35
12 changed files with 367 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
blender_asset_tracer/blendfile/__init__.py
+3 -3
View File
@@ -49,7 +49,7 @@ def open_cached(path: pathlib.Path, mode='rb',
assert_cached: typing.Optional[bool] = None) -> 'BlendFile':
"""Open a blend file, ensuring it is only opened once."""
my_log = log.getChild('open_cached')
bfile_path = path.absolute().resolve()
bfile_path = bpathlib.make_absolute(path)
if assert_cached is not None:
is_cached = bfile_path in _cached_bfiles
@@ -84,13 +84,13 @@ def close_all_cached() -> None:
def _cache(path: pathlib.Path, bfile: 'BlendFile'):
"""Add a BlendFile to the cache."""
bfile_path = path.absolute().resolve()
bfile_path = bpathlib.make_absolute(path)
_cached_bfiles[bfile_path] = bfile
def _uncache(path: pathlib.Path):
"""Remove a BlendFile object from the cache."""
bfile_path = path.absolute().resolve()
bfile_path = bpathlib.make_absolute(path)
_cached_bfiles.pop(bfile_path, None)
blender_asset_tracer/bpathlib.py
+66 -2
View File
@@ -25,6 +25,7 @@ or vice versa.
import os.path
import pathlib
import platform
import string
import sys
@@ -41,18 +42,29 @@ class BlendPath(bytes):
return super().__new__(cls, path.replace(b'\\', b'/'))
@classmethod
def mkrelative(cls, asset_path: pathlib.Path, bfile_path: pathlib.PurePath) -> 'BlendPath':
def mkrelative(cls, asset_path: pathlib.PurePath, bfile_path: pathlib.PurePath) -> 'BlendPath':
"""Construct a BlendPath to the asset relative to the blend file.
Assumes that bfile_path is absolute.
Note that this can return an absolute path on Windows when 'asset_path'
and 'bfile_path' are on different drives.
"""
from collections import deque
# Only compare absolute paths.
assert bfile_path.is_absolute(), \
'BlendPath().mkrelative(bfile_path=%r) should get absolute bfile_path' % bfile_path
assert asset_path.is_absolute(), \
'BlendPath().mkrelative(asset_path=%r) should get absolute asset_path' % asset_path
# There is no way to construct a relative path between drives.
if bfile_path.drive != asset_path.drive:
return cls(asset_path)
bdir_parts = deque(bfile_path.parent.parts)
asset_parts = deque(asset_path.absolute().parts)
asset_path = make_absolute(asset_path)
asset_parts = deque(asset_path.parts)
# Remove matching initial parts. What is left in bdir_parts represents
# the number of '..' we need. What is left in asset_parts represents
@@ -101,6 +113,8 @@ class BlendPath(bytes):
Interprets the path as UTF-8, and if that fails falls back to the local
filesystem encoding.
The exact type returned is determined by the current platform.
"""
# TODO(Sybren): once we target Python 3.6, implement __fspath__().
try:
@@ -146,3 +160,53 @@ class BlendPath(bytes):
else:
my_relpath = self
return BlendPath(os.path.join(root, my_relpath))
def make_absolute(path: pathlib.PurePath) -> pathlib.Path:
"""Make the path absolute without resolving symlinks or drive letters.
This function is an alternative to `Path.resolve()`. It make the path absolute,
and resolves `../../`, but contrary to `Path.resolve()` does NOT perform these
changes:
- Symlinks are NOT followed.
- Windows Network shares that are mapped to a drive letter are NOT resolved
to their UNC notation.
The type of the returned path is determined by the current platform.
"""
str_path = path.as_posix()
if len(str_path) >= 2 and str_path[0].isalpha() and str_path[1] == ':':
# This is an absolute Windows path. It must be handled with care on non-Windows platforms.
if platform.system() != 'Windows':
# Normalize the POSIX-like part of the path, but leave out the drive letter.
non_drive_path = str_path[2:]
normalized = os.path.normpath(non_drive_path)
# Stick the drive letter back on the normalized path.
return pathlib.Path(str_path[:2] + normalized)
return pathlib.Path(os.path.abspath(str_path))
def strip_root(path: pathlib.PurePath) -> pathlib.PurePosixPath:
"""Turn the path into a relative path by stripping the root.
This also turns any drive letter into a normal path component.
This changes "C:/Program Files/Blender" to "C/Program Files/Blender",
and "/absolute/path.txt" to "absolute/path.txt", making it possible to
treat it as a relative path.
"""
if path.drive:
return pathlib.PurePosixPath(path.drive[0], *path.parts[1:])
if isinstance(path, pathlib.PurePosixPath):
# This happens when running on POSIX but still handling paths
# originating from a Windows machine.
parts = path.parts
if parts and len(parts[0]) == 2 and parts[0][0].isalpha() and parts[0][1] == ':':
# The first part is a drive letter.
return pathlib.PurePosixPath(parts[0][0], *path.parts[1:])
if path.is_absolute():
return pathlib.PurePosixPath(*path.parts[1:])
return pathlib.PurePosixPath(path)
blender_asset_tracer/cli/list_deps.py
+2 -2
View File
@@ -27,7 +27,7 @@ import sys
import time
import typing
from blender_asset_tracer import trace
from blender_asset_tracer import trace, bpathlib
from . import common
log = logging.getLogger(__name__)
@@ -106,7 +106,7 @@ def report_text(bpath, *, include_sha256: bool, show_timing: bool):
last_reported_bfile = filepath
for assetpath in usage.files():
assetpath = assetpath.resolve()
assetpath = bpathlib.make_absolute(assetpath)
if assetpath in reported_assets:
log.debug('Already reported %s', assetpath)
continue
blender_asset_tracer/cli/pack.py
+4 -4
View File
@@ -24,7 +24,7 @@ import sys
import typing
import blender_asset_tracer.pack.transfer
from blender_asset_tracer import pack
from blender_asset_tracer import pack, bpathlib
log = logging.getLogger(__name__)
@@ -168,15 +168,15 @@ def paths_from_cli(args) -> typing.Tuple[pathlib.Path, pathlib.Path, str]:
if bpath.is_dir():
log.critical('%s is a directory, should be a blend file')
sys.exit(3)
bpath = bpath.absolute().resolve()
bpath = bpathlib.make_absolute(bpath)
tpath = args.target
if args.project is None:
ppath = bpath.absolute().parent.resolve()
ppath = bpathlib.make_absolute(bpath).parent
log.warning('No project path given, using %s', ppath)
else:
ppath = args.project.absolute().resolve()
ppath = bpathlib.make_absolute(args.project)
if not ppath.exists():
log.critical('Project directory %s does not exist', ppath)
blender_asset_tracer/pack/__init__.py
+8 -15
View File
@@ -230,13 +230,13 @@ class Packer:
# The blendfile that we pack is generally not its own dependency, so
# we have to explicitly add it to the _packed_paths.
bfile_path = self.blendfile.absolute()
bfile_path = bpathlib.make_absolute(self.blendfile)
# Both paths have to be resolved first, because this also translates
# network shares mapped to Windows drive letters back to their UNC
# notation. Only resolving one but not the other (which can happen
# with the abosolute() call above) can cause errors.
bfile_pp = self._target_path / bfile_path.resolve().relative_to(self.project.resolve())
bfile_pp = self._target_path / bfile_path.relative_to(bpathlib.make_absolute(self.project))
self._output_path = bfile_pp
self._progress_cb.pack_start()
@@ -330,15 +330,8 @@ class Packer:
act = self._actions[path]
assert isinstance(act, AssetAction)
# Remove the base of the path, effectively removing the 'absoluteness'.
# On POSIX this turns '/path/file.txt' into 'path/file.txt'.
# On Windows this turns 'X:/path/file.txt' into 'X/path/file.txt'.
if path.drive:
path_parts = (path.drive[0], *path.parts[1:])
else:
path_parts = path.parts[1:]
act.new_path = pathlib.Path(self._target_path, '_outside_project', *path_parts)
relpath = bpathlib.strip_root(path)
act.new_path = pathlib.Path(self._target_path, '_outside_project', relpath)
def _group_rewrites(self) -> None:
"""For each blend file, collect which fields need rewriting.
@@ -358,7 +351,7 @@ class Packer:
continue
for usage in action.usages:
bfile_path = usage.block.bfile.filepath.absolute().resolve()
bfile_path = bpathlib.make_absolute(usage.block.bfile.filepath)
insert_new_action = bfile_path not in self._actions
self._actions[bfile_path].rewrites.append(usage)
@@ -367,10 +360,10 @@ class Packer:
actions.add(self._actions[bfile_path])
def _path_in_project(self, path: pathlib.Path) -> bool:
abs_path = bpathlib.make_absolute(path)
abs_project = bpathlib.make_absolute(self.project)
try:
# MUST use resolve(), otherwise /path/to/proj/../../asset.png
# will return True (relative_to will return ../../asset.png).
path.resolve().relative_to(self.project)
abs_path.relative_to(abs_project)
except ValueError:
return False
return True
blender_asset_tracer/pack/shaman/transfer.py
+5 -1
View File
@@ -26,6 +26,7 @@ import typing
import requests
import blender_asset_tracer.pack.transfer as bat_transfer
from blender_asset_tracer import bpathlib
MAX_DEFERRED_PATHS = 8
MAX_FAILED_PATHS = 8
@@ -83,6 +84,9 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
self.log.info('Created checkout definition file of %d KiB',
len(definition_file) // 1024)
self.log.info('Feeding %d files to the Shaman', len(self._file_info))
if self.log.isEnabledFor(logging.INFO):
for path in self._file_info:
self.log.info(' - %s', path)
# Try to upload all the files.
failed_paths = set() # type: typing.Set[str]
@@ -151,7 +155,7 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
checksum = cache.compute_cached_checksum(src)
filesize = src.stat().st_size
# relpath = dst.relative_to(self.project_root)
relpath = str(dst)[1:]
relpath = bpathlib.strip_root(dst).as_posix()
self._file_info[relpath] = FileInfo(
checksum=checksum,
blender_asset_tracer/trace/file2blocks.py
+5 -5
View File
@@ -82,7 +82,7 @@ class BlockIterator:
yield from self._visit_linked_blocks(blocks_per_lib)
def _visit_blocks(self, bfile, limit_to):
bpath = bfile.filepath.absolute().resolve()
bpath = bpathlib.make_absolute(bfile.filepath)
root_dir = bpathlib.BlendPath(bpath.parent)
# Mapping from library path to data blocks to expand.
@@ -123,10 +123,10 @@ class BlockIterator:
# We've gone through all the blocks in this file, now open the libraries
# and iterate over the blocks referred there.
for lib_bpath, idblocks in blocks_per_lib.items():
lib_path = pathlib.Path(lib_bpath.to_path())
try:
lib_path = lib_path.resolve()
except FileNotFoundError:
lib_path = bpathlib.make_absolute(lib_bpath.to_path())
assert lib_path.exists()
if not lib_path.exists():
log.warning('Library %s does not exist', lib_path)
continue
blender_asset_tracer/trace/result.py
+9 -10
View File
@@ -107,8 +107,8 @@ class BlockUsage:
def __repr__(self):
if self.path_full_field is None:
field_name = self.path_dir_field.name.name_full.decode() + \
'/' + \
self.path_base_field.name.name_full.decode()
'/' + \
self.path_base_field.name.name_full.decode()
else:
field_name = self.path_full_field.name.name_full.decode()
return '<BlockUsage name=%r type=%r field=%r asset=%r%s>' % (
@@ -141,20 +141,19 @@ class BlockUsage:
log.warning('Path %s does not exist for %s', path, self)
def __fspath__(self) -> pathlib.Path:
"""Determine the absolute path of the asset on the filesystem.
The path is resolved (see pathlib.Path.resolve()) if it exists on the
filesystem.
"""
"""Determine the absolute path of the asset on the filesystem."""
if self._abspath is None:
bpath = self.block.bfile.abspath(self.asset_path)
log.info('Resolved %s rel to %s -> %s',
self.asset_path, self.block.bfile.filepath, bpath)
as_path = pathlib.Path(bpath.to_path())
# Windows cannot resolve() a path that has a glob pattern in it.
# Windows cannot make a path that has a glob pattern in it absolute.
# Since globs are generally only on the filename part, we take that off,
# resolve() the parent directory, then put the filename back.
# make the parent directory absolute, then put the filename back.
try:
abs_parent = as_path.parent.resolve()
abs_parent = bpathlib.make_absolute(as_path.parent)
except FileNotFoundError:
self._abspath = as_path
else: