autour_de_minuit/blender-asset-tracer
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Convert target path from Path to str & PurePath

Browse Source 
The target path is just read as string from the CLI now, to allow more
complex targets (such as URLs) that don't directly map to a path.

The Packer subclass now handles the conversion from that string to a
`pathlib.PurePath`, and specific subclasses & transfer classes can convert
those to a `pathlib.Path` to perform actual filesystem operations when
necessary.
This commit is contained in:
Sybren A. Stüvel 2019-02-26 15:22:14 +01:00
parent 754edd8711
commit 03fb4da583
9 changed files with 60 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
blender_asset_tracer/bpathlib.py
View File

@ -41,11 +41,17 @@ class BlendPath(bytes):
return super().__new__(cls, path.replace(b'\\', b'/'))
@classmethod
def mkrelative(cls, asset_path: pathlib.Path, bfile_path: pathlib.Path) -> 'BlendPath':
"""Construct a BlendPath to the asset relative to the blend file."""
def mkrelative(cls, asset_path: pathlib.Path, bfile_path: pathlib.PurePath) -> 'BlendPath':
"""Construct a BlendPath to the asset relative to the blend file.
Assumes that bfile_path is absolute.
"""
from collections import deque
bdir_parts = deque(bfile_path.absolute().parent.parts)
assert bfile_path.is_absolute(), \
'BlendPath().mkrelative(bfile_path=%r) should get absolute bfile_path' % bfile_path
bdir_parts = deque(bfile_path.parent.parts)
asset_parts = deque(asset_path.absolute().parts)
# Remove matching initial parts. What is left in bdir_parts represents

19
blender_asset_tracer/cli/pack.py
View File

@ -36,7 +36,7 @@ def add_parser(subparsers):
parser.set_defaults(func=cli_pack)
parser.add_argument('blendfile', type=pathlib.Path,
help='The Blend file to pack.')
parser.add_argument('target', type=pathlib.Path,
parser.add_argument('target', type=str,
help='The target can be a directory, a ZIP file (does not have to exist '
"yet, just use 'something.zip' as target), or a URL of S3 storage "
'(s3://endpoint/path).')
@ -76,9 +76,8 @@ def cli_pack(args):
raise SystemExit(1)
def create_packer(args, bpath: pathlib.Path, ppath: pathlib.Path,
tpath: pathlib.Path) -> pack.Packer:
if str(tpath).startswith('s3:/'):
def create_packer(args, bpath: pathlib.Path, ppath: pathlib.Path, target: str) -> pack.Packer:
if target.startswith('s3:/'):
if args.noop:
raise ValueError('S3 uploader does not support no-op.')
@ -88,18 +87,18 @@ def create_packer(args, bpath: pathlib.Path, ppath: pathlib.Path,
if args.relative_only:
raise ValueError('S3 uploader does not support the --relative-only option')
packer = create_s3packer(bpath, ppath, tpath)
elif tpath.suffix.lower() == '.zip':
packer = create_s3packer(bpath, ppath, pathlib.PurePosixPath(target))
elif target.lower().endswith('.zip'):
from blender_asset_tracer.pack import zipped
if args.compress:
raise ValueError('ZIP packer does not support on-the-fly compression')
packer = zipped.ZipPacker(bpath, ppath, tpath, noop=args.noop,
packer = zipped.ZipPacker(bpath, ppath, target, noop=args.noop,
relative_only=args.relative_only)
else:
packer = pack.Packer(bpath, ppath, tpath, noop=args.noop, compress=args.compress,
relative_only=args.relative_only)
packer = pack.Packer(bpath, ppath, target, noop=args.noop,
compress=args.compress, relative_only=args.relative_only)
if args.exclude:
# args.exclude is a list, due to nargs='*', so we have to split and flatten.
@ -123,7 +122,7 @@ def create_s3packer(bpath, ppath, tpath) -> pack.Packer:
return s3.S3Packer(bpath, ppath, tpath, endpoint=endpoint)
def paths_from_cli(args) -> typing.Tuple[pathlib.Path, pathlib.Path, pathlib.Path]:
def paths_from_cli(args) -> typing.Tuple[pathlib.Path, pathlib.Path, str]:
"""Return paths to blendfile, project, and pack target.
Calls sys.exit() if anything is wrong.

31
blender_asset_tracer/pack/__init__.py
View File

@ -50,7 +50,7 @@ class AssetAction:
(if the asset is a blend file) or in another blend file.
"""
self.new_path = None # type: typing.Optional[pathlib.Path]
self.new_path = None # type: typing.Optional[pathlib.PurePath]
"""Absolute path to the asset in the BAT Pack.
This path may not exist on the local file system at all, for example
@ -96,7 +96,7 @@ class Packer:
def __init__(self,
bfile: pathlib.Path,
project: pathlib.Path,
target: pathlib.Path,
target: str,
*,
noop=False,
compress=False,
@ -104,6 +104,7 @@ class Packer:
self.blendfile = bfile
self.project = project
self.target = target
self._target_path = self._make_target_path(target)
self.noop = noop
self.compress = compress
self.relative_only = relative_only
@ -128,7 +129,7 @@ class Packer:
# type: typing.DefaultDict[pathlib.Path, AssetAction]
self.missing_files = set() # type: typing.Set[pathlib.Path]
self._new_location_paths = set() # type: typing.Set[pathlib.Path]
self._output_path = None # type: typing.Optional[pathlib.Path]
self._output_path = None # type: typing.Optional[pathlib.PurePath]
# Filled by execute()
self._file_transferer = None # type: typing.Optional[transfer.FileTransferer]
@ -139,6 +140,15 @@ class Packer:
self._tmpdir = tempfile.TemporaryDirectory(prefix='bat-', suffix='-batpack')
self._rewrite_in = pathlib.Path(self._tmpdir.name)
def _make_target_path(self, target: str) -> pathlib.PurePath:
"""Return a Path for the given target.
This can be the target directory itself, but can also be a non-existent
directory if the target doesn't support direct file access. It should
only be used to perform path operations, and never for file operations.
"""
return pathlib.Path(target).absolute()
def close(self) -> None:
"""Clean up any temporary files."""
self._tscb.flush()
@ -151,7 +161,7 @@ class Packer:
self.close()
@property
def output_path(self) -> pathlib.Path:
def output_path(self) -> pathlib.PurePath:
"""The path of the packed blend file in the target directory."""
assert self._output_path is not None
return self._output_path
@ -217,7 +227,7 @@ class Packer:
# The blendfile that we pack is generally not its own dependency, so
# we have to explicitly add it to the _packed_paths.
bfile_path = self.blendfile.absolute()
bfile_pp = self.target / bfile_path.relative_to(self.project)
bfile_pp = self._target_path / bfile_path.relative_to(self.project)
self._output_path = bfile_pp
self._progress_cb.pack_start()
@ -301,7 +311,7 @@ class Packer:
self._new_location_paths.add(asset_path)
else:
log.debug('%s can keep using %s', bfile_path, usage.asset_path)
asset_pp = self.target / asset_path.relative_to(self.project)
asset_pp = self._target_path / asset_path.relative_to(self.project)
act.new_path = asset_pp
def _find_new_paths(self):
@ -311,7 +321,7 @@ class Packer:
act = self._actions[path]
assert isinstance(act, AssetAction)
# Like a join, but ignoring the fact that 'path' is absolute.
act.new_path = pathlib.Path(self.target, '_outside_project', *path.parts[1:])
act.new_path = pathlib.Path(self._target_path, '_outside_project', *path.parts[1:])
def _group_rewrites(self) -> None:
"""For each blend file, collect which fields need rewriting.
@ -515,7 +525,7 @@ class Packer:
def _send_to_target(self,
asset_path: pathlib.Path,
target: pathlib.Path,
target: pathlib.PurePath,
may_move=False):
if self.noop:
print('%s -> %s' % (asset_path, target))
@ -542,6 +552,7 @@ class Packer:
with infopath.open('wt', encoding='utf8') as infofile:
print('This is a Blender Asset Tracer pack.', file=infofile)
print('Start by opening the following blend file:', file=infofile)
print(' %s' % self._output_path.relative_to(self.target).as_posix(), file=infofile)
print(' %s' % self._output_path.relative_to(self._target_path).as_posix(),
file=infofile)
self._file_transferer.queue_move(infopath, self.target / infoname)
self._file_transferer.queue_move(infopath, self._target_path / infoname)

3
blender_asset_tracer/pack/filesystem.py
View File

@ -59,11 +59,12 @@ class FileCopier(transfer.FileTransferer):
pool = multiprocessing.pool.ThreadPool(processes=self.transfer_threads)
for src, dst, act in self.iter_queue():
for src, pure_dst, act in self.iter_queue():
try:
if self._error.is_set() or self._abort.is_set():
raise AbortTransfer()
dst = pathlib.Path(pure_dst)
if self._skip_file(src, dst, act):
continue

12
blender_asset_tracer/pack/progress.py
View File

@ -38,7 +38,7 @@ class Callback(blender_asset_tracer.trace.progress.Callback):
"""Called when packing starts."""
def pack_done(self,
output_blendfile: pathlib.Path,
output_blendfile: pathlib.PurePath,
missing_files: typing.Set[pathlib.Path]) -> None:
"""Called when packing is done."""
@ -57,10 +57,10 @@ class Callback(blender_asset_tracer.trace.progress.Callback):
def rewrite_blendfile(self, orig_filename: pathlib.Path) -> None:
"""Called for every rewritten blendfile."""
def transfer_file(self, src: pathlib.Path, dst: pathlib.Path) -> None:
def transfer_file(self, src: pathlib.Path, dst: pathlib.PurePath) -> None:
"""Called when a file transfer starts."""
def transfer_file_skipped(self, src: pathlib.Path, dst: pathlib.Path) -> None:
def transfer_file_skipped(self, src: pathlib.Path, dst: pathlib.PurePath) -> None:
"""Called when a file is skipped because it already exists."""
def transfer_progress(self, total_bytes: int, transferred_bytes: int) -> None:
@ -105,7 +105,7 @@ class ThreadSafeCallback(Callback):
self._queue(self.wrapped.pack_start)
def pack_done(self,
output_blendfile: pathlib.Path,
output_blendfile: pathlib.PurePath,
missing_files: typing.Set[pathlib.Path]) -> None:
self._queue(self.wrapped.pack_done, output_blendfile, missing_files)
@ -115,10 +115,10 @@ class ThreadSafeCallback(Callback):
def trace_asset(self, filename: pathlib.Path) -> None:
self._queue(self.wrapped.trace_asset, filename)
def transfer_file(self, src: pathlib.Path, dst: pathlib.Path) -> None:
def transfer_file(self, src: pathlib.Path, dst: pathlib.PurePath) -> None:
self._queue(self.wrapped.transfer_file, src, dst)
def transfer_file_skipped(self, src: pathlib.Path, dst: pathlib.Path) -> None:
def transfer_file_skipped(self, src: pathlib.Path, dst: pathlib.PurePath) -> None:
self._queue(self.wrapped.transfer_file_skipped, src, dst)
def transfer_progress(self, total_bytes: int, transferred_bytes: int) -> None:

2
blender_asset_tracer/pack/s3.py
View File

@ -120,7 +120,7 @@ class S3Transferrer(transfer.FileTransferer):
if files_skipped:
log.info('Skipped %d files', files_skipped)
def upload_file(self, src: pathlib.Path, dst: pathlib.Path) -> bool:
def upload_file(self, src: pathlib.Path, dst: pathlib.PurePath) -> bool:
"""Upload a file to an S3 bucket.
The first part of 'dst' is used as the bucket name, the remained as the

6
blender_asset_tracer/pack/transfer.py
View File

@ -44,7 +44,7 @@ class Action(enum.Enum):
MOVE = 2
QueueItem = typing.Tuple[pathlib.Path, pathlib.Path, Action]
QueueItem = typing.Tuple[pathlib.Path, pathlib.PurePath, Action]
class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
@ -82,7 +82,7 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
def run(self):
"""Perform actual file transfer in a thread."""
def queue_copy(self, src: pathlib.Path, dst: pathlib.Path):
def queue_copy(self, src: pathlib.Path, dst: pathlib.PurePath):
"""Queue a copy action from 'src' to 'dst'."""
assert not self.done.is_set(), 'Queueing not allowed after done_and_join() was called'
assert not self._abort.is_set(), 'Queueing not allowed after abort_and_join() was called'
@ -91,7 +91,7 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
self.queue.put((src, dst, Action.COPY))
self.total_queued_bytes += src.stat().st_size
def queue_move(self, src: pathlib.Path, dst: pathlib.Path):
def queue_move(self, src: pathlib.Path, dst: pathlib.PurePath):
"""Queue a move action from 'src' to 'dst'."""
assert not self.done.is_set(), 'Queueing not allowed after done_and_join() was called'
assert not self._abort.is_set(), 'Queueing not allowed after abort_and_join() was called'

5
blender_asset_tracer/pack/zipped.py
View File

@ -37,7 +37,8 @@ class ZipPacker(Packer):
"""Creates a zipped BAT Pack instead of a directory."""
def _create_file_transferer(self) -> transfer.FileTransferer:
return ZipTransferrer(self.target.absolute())
target_path = pathlib.Path(self._target_path)
return ZipTransferrer(target_path.absolute())
class ZipTransferrer(transfer.FileTransferer):
@ -61,7 +62,7 @@ class ZipTransferrer(transfer.FileTransferer):
for src, dst, act in self.iter_queue():
assert src.is_absolute(), 'expecting only absolute paths, not %r' % src
dst = dst.absolute()
dst = pathlib.Path(dst).absolute()
try:
relpath = dst.relative_to(zippath)

12
tests/test_bpathlib.py
View File

@ -87,25 +87,25 @@ class BlendPathTest(unittest.TestCase):
def test_mkrelative(self):
self.assertEqual(b'//asset.png', BlendPath.mkrelative(
Path('/path/to/asset.png'),
Path('/path/to/bfile.blend'),
PurePosixPath('/path/to/bfile.blend'),
))
self.assertEqual(b'//to/asset.png', BlendPath.mkrelative(
Path('/path/to/asset.png'),
Path('/path/bfile.blend'),
PurePosixPath('/path/bfile.blend'),
))
self.assertEqual(b'//../of/asset.png', BlendPath.mkrelative(
Path('/path/of/asset.png'),
Path('/path/to/bfile.blend'),
PurePosixPath('/path/to/bfile.blend'),
))
self.assertEqual(b'//../../path/of/asset.png', BlendPath.mkrelative(
Path('/path/of/asset.png'),
Path('/some/weird/bfile.blend'),
PurePosixPath('/some/weird/bfile.blend'),
))
self.assertEqual(b'//very/very/very/very/very/deep/asset.png', BlendPath.mkrelative(
Path('/path/to/very/very/very/very/very/deep/asset.png'),
Path('/path/to/bfile.blend'),
PurePosixPath('/path/to/bfile.blend'),
))
self.assertEqual(b'//../../../../../../../../shallow/asset.png', BlendPath.mkrelative(
Path('/shallow/asset.png'),
Path('/path/to/very/very/very/very/very/deep/bfile.blend'),
PurePosixPath('/path/to/very/very/very/very/very/deep/bfile.blend'),
))