diff --git a/blender_asset_tracer/bpathlib.py b/blender_asset_tracer/bpathlib.py index 4bd0c22..1a2fde6 100644 --- a/blender_asset_tracer/bpathlib.py +++ b/blender_asset_tracer/bpathlib.py @@ -41,11 +41,17 @@ class BlendPath(bytes): return super().__new__(cls, path.replace(b'\\', b'/')) @classmethod - def mkrelative(cls, asset_path: pathlib.Path, bfile_path: pathlib.Path) -> 'BlendPath': - """Construct a BlendPath to the asset relative to the blend file.""" + def mkrelative(cls, asset_path: pathlib.Path, bfile_path: pathlib.PurePath) -> 'BlendPath': + """Construct a BlendPath to the asset relative to the blend file. + + Assumes that bfile_path is absolute. + """ from collections import deque - bdir_parts = deque(bfile_path.absolute().parent.parts) + assert bfile_path.is_absolute(), \ + 'BlendPath().mkrelative(bfile_path=%r) should get absolute bfile_path' % bfile_path + + bdir_parts = deque(bfile_path.parent.parts) asset_parts = deque(asset_path.absolute().parts) # Remove matching initial parts. What is left in bdir_parts represents diff --git a/blender_asset_tracer/cli/pack.py b/blender_asset_tracer/cli/pack.py index 894627f..c598209 100644 --- a/blender_asset_tracer/cli/pack.py +++ b/blender_asset_tracer/cli/pack.py @@ -36,7 +36,7 @@ def add_parser(subparsers): parser.set_defaults(func=cli_pack) parser.add_argument('blendfile', type=pathlib.Path, help='The Blend file to pack.') - parser.add_argument('target', type=pathlib.Path, + parser.add_argument('target', type=str, help='The target can be a directory, a ZIP file (does not have to exist ' "yet, just use 'something.zip' as target), or a URL of S3 storage " '(s3://endpoint/path).') @@ -76,9 +76,8 @@ def cli_pack(args): raise SystemExit(1) -def create_packer(args, bpath: pathlib.Path, ppath: pathlib.Path, - tpath: pathlib.Path) -> pack.Packer: - if str(tpath).startswith('s3:/'): +def create_packer(args, bpath: pathlib.Path, ppath: pathlib.Path, target: str) -> pack.Packer: + if target.startswith('s3:/'): if args.noop: raise ValueError('S3 uploader does not support no-op.') @@ -88,18 +87,18 @@ def create_packer(args, bpath: pathlib.Path, ppath: pathlib.Path, if args.relative_only: raise ValueError('S3 uploader does not support the --relative-only option') - packer = create_s3packer(bpath, ppath, tpath) - elif tpath.suffix.lower() == '.zip': + packer = create_s3packer(bpath, ppath, pathlib.PurePosixPath(target)) + elif target.lower().endswith('.zip'): from blender_asset_tracer.pack import zipped if args.compress: raise ValueError('ZIP packer does not support on-the-fly compression') - packer = zipped.ZipPacker(bpath, ppath, tpath, noop=args.noop, + packer = zipped.ZipPacker(bpath, ppath, target, noop=args.noop, relative_only=args.relative_only) else: - packer = pack.Packer(bpath, ppath, tpath, noop=args.noop, compress=args.compress, - relative_only=args.relative_only) + packer = pack.Packer(bpath, ppath, target, noop=args.noop, + compress=args.compress, relative_only=args.relative_only) if args.exclude: # args.exclude is a list, due to nargs='*', so we have to split and flatten. @@ -123,7 +122,7 @@ def create_s3packer(bpath, ppath, tpath) -> pack.Packer: return s3.S3Packer(bpath, ppath, tpath, endpoint=endpoint) -def paths_from_cli(args) -> typing.Tuple[pathlib.Path, pathlib.Path, pathlib.Path]: +def paths_from_cli(args) -> typing.Tuple[pathlib.Path, pathlib.Path, str]: """Return paths to blendfile, project, and pack target. Calls sys.exit() if anything is wrong. diff --git a/blender_asset_tracer/pack/__init__.py b/blender_asset_tracer/pack/__init__.py index 622ff05..c272288 100644 --- a/blender_asset_tracer/pack/__init__.py +++ b/blender_asset_tracer/pack/__init__.py @@ -50,7 +50,7 @@ class AssetAction: (if the asset is a blend file) or in another blend file. """ - self.new_path = None # type: typing.Optional[pathlib.Path] + self.new_path = None # type: typing.Optional[pathlib.PurePath] """Absolute path to the asset in the BAT Pack. This path may not exist on the local file system at all, for example @@ -96,7 +96,7 @@ class Packer: def __init__(self, bfile: pathlib.Path, project: pathlib.Path, - target: pathlib.Path, + target: str, *, noop=False, compress=False, @@ -104,6 +104,7 @@ class Packer: self.blendfile = bfile self.project = project self.target = target + self._target_path = self._make_target_path(target) self.noop = noop self.compress = compress self.relative_only = relative_only @@ -128,7 +129,7 @@ class Packer: # type: typing.DefaultDict[pathlib.Path, AssetAction] self.missing_files = set() # type: typing.Set[pathlib.Path] self._new_location_paths = set() # type: typing.Set[pathlib.Path] - self._output_path = None # type: typing.Optional[pathlib.Path] + self._output_path = None # type: typing.Optional[pathlib.PurePath] # Filled by execute() self._file_transferer = None # type: typing.Optional[transfer.FileTransferer] @@ -139,6 +140,15 @@ class Packer: self._tmpdir = tempfile.TemporaryDirectory(prefix='bat-', suffix='-batpack') self._rewrite_in = pathlib.Path(self._tmpdir.name) + def _make_target_path(self, target: str) -> pathlib.PurePath: + """Return a Path for the given target. + + This can be the target directory itself, but can also be a non-existent + directory if the target doesn't support direct file access. It should + only be used to perform path operations, and never for file operations. + """ + return pathlib.Path(target).absolute() + def close(self) -> None: """Clean up any temporary files.""" self._tscb.flush() @@ -151,7 +161,7 @@ class Packer: self.close() @property - def output_path(self) -> pathlib.Path: + def output_path(self) -> pathlib.PurePath: """The path of the packed blend file in the target directory.""" assert self._output_path is not None return self._output_path @@ -217,7 +227,7 @@ class Packer: # The blendfile that we pack is generally not its own dependency, so # we have to explicitly add it to the _packed_paths. bfile_path = self.blendfile.absolute() - bfile_pp = self.target / bfile_path.relative_to(self.project) + bfile_pp = self._target_path / bfile_path.relative_to(self.project) self._output_path = bfile_pp self._progress_cb.pack_start() @@ -301,7 +311,7 @@ class Packer: self._new_location_paths.add(asset_path) else: log.debug('%s can keep using %s', bfile_path, usage.asset_path) - asset_pp = self.target / asset_path.relative_to(self.project) + asset_pp = self._target_path / asset_path.relative_to(self.project) act.new_path = asset_pp def _find_new_paths(self): @@ -311,7 +321,7 @@ class Packer: act = self._actions[path] assert isinstance(act, AssetAction) # Like a join, but ignoring the fact that 'path' is absolute. - act.new_path = pathlib.Path(self.target, '_outside_project', *path.parts[1:]) + act.new_path = pathlib.Path(self._target_path, '_outside_project', *path.parts[1:]) def _group_rewrites(self) -> None: """For each blend file, collect which fields need rewriting. @@ -515,7 +525,7 @@ class Packer: def _send_to_target(self, asset_path: pathlib.Path, - target: pathlib.Path, + target: pathlib.PurePath, may_move=False): if self.noop: print('%s -> %s' % (asset_path, target)) @@ -542,6 +552,7 @@ class Packer: with infopath.open('wt', encoding='utf8') as infofile: print('This is a Blender Asset Tracer pack.', file=infofile) print('Start by opening the following blend file:', file=infofile) - print(' %s' % self._output_path.relative_to(self.target).as_posix(), file=infofile) + print(' %s' % self._output_path.relative_to(self._target_path).as_posix(), + file=infofile) - self._file_transferer.queue_move(infopath, self.target / infoname) + self._file_transferer.queue_move(infopath, self._target_path / infoname) diff --git a/blender_asset_tracer/pack/filesystem.py b/blender_asset_tracer/pack/filesystem.py index c05de6c..74b672e 100644 --- a/blender_asset_tracer/pack/filesystem.py +++ b/blender_asset_tracer/pack/filesystem.py @@ -59,11 +59,12 @@ class FileCopier(transfer.FileTransferer): pool = multiprocessing.pool.ThreadPool(processes=self.transfer_threads) - for src, dst, act in self.iter_queue(): + for src, pure_dst, act in self.iter_queue(): try: if self._error.is_set() or self._abort.is_set(): raise AbortTransfer() + dst = pathlib.Path(pure_dst) if self._skip_file(src, dst, act): continue diff --git a/blender_asset_tracer/pack/progress.py b/blender_asset_tracer/pack/progress.py index 82b10f2..51edd47 100644 --- a/blender_asset_tracer/pack/progress.py +++ b/blender_asset_tracer/pack/progress.py @@ -38,7 +38,7 @@ class Callback(blender_asset_tracer.trace.progress.Callback): """Called when packing starts.""" def pack_done(self, - output_blendfile: pathlib.Path, + output_blendfile: pathlib.PurePath, missing_files: typing.Set[pathlib.Path]) -> None: """Called when packing is done.""" @@ -57,10 +57,10 @@ class Callback(blender_asset_tracer.trace.progress.Callback): def rewrite_blendfile(self, orig_filename: pathlib.Path) -> None: """Called for every rewritten blendfile.""" - def transfer_file(self, src: pathlib.Path, dst: pathlib.Path) -> None: + def transfer_file(self, src: pathlib.Path, dst: pathlib.PurePath) -> None: """Called when a file transfer starts.""" - def transfer_file_skipped(self, src: pathlib.Path, dst: pathlib.Path) -> None: + def transfer_file_skipped(self, src: pathlib.Path, dst: pathlib.PurePath) -> None: """Called when a file is skipped because it already exists.""" def transfer_progress(self, total_bytes: int, transferred_bytes: int) -> None: @@ -105,7 +105,7 @@ class ThreadSafeCallback(Callback): self._queue(self.wrapped.pack_start) def pack_done(self, - output_blendfile: pathlib.Path, + output_blendfile: pathlib.PurePath, missing_files: typing.Set[pathlib.Path]) -> None: self._queue(self.wrapped.pack_done, output_blendfile, missing_files) @@ -115,10 +115,10 @@ class ThreadSafeCallback(Callback): def trace_asset(self, filename: pathlib.Path) -> None: self._queue(self.wrapped.trace_asset, filename) - def transfer_file(self, src: pathlib.Path, dst: pathlib.Path) -> None: + def transfer_file(self, src: pathlib.Path, dst: pathlib.PurePath) -> None: self._queue(self.wrapped.transfer_file, src, dst) - def transfer_file_skipped(self, src: pathlib.Path, dst: pathlib.Path) -> None: + def transfer_file_skipped(self, src: pathlib.Path, dst: pathlib.PurePath) -> None: self._queue(self.wrapped.transfer_file_skipped, src, dst) def transfer_progress(self, total_bytes: int, transferred_bytes: int) -> None: diff --git a/blender_asset_tracer/pack/s3.py b/blender_asset_tracer/pack/s3.py index bee5e09..d12c54d 100644 --- a/blender_asset_tracer/pack/s3.py +++ b/blender_asset_tracer/pack/s3.py @@ -120,7 +120,7 @@ class S3Transferrer(transfer.FileTransferer): if files_skipped: log.info('Skipped %d files', files_skipped) - def upload_file(self, src: pathlib.Path, dst: pathlib.Path) -> bool: + def upload_file(self, src: pathlib.Path, dst: pathlib.PurePath) -> bool: """Upload a file to an S3 bucket. The first part of 'dst' is used as the bucket name, the remained as the diff --git a/blender_asset_tracer/pack/transfer.py b/blender_asset_tracer/pack/transfer.py index 35baa14..06e5ed7 100644 --- a/blender_asset_tracer/pack/transfer.py +++ b/blender_asset_tracer/pack/transfer.py @@ -44,7 +44,7 @@ class Action(enum.Enum): MOVE = 2 -QueueItem = typing.Tuple[pathlib.Path, pathlib.Path, Action] +QueueItem = typing.Tuple[pathlib.Path, pathlib.PurePath, Action] class FileTransferer(threading.Thread, metaclass=abc.ABCMeta): @@ -82,7 +82,7 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta): def run(self): """Perform actual file transfer in a thread.""" - def queue_copy(self, src: pathlib.Path, dst: pathlib.Path): + def queue_copy(self, src: pathlib.Path, dst: pathlib.PurePath): """Queue a copy action from 'src' to 'dst'.""" assert not self.done.is_set(), 'Queueing not allowed after done_and_join() was called' assert not self._abort.is_set(), 'Queueing not allowed after abort_and_join() was called' @@ -91,7 +91,7 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta): self.queue.put((src, dst, Action.COPY)) self.total_queued_bytes += src.stat().st_size - def queue_move(self, src: pathlib.Path, dst: pathlib.Path): + def queue_move(self, src: pathlib.Path, dst: pathlib.PurePath): """Queue a move action from 'src' to 'dst'.""" assert not self.done.is_set(), 'Queueing not allowed after done_and_join() was called' assert not self._abort.is_set(), 'Queueing not allowed after abort_and_join() was called' diff --git a/blender_asset_tracer/pack/zipped.py b/blender_asset_tracer/pack/zipped.py index d558515..3042891 100644 --- a/blender_asset_tracer/pack/zipped.py +++ b/blender_asset_tracer/pack/zipped.py @@ -37,7 +37,8 @@ class ZipPacker(Packer): """Creates a zipped BAT Pack instead of a directory.""" def _create_file_transferer(self) -> transfer.FileTransferer: - return ZipTransferrer(self.target.absolute()) + target_path = pathlib.Path(self._target_path) + return ZipTransferrer(target_path.absolute()) class ZipTransferrer(transfer.FileTransferer): @@ -61,7 +62,7 @@ class ZipTransferrer(transfer.FileTransferer): for src, dst, act in self.iter_queue(): assert src.is_absolute(), 'expecting only absolute paths, not %r' % src - dst = dst.absolute() + dst = pathlib.Path(dst).absolute() try: relpath = dst.relative_to(zippath) diff --git a/tests/test_bpathlib.py b/tests/test_bpathlib.py index 165fafa..09be835 100644 --- a/tests/test_bpathlib.py +++ b/tests/test_bpathlib.py @@ -87,25 +87,25 @@ class BlendPathTest(unittest.TestCase): def test_mkrelative(self): self.assertEqual(b'//asset.png', BlendPath.mkrelative( Path('/path/to/asset.png'), - Path('/path/to/bfile.blend'), + PurePosixPath('/path/to/bfile.blend'), )) self.assertEqual(b'//to/asset.png', BlendPath.mkrelative( Path('/path/to/asset.png'), - Path('/path/bfile.blend'), + PurePosixPath('/path/bfile.blend'), )) self.assertEqual(b'//../of/asset.png', BlendPath.mkrelative( Path('/path/of/asset.png'), - Path('/path/to/bfile.blend'), + PurePosixPath('/path/to/bfile.blend'), )) self.assertEqual(b'//../../path/of/asset.png', BlendPath.mkrelative( Path('/path/of/asset.png'), - Path('/some/weird/bfile.blend'), + PurePosixPath('/some/weird/bfile.blend'), )) self.assertEqual(b'//very/very/very/very/very/deep/asset.png', BlendPath.mkrelative( Path('/path/to/very/very/very/very/very/deep/asset.png'), - Path('/path/to/bfile.blend'), + PurePosixPath('/path/to/bfile.blend'), )) self.assertEqual(b'//../../../../../../../../shallow/asset.png', BlendPath.mkrelative( Path('/shallow/asset.png'), - Path('/path/to/very/very/very/very/very/deep/bfile.blend'), + PurePosixPath('/path/to/very/very/very/very/very/deep/bfile.blend'), ))