Convert target path from Path to str & PurePath

The target path is just read as string from the CLI now, to allow more
complex targets (such as URLs) that don't directly map to a path.

The Packer subclass now handles the conversion from that string to a
`pathlib.PurePath`, and specific subclasses & transfer classes can convert
those to a `pathlib.Path` to perform actual filesystem operations when
necessary.

blender_asset_tracer/bpathlib.py

@@ -41,11 +41,17 @@ class BlendPath(bytes):
         return super().__new__(cls, path.replace(b'\\', b'/'))
 
     @classmethod
-    def mkrelative(cls, asset_path: pathlib.Path, bfile_path: pathlib.Path) -> 'BlendPath':
+    def mkrelative(cls, asset_path: pathlib.Path, bfile_path: pathlib.PurePath) -> 'BlendPath':
-        """Construct a BlendPath to the asset relative to the blend file."""
+        """Construct a BlendPath to the asset relative to the blend file.
+
+        Assumes that bfile_path is absolute.
+        """
         from collections import deque
 
-        bdir_parts = deque(bfile_path.absolute().parent.parts)
+        assert bfile_path.is_absolute(), \
+            'BlendPath().mkrelative(bfile_path=%r) should get absolute bfile_path' % bfile_path
+
+        bdir_parts = deque(bfile_path.parent.parts)
         asset_parts = deque(asset_path.absolute().parts)
 
         # Remove matching initial parts. What is left in bdir_parts represents

blender_asset_tracer/cli/pack.py

@@ -36,7 +36,7 @@ def add_parser(subparsers):
     parser.set_defaults(func=cli_pack)
     parser.add_argument('blendfile', type=pathlib.Path,
                         help='The Blend file to pack.')
-    parser.add_argument('target', type=pathlib.Path,
+    parser.add_argument('target', type=str,
                         help='The target can be a directory, a ZIP file (does not have to exist '
                              "yet, just use 'something.zip' as target), or a URL of S3 storage "
                              '(s3://endpoint/path).')
@@ -76,9 +76,8 @@ def cli_pack(args):
             raise SystemExit(1)
 
 
-def create_packer(args, bpath: pathlib.Path, ppath: pathlib.Path,
+def create_packer(args, bpath: pathlib.Path, ppath: pathlib.Path, target: str) -> pack.Packer:
-                  tpath: pathlib.Path) -> pack.Packer:
+    if target.startswith('s3:/'):
-    if str(tpath).startswith('s3:/'):
         if args.noop:
             raise ValueError('S3 uploader does not support no-op.')
 
@@ -88,18 +87,18 @@ def create_packer(args, bpath: pathlib.Path, ppath: pathlib.Path,
         if args.relative_only:
             raise ValueError('S3 uploader does not support the --relative-only option')
 
-        packer = create_s3packer(bpath, ppath, tpath)
+        packer = create_s3packer(bpath, ppath, pathlib.PurePosixPath(target))
-    elif tpath.suffix.lower() == '.zip':
+    elif target.lower().endswith('.zip'):
         from blender_asset_tracer.pack import zipped
 
         if args.compress:
             raise ValueError('ZIP packer does not support on-the-fly compression')
 
-        packer = zipped.ZipPacker(bpath, ppath, tpath, noop=args.noop,
+        packer = zipped.ZipPacker(bpath, ppath, target, noop=args.noop,
                                   relative_only=args.relative_only)
     else:
-        packer = pack.Packer(bpath, ppath, tpath, noop=args.noop, compress=args.compress,
+        packer = pack.Packer(bpath, ppath, target, noop=args.noop,
-                             relative_only=args.relative_only)
+                             compress=args.compress, relative_only=args.relative_only)
 
     if args.exclude:
         # args.exclude is a list, due to nargs='*', so we have to split and flatten.
@@ -123,7 +122,7 @@ def create_s3packer(bpath, ppath, tpath) -> pack.Packer:
     return s3.S3Packer(bpath, ppath, tpath, endpoint=endpoint)
 
 
-def paths_from_cli(args) -> typing.Tuple[pathlib.Path, pathlib.Path, pathlib.Path]:
+def paths_from_cli(args) -> typing.Tuple[pathlib.Path, pathlib.Path, str]:
     """Return paths to blendfile, project, and pack target.
 
     Calls sys.exit() if anything is wrong.

blender_asset_tracer/pack/__init__.py

@@ -50,7 +50,7 @@ class AssetAction:
         (if the asset is a blend file) or in another blend file.
         """
 
-        self.new_path = None  # type: typing.Optional[pathlib.Path]
+        self.new_path = None  # type: typing.Optional[pathlib.PurePath]
         """Absolute path to the asset in the BAT Pack.
 
         This path may not exist on the local file system at all, for example
@@ -96,7 +96,7 @@ class Packer:
     def __init__(self,
                  bfile: pathlib.Path,
                  project: pathlib.Path,
-                 target: pathlib.Path,
+                 target: str,
                  *,
                  noop=False,
                  compress=False,
@@ -104,6 +104,7 @@ class Packer:
         self.blendfile = bfile
         self.project = project
         self.target = target
+        self._target_path = self._make_target_path(target)
         self.noop = noop
         self.compress = compress
         self.relative_only = relative_only
@@ -128,7 +129,7 @@ class Packer:
             # type: typing.DefaultDict[pathlib.Path, AssetAction]
         self.missing_files = set()  # type: typing.Set[pathlib.Path]
         self._new_location_paths = set()  # type: typing.Set[pathlib.Path]
-        self._output_path = None  # type: typing.Optional[pathlib.Path]
+        self._output_path = None  # type: typing.Optional[pathlib.PurePath]
 
         # Filled by execute()
         self._file_transferer = None  # type: typing.Optional[transfer.FileTransferer]
@@ -139,6 +140,15 @@ class Packer:
         self._tmpdir = tempfile.TemporaryDirectory(prefix='bat-', suffix='-batpack')
         self._rewrite_in = pathlib.Path(self._tmpdir.name)
 
+    def _make_target_path(self, target: str) -> pathlib.PurePath:
+        """Return a Path for the given target.
+
+        This can be the target directory itself, but can also be a non-existent
+        directory if the target doesn't support direct file access. It should
+        only be used to perform path operations, and never for file operations.
+        """
+        return pathlib.Path(target).absolute()
+
     def close(self) -> None:
         """Clean up any temporary files."""
         self._tscb.flush()
@@ -151,7 +161,7 @@ class Packer:
         self.close()
 
     @property
-    def output_path(self) -> pathlib.Path:
+    def output_path(self) -> pathlib.PurePath:
         """The path of the packed blend file in the target directory."""
         assert self._output_path is not None
         return self._output_path
@@ -217,7 +227,7 @@ class Packer:
         # The blendfile that we pack is generally not its own dependency, so
         # we have to explicitly add it to the _packed_paths.
         bfile_path = self.blendfile.absolute()
-        bfile_pp = self.target / bfile_path.relative_to(self.project)
+        bfile_pp = self._target_path / bfile_path.relative_to(self.project)
         self._output_path = bfile_pp
 
         self._progress_cb.pack_start()
@@ -301,7 +311,7 @@ class Packer:
             self._new_location_paths.add(asset_path)
         else:
             log.debug('%s can keep using %s', bfile_path, usage.asset_path)
-            asset_pp = self.target / asset_path.relative_to(self.project)
+            asset_pp = self._target_path / asset_path.relative_to(self.project)
             act.new_path = asset_pp
 
     def _find_new_paths(self):
@@ -311,7 +321,7 @@ class Packer:
             act = self._actions[path]
             assert isinstance(act, AssetAction)
             # Like a join, but ignoring the fact that 'path' is absolute.
-            act.new_path = pathlib.Path(self.target, '_outside_project', *path.parts[1:])
+            act.new_path = pathlib.Path(self._target_path, '_outside_project', *path.parts[1:])
 
     def _group_rewrites(self) -> None:
         """For each blend file, collect which fields need rewriting.
@@ -515,7 +525,7 @@ class Packer:
 
     def _send_to_target(self,
                         asset_path: pathlib.Path,
-                        target: pathlib.Path,
+                        target: pathlib.PurePath,
                         may_move=False):
         if self.noop:
             print('%s -> %s' % (asset_path, target))
@@ -542,6 +552,7 @@ class Packer:
         with infopath.open('wt', encoding='utf8') as infofile:
             print('This is a Blender Asset Tracer pack.', file=infofile)
             print('Start by opening the following blend file:', file=infofile)
-            print('    %s' % self._output_path.relative_to(self.target).as_posix(), file=infofile)
+            print('    %s' % self._output_path.relative_to(self._target_path).as_posix(),
+                  file=infofile)
 
-        self._file_transferer.queue_move(infopath, self.target / infoname)
+        self._file_transferer.queue_move(infopath, self._target_path / infoname)

blender_asset_tracer/pack/filesystem.py

@@ -59,11 +59,12 @@ class FileCopier(transfer.FileTransferer):
 
         pool = multiprocessing.pool.ThreadPool(processes=self.transfer_threads)
 
-        for src, dst, act in self.iter_queue():
+        for src, pure_dst, act in self.iter_queue():
             try:
                 if self._error.is_set() or self._abort.is_set():
                     raise AbortTransfer()
 
+                dst = pathlib.Path(pure_dst)
                 if self._skip_file(src, dst, act):
                     continue
 

blender_asset_tracer/pack/progress.py

@@ -38,7 +38,7 @@ class Callback(blender_asset_tracer.trace.progress.Callback):
         """Called when packing starts."""
 
     def pack_done(self,
-                  output_blendfile: pathlib.Path,
+                  output_blendfile: pathlib.PurePath,
                   missing_files: typing.Set[pathlib.Path]) -> None:
         """Called when packing is done."""
 
@@ -57,10 +57,10 @@ class Callback(blender_asset_tracer.trace.progress.Callback):
     def rewrite_blendfile(self, orig_filename: pathlib.Path) -> None:
         """Called for every rewritten blendfile."""
 
-    def transfer_file(self, src: pathlib.Path, dst: pathlib.Path) -> None:
+    def transfer_file(self, src: pathlib.Path, dst: pathlib.PurePath) -> None:
         """Called when a file transfer starts."""
 
-    def transfer_file_skipped(self, src: pathlib.Path, dst: pathlib.Path) -> None:
+    def transfer_file_skipped(self, src: pathlib.Path, dst: pathlib.PurePath) -> None:
         """Called when a file is skipped because it already exists."""
 
     def transfer_progress(self, total_bytes: int, transferred_bytes: int) -> None:
@@ -105,7 +105,7 @@ class ThreadSafeCallback(Callback):
         self._queue(self.wrapped.pack_start)
 
     def pack_done(self,
-                  output_blendfile: pathlib.Path,
+                  output_blendfile: pathlib.PurePath,
                   missing_files: typing.Set[pathlib.Path]) -> None:
         self._queue(self.wrapped.pack_done, output_blendfile, missing_files)
 
@@ -115,10 +115,10 @@ class ThreadSafeCallback(Callback):
     def trace_asset(self, filename: pathlib.Path) -> None:
         self._queue(self.wrapped.trace_asset, filename)
 
-    def transfer_file(self, src: pathlib.Path, dst: pathlib.Path) -> None:
+    def transfer_file(self, src: pathlib.Path, dst: pathlib.PurePath) -> None:
         self._queue(self.wrapped.transfer_file, src, dst)
 
-    def transfer_file_skipped(self, src: pathlib.Path, dst: pathlib.Path) -> None:
+    def transfer_file_skipped(self, src: pathlib.Path, dst: pathlib.PurePath) -> None:
         self._queue(self.wrapped.transfer_file_skipped, src, dst)
 
     def transfer_progress(self, total_bytes: int, transferred_bytes: int) -> None:

blender_asset_tracer/pack/s3.py

@@ -120,7 +120,7 @@ class S3Transferrer(transfer.FileTransferer):
         if files_skipped:
             log.info('Skipped %d files', files_skipped)
 
-    def upload_file(self, src: pathlib.Path, dst: pathlib.Path) -> bool:
+    def upload_file(self, src: pathlib.Path, dst: pathlib.PurePath) -> bool:
         """Upload a file to an S3 bucket.
 
         The first part of 'dst' is used as the bucket name, the remained as the

blender_asset_tracer/pack/transfer.py

@@ -44,7 +44,7 @@ class Action(enum.Enum):
     MOVE = 2
 
 
-QueueItem = typing.Tuple[pathlib.Path, pathlib.Path, Action]
+QueueItem = typing.Tuple[pathlib.Path, pathlib.PurePath, Action]
 
 
 class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
@@ -82,7 +82,7 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
     def run(self):
         """Perform actual file transfer in a thread."""
 
-    def queue_copy(self, src: pathlib.Path, dst: pathlib.Path):
+    def queue_copy(self, src: pathlib.Path, dst: pathlib.PurePath):
         """Queue a copy action from 'src' to 'dst'."""
         assert not self.done.is_set(), 'Queueing not allowed after done_and_join() was called'
         assert not self._abort.is_set(), 'Queueing not allowed after abort_and_join() was called'
@@ -91,7 +91,7 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
         self.queue.put((src, dst, Action.COPY))
         self.total_queued_bytes += src.stat().st_size
 
-    def queue_move(self, src: pathlib.Path, dst: pathlib.Path):
+    def queue_move(self, src: pathlib.Path, dst: pathlib.PurePath):
         """Queue a move action from 'src' to 'dst'."""
         assert not self.done.is_set(), 'Queueing not allowed after done_and_join() was called'
         assert not self._abort.is_set(), 'Queueing not allowed after abort_and_join() was called'

blender_asset_tracer/pack/zipped.py

@@ -37,7 +37,8 @@ class ZipPacker(Packer):
     """Creates a zipped BAT Pack instead of a directory."""
 
     def _create_file_transferer(self) -> transfer.FileTransferer:
-        return ZipTransferrer(self.target.absolute())
+        target_path = pathlib.Path(self._target_path)
+        return ZipTransferrer(target_path.absolute())
 
 
 class ZipTransferrer(transfer.FileTransferer):
@@ -61,7 +62,7 @@ class ZipTransferrer(transfer.FileTransferer):
             for src, dst, act in self.iter_queue():
                 assert src.is_absolute(), 'expecting only absolute paths, not %r' % src
 
-                dst = dst.absolute()
+                dst = pathlib.Path(dst).absolute()
                 try:
                     relpath = dst.relative_to(zippath)
 

tests/test_bpathlib.py

@@ -87,25 +87,25 @@ class BlendPathTest(unittest.TestCase):
     def test_mkrelative(self):
         self.assertEqual(b'//asset.png', BlendPath.mkrelative(
             Path('/path/to/asset.png'),
-            Path('/path/to/bfile.blend'),
+            PurePosixPath('/path/to/bfile.blend'),
         ))
         self.assertEqual(b'//to/asset.png', BlendPath.mkrelative(
             Path('/path/to/asset.png'),
-            Path('/path/bfile.blend'),
+            PurePosixPath('/path/bfile.blend'),
         ))
         self.assertEqual(b'//../of/asset.png', BlendPath.mkrelative(
             Path('/path/of/asset.png'),
-            Path('/path/to/bfile.blend'),
+            PurePosixPath('/path/to/bfile.blend'),
         ))
         self.assertEqual(b'//../../path/of/asset.png', BlendPath.mkrelative(
             Path('/path/of/asset.png'),
-            Path('/some/weird/bfile.blend'),
+            PurePosixPath('/some/weird/bfile.blend'),
         ))
         self.assertEqual(b'//very/very/very/very/very/deep/asset.png', BlendPath.mkrelative(
             Path('/path/to/very/very/very/very/very/deep/asset.png'),
-            Path('/path/to/bfile.blend'),
+            PurePosixPath('/path/to/bfile.blend'),
         ))
         self.assertEqual(b'//../../../../../../../../shallow/asset.png', BlendPath.mkrelative(
             Path('/shallow/asset.png'),
-            Path('/path/to/very/very/very/very/very/deep/bfile.blend'),
+            PurePosixPath('/path/to/very/very/very/very/very/deep/bfile.blend'),
         ))