diff --git a/blender_asset_tracer/cli/pack.py b/blender_asset_tracer/cli/pack.py index a54ee93..1b7bc63 100644 --- a/blender_asset_tracer/cli/pack.py +++ b/blender_asset_tracer/cli/pack.py @@ -36,7 +36,7 @@ def cli_pack(args): packer.execute() except pack.queued_copy.FileCopyError as ex: log.error("%d files couldn't be copied, starting with %s", - len(ex.files_not_copied), ex.files_not_copied[0]) + len(ex.files_remaining), ex.files_remaining[0]) raise SystemExit(1) diff --git a/blender_asset_tracer/pack/__init__.py b/blender_asset_tracer/pack/__init__.py index 565adc0..d187ec9 100644 --- a/blender_asset_tracer/pack/__init__.py +++ b/blender_asset_tracer/pack/__init__.py @@ -265,10 +265,8 @@ class Packer: # Copy the asset itself. packed_path = action.new_path read_path = action.read_from or asset_path - - # TODO(Sybren): if the asset is a rewritten blend file (and thus a copy), - # do a move instead of a copy. - self._copy_to_target(read_path, packed_path, fc) + self._send_to_target(read_path, packed_path, fc, + may_move=action.read_from is not None) # Copy its sequence dependencies. for usage in action.usages: @@ -284,15 +282,26 @@ class Packer: packed_base_dir = first_pp.parent for file_path in usage.files(): packed_path = packed_base_dir / file_path.name - self._copy_to_target(file_path, packed_path, fc) + # Assumption: assets in a sequence are never blend files. + self._send_to_target(file_path, packed_path, fc) # Assumption: all data blocks using this asset use it the same way. break - def _copy_to_target(self, asset_path: pathlib.Path, target: pathlib.Path, fc): + def _send_to_target(self, + asset_path: pathlib.Path, + target: pathlib.Path, + fc: queued_copy.FileCopier, + may_move=False): if self.noop: print('%s → %s' % (asset_path, target)) self._file_count += 1 return - log.debug('Queueing copy of %s', asset_path) - fc.queue(asset_path, target) + + verb = 'move' if may_move else 'copy' + log.debug('Queueing %s of %s', verb, asset_path) + + if may_move: + fc.queue_move(asset_path, target) + else: + fc.queue_copy(asset_path, target) diff --git a/blender_asset_tracer/pack/queued_copy.py b/blender_asset_tracer/pack/queued_copy.py index 78f6d4f..ba0def8 100644 --- a/blender_asset_tracer/pack/queued_copy.py +++ b/blender_asset_tracer/pack/queued_copy.py @@ -1,3 +1,4 @@ +import enum import logging import threading import pathlib @@ -9,15 +10,20 @@ log = logging.getLogger(__name__) class FileCopyError(IOError): - """Raised when one or more files could not be copied.""" + """Raised when one or more files could not be transferred.""" - def __init__(self, message, files_not_copied: typing.List[pathlib.Path]) -> None: + def __init__(self, message, files_remaining: typing.List[pathlib.Path]) -> None: super().__init__(message) - self.files_not_copied = files_not_copied + self.files_remaining = files_remaining + + +class Action(enum.Enum): + COPY = 1 + MOVE = 2 class FileCopier(threading.Thread): - """Copies files in directory order.""" + """Copies or moves files in source directory order.""" def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -31,39 +37,48 @@ class FileCopier(threading.Thread): # maxsize=100 is just a guess as to a reasonable upper limit. When this limit # is reached, the main thread will simply block while waiting for this thread # to finish copying a file. - self.file_copy_queue = queue.PriorityQueue( - maxsize=100) # type: queue.PriorityQueue[typing.Tuple[pathlib.Path, pathlib.Path]] - self.file_copy_done = threading.Event() + self.queue = queue.PriorityQueue(maxsize=100) \ + # type: queue.PriorityQueue[typing.Tuple[pathlib.Path, pathlib.Path, Action]] + self.done = threading.Event() - def queue(self, src: pathlib.Path, dst: pathlib.Path): + def queue_copy(self, src: pathlib.Path, dst: pathlib.Path): """Queue a copy action from 'src' to 'dst'.""" - self.file_copy_queue.put((src, dst)) + self.queue.put((src, dst, Action.COPY)) + + def queue_move(self, src: pathlib.Path, dst: pathlib.Path): + """Queue a move action from 'src' to 'dst'.""" + self.queue.put((src, dst, Action.MOVE)) def done_and_join(self): """Indicate all files have been queued, and wait until done.""" - self.file_copy_done.set() + self.done.set() self.join() - if not self.file_copy_queue.empty(): + if not self.queue.empty(): # Flush the queue so that we can report which files weren't copied yet. files_remaining = [] - while not self.file_copy_queue.empty(): - src, dst = self.file_copy_queue.get_nowait() + while not self.queue.empty(): + src, dst = self.queue.get_nowait() files_remaining.append(src) assert files_remaining - raise FileCopyError("%d files couldn't be copied" % len(files_remaining), + raise FileCopyError("%d files couldn't be transferred" % len(files_remaining), files_remaining) def run(self): - files_copied = 0 + files_transferred = 0 files_skipped = 0 + transfer_funcs = { + Action.COPY: shutil.copy, + Action.MOVE: shutil.move, + } + while True: try: - src, dst = self.file_copy_queue.get(timeout=0.1) + src, dst, act = self.queue.get(timeout=0.1) except queue.Empty: - if self.file_copy_done.is_set(): + if self.done.is_set(): break continue @@ -72,27 +87,33 @@ class FileCopier(threading.Thread): st_src = src.stat() st_dst = dst.stat() if st_dst.st_size == st_src.st_size and st_dst.st_mtime >= st_src.st_mtime: - log.info('Skipping %s; already exists', src) + log.info('SKIP %s; already exists', src) + if act == Action.MOVE: + log.debug('Deleting %s', src) + src.unlink() files_skipped += 1 continue - log.info('Copying %s → %s', src, dst) + log.info('%s %s → %s', act.name, src, dst) dst.parent.mkdir(parents=True, exist_ok=True) + # TODO(Sybren): when we target Py 3.6+, remove the str() calls. - shutil.copy(str(src), str(dst)) - files_copied += 1 + transfer = transfer_funcs[act] + transfer(str(src), str(dst)) + + files_transferred += 1 except Exception: # We have to catch exceptions in a broad way, as this is running in # a separate thread, and exceptions won't otherwise be seen. - log.exception('Error copying %s to %s', src, dst) + log.exception('Error transferring %s to %s', src, dst) # Put the files to copy back into the queue, and abort. This allows # the main thread to inspect the queue and see which files were not # copied. The one we just failed (due to this exception) should also # be reported there. - self.file_copy_queue.put((src, dst)) + self.queue.put((src, dst, act)) return - if files_copied: - log.info('Copied %d files', files_copied) + if files_transferred: + log.info('Transferred %d files', files_transferred) if files_skipped: log.info('Skipped %d files', files_skipped)