autour_de_minuit/blender-asset-tracer
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Move rewritten blend files instead of copying them

Browse Source 
This will cause a nice speedup if the temporary directory happens to be on
the same file system as the target directory for the BAT Pack. This can be
influenced by setting an environment variable; see the documentation for
[tempfile.gettempdir](https://docs.python.org/3/library/tempfile.html#tempfile.gettempdir)
This commit is contained in:
Sybren A. Stüvel 2018-03-09 12:46:22 +01:00
parent 824ca4bcb7
commit ac3f6142b3
3 changed files with 64 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
blender_asset_tracer/cli/pack.py
View File

@ -36,7 +36,7 @@ def cli_pack(args):
packer.execute() packer.execute()
except pack.queued_copy.FileCopyError as ex: except pack.queued_copy.FileCopyError as ex:
log.error("%d files couldn't be copied, starting with %s", log.error("%d files couldn't be copied, starting with %s",
len(ex.files_not_copied), ex.files_not_copied[0]) len(ex.files_remaining), ex.files_remaining[0])
raise SystemExit(1) raise SystemExit(1)

25
blender_asset_tracer/pack/__init__.py
View File

@ -265,10 +265,8 @@ class Packer:
# Copy the asset itself. # Copy the asset itself.
packed_path = action.new_path packed_path = action.new_path
read_path = action.read_from or asset_path read_path = action.read_from or asset_path
self._send_to_target(read_path, packed_path, fc,
# TODO(Sybren): if the asset is a rewritten blend file (and thus a copy), may_move=action.read_from is not None)
# do a move instead of a copy.
self._copy_to_target(read_path, packed_path, fc)
# Copy its sequence dependencies. # Copy its sequence dependencies.
for usage in action.usages: for usage in action.usages:
@ -284,15 +282,26 @@ class Packer:
packed_base_dir = first_pp.parent packed_base_dir = first_pp.parent
for file_path in usage.files(): for file_path in usage.files():
packed_path = packed_base_dir / file_path.name packed_path = packed_base_dir / file_path.name
self._copy_to_target(file_path, packed_path, fc) # Assumption: assets in a sequence are never blend files.
self._send_to_target(file_path, packed_path, fc)
# Assumption: all data blocks using this asset use it the same way. # Assumption: all data blocks using this asset use it the same way.
break break
def _copy_to_target(self, asset_path: pathlib.Path, target: pathlib.Path, fc): def _send_to_target(self,
asset_path: pathlib.Path,
target: pathlib.Path,
fc: queued_copy.FileCopier,
may_move=False):
if self.noop: if self.noop:
print('%s → %s' % (asset_path, target)) print('%s → %s' % (asset_path, target))
self._file_count += 1 self._file_count += 1
return return
log.debug('Queueing copy of %s', asset_path)
fc.queue(asset_path, target) verb = 'move' if may_move else 'copy'
log.debug('Queueing %s of %s', verb, asset_path)
if may_move:
fc.queue_move(asset_path, target)
else:
fc.queue_copy(asset_path, target)

71
blender_asset_tracer/pack/queued_copy.py
View File

@ -1,3 +1,4 @@
import enum
import logging import logging
import threading import threading
import pathlib import pathlib
@ -9,15 +10,20 @@ log = logging.getLogger(__name__)
class FileCopyError(IOError): class FileCopyError(IOError):
"""Raised when one or more files could not be copied.""" """Raised when one or more files could not be transferred."""
def __init__(self, message, files_not_copied: typing.List[pathlib.Path]) -> None: def __init__(self, message, files_remaining: typing.List[pathlib.Path]) -> None:
super().__init__(message) super().__init__(message)
self.files_not_copied = files_not_copied self.files_remaining = files_remaining
class Action(enum.Enum):
COPY = 1
MOVE = 2
class FileCopier(threading.Thread): class FileCopier(threading.Thread):
"""Copies files in directory order.""" """Copies or moves files in source directory order."""
def __init__(self, *args, **kwargs) -> None: def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
@ -31,39 +37,48 @@ class FileCopier(threading.Thread):
# maxsize=100 is just a guess as to a reasonable upper limit. When this limit # maxsize=100 is just a guess as to a reasonable upper limit. When this limit
# is reached, the main thread will simply block while waiting for this thread # is reached, the main thread will simply block while waiting for this thread
# to finish copying a file. # to finish copying a file.
self.file_copy_queue = queue.PriorityQueue( self.queue = queue.PriorityQueue(maxsize=100) \
maxsize=100) # type: queue.PriorityQueue[typing.Tuple[pathlib.Path, pathlib.Path]] # type: queue.PriorityQueue[typing.Tuple[pathlib.Path, pathlib.Path, Action]]
self.file_copy_done = threading.Event() self.done = threading.Event()
def queue(self, src: pathlib.Path, dst: pathlib.Path): def queue_copy(self, src: pathlib.Path, dst: pathlib.Path):
"""Queue a copy action from 'src' to 'dst'.""" """Queue a copy action from 'src' to 'dst'."""
self.file_copy_queue.put((src, dst)) self.queue.put((src, dst, Action.COPY))
def queue_move(self, src: pathlib.Path, dst: pathlib.Path):
"""Queue a move action from 'src' to 'dst'."""
self.queue.put((src, dst, Action.MOVE))
def done_and_join(self): def done_and_join(self):
"""Indicate all files have been queued, and wait until done.""" """Indicate all files have been queued, and wait until done."""
self.file_copy_done.set() self.done.set()
self.join() self.join()
if not self.file_copy_queue.empty(): if not self.queue.empty():
# Flush the queue so that we can report which files weren't copied yet. # Flush the queue so that we can report which files weren't copied yet.
files_remaining = [] files_remaining = []
while not self.file_copy_queue.empty(): while not self.queue.empty():
src, dst = self.file_copy_queue.get_nowait() src, dst = self.queue.get_nowait()
files_remaining.append(src) files_remaining.append(src)
assert files_remaining assert files_remaining
raise FileCopyError("%d files couldn't be copied" % len(files_remaining), raise FileCopyError("%d files couldn't be transferred" % len(files_remaining),
files_remaining) files_remaining)
def run(self): def run(self):
files_copied = 0 files_transferred = 0
files_skipped = 0 files_skipped = 0
transfer_funcs = {
Action.COPY: shutil.copy,
Action.MOVE: shutil.move,
}
while True: while True:
try: try:
src, dst = self.file_copy_queue.get(timeout=0.1) src, dst, act = self.queue.get(timeout=0.1)
except queue.Empty: except queue.Empty:
if self.file_copy_done.is_set(): if self.done.is_set():
break break
continue continue
@ -72,27 +87,33 @@ class FileCopier(threading.Thread):
st_src = src.stat() st_src = src.stat()
st_dst = dst.stat() st_dst = dst.stat()
if st_dst.st_size == st_src.st_size and st_dst.st_mtime >= st_src.st_mtime: if st_dst.st_size == st_src.st_size and st_dst.st_mtime >= st_src.st_mtime:
log.info('Skipping %s; already exists', src) log.info('SKIP %s; already exists', src)
if act == Action.MOVE:
log.debug('Deleting %s', src)
src.unlink()
files_skipped += 1 files_skipped += 1
continue continue
log.info('Copying %s → %s', src, dst) log.info('%s %s → %s', act.name, src, dst)
dst.parent.mkdir(parents=True, exist_ok=True) dst.parent.mkdir(parents=True, exist_ok=True)
# TODO(Sybren): when we target Py 3.6+, remove the str() calls. # TODO(Sybren): when we target Py 3.6+, remove the str() calls.
shutil.copy(str(src), str(dst)) transfer = transfer_funcs[act]
files_copied += 1 transfer(str(src), str(dst))
files_transferred += 1
except Exception: except Exception:
# We have to catch exceptions in a broad way, as this is running in # We have to catch exceptions in a broad way, as this is running in
# a separate thread, and exceptions won't otherwise be seen. # a separate thread, and exceptions won't otherwise be seen.
log.exception('Error copying %s to %s', src, dst) log.exception('Error transferring %s to %s', src, dst)
# Put the files to copy back into the queue, and abort. This allows # Put the files to copy back into the queue, and abort. This allows
# the main thread to inspect the queue and see which files were not # the main thread to inspect the queue and see which files were not
# copied. The one we just failed (due to this exception) should also # copied. The one we just failed (due to this exception) should also
# be reported there. # be reported there.
self.file_copy_queue.put((src, dst)) self.queue.put((src, dst, act))
return return
if files_copied: if files_transferred:
log.info('Copied %d files', files_copied) log.info('Transferred %d files', files_transferred)
if files_skipped: if files_skipped:
log.info('Skipped %d files', files_skipped) log.info('Skipped %d files', files_skipped)