autour_de_minuit/blender-asset-tracer
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Move rewritten blend files instead of copying them

Browse Source 
This will cause a nice speedup if the temporary directory happens to be on
the same file system as the target directory for the BAT Pack. This can be
influenced by setting an environment variable; see the documentation for
[tempfile.gettempdir](https://docs.python.org/3/library/tempfile.html#tempfile.gettempdir)
This commit is contained in:
Sybren A. Stüvel 2018-03-09 12:46:22 +01:00
parent 824ca4bcb7
commit ac3f6142b3
3 changed files with 64 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
blender_asset_tracer/cli/pack.py
View File

@ -36,7 +36,7 @@ def cli_pack(args):
packer.execute()
except pack.queued_copy.FileCopyError as ex:
log.error("%d files couldn't be copied, starting with %s",
len(ex.files_not_copied), ex.files_not_copied[0])
len(ex.files_remaining), ex.files_remaining[0])
raise SystemExit(1)

25
blender_asset_tracer/pack/__init__.py
View File

@ -265,10 +265,8 @@ class Packer:
# Copy the asset itself.
packed_path = action.new_path
read_path = action.read_from or asset_path
# TODO(Sybren): if the asset is a rewritten blend file (and thus a copy),
# do a move instead of a copy.
self._copy_to_target(read_path, packed_path, fc)
self._send_to_target(read_path, packed_path, fc,
may_move=action.read_from is not None)
# Copy its sequence dependencies.
for usage in action.usages:
@ -284,15 +282,26 @@ class Packer:
packed_base_dir = first_pp.parent
for file_path in usage.files():
packed_path = packed_base_dir / file_path.name
self._copy_to_target(file_path, packed_path, fc)
# Assumption: assets in a sequence are never blend files.
self._send_to_target(file_path, packed_path, fc)
# Assumption: all data blocks using this asset use it the same way.
break
def _copy_to_target(self, asset_path: pathlib.Path, target: pathlib.Path, fc):
def _send_to_target(self,
asset_path: pathlib.Path,
target: pathlib.Path,
fc: queued_copy.FileCopier,
may_move=False):
if self.noop:
print('%s → %s' % (asset_path, target))
self._file_count += 1
return
log.debug('Queueing copy of %s', asset_path)
fc.queue(asset_path, target)
verb = 'move' if may_move else 'copy'
log.debug('Queueing %s of %s', verb, asset_path)
if may_move:
fc.queue_move(asset_path, target)
else:
fc.queue_copy(asset_path, target)

71
blender_asset_tracer/pack/queued_copy.py
View File

@ -1,3 +1,4 @@
import enum
import logging
import threading
import pathlib
@ -9,15 +10,20 @@ log = logging.getLogger(__name__)
class FileCopyError(IOError):
"""Raised when one or more files could not be copied."""
"""Raised when one or more files could not be transferred."""
def __init__(self, message, files_not_copied: typing.List[pathlib.Path]) -> None:
def __init__(self, message, files_remaining: typing.List[pathlib.Path]) -> None:
super().__init__(message)
self.files_not_copied = files_not_copied
self.files_remaining = files_remaining
class Action(enum.Enum):
COPY = 1
MOVE = 2
class FileCopier(threading.Thread):
"""Copies files in directory order."""
"""Copies or moves files in source directory order."""
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
@ -31,39 +37,48 @@ class FileCopier(threading.Thread):
# maxsize=100 is just a guess as to a reasonable upper limit. When this limit
# is reached, the main thread will simply block while waiting for this thread
# to finish copying a file.
self.file_copy_queue = queue.PriorityQueue(
maxsize=100) # type: queue.PriorityQueue[typing.Tuple[pathlib.Path, pathlib.Path]]
self.file_copy_done = threading.Event()
self.queue = queue.PriorityQueue(maxsize=100) \
# type: queue.PriorityQueue[typing.Tuple[pathlib.Path, pathlib.Path, Action]]
self.done = threading.Event()
def queue(self, src: pathlib.Path, dst: pathlib.Path):
def queue_copy(self, src: pathlib.Path, dst: pathlib.Path):
"""Queue a copy action from 'src' to 'dst'."""
self.file_copy_queue.put((src, dst))
self.queue.put((src, dst, Action.COPY))
def queue_move(self, src: pathlib.Path, dst: pathlib.Path):
"""Queue a move action from 'src' to 'dst'."""
self.queue.put((src, dst, Action.MOVE))
def done_and_join(self):
"""Indicate all files have been queued, and wait until done."""
self.file_copy_done.set()
self.done.set()
self.join()
if not self.file_copy_queue.empty():
if not self.queue.empty():
# Flush the queue so that we can report which files weren't copied yet.
files_remaining = []
while not self.file_copy_queue.empty():
src, dst = self.file_copy_queue.get_nowait()
while not self.queue.empty():
src, dst = self.queue.get_nowait()
files_remaining.append(src)
assert files_remaining
raise FileCopyError("%d files couldn't be copied" % len(files_remaining),
raise FileCopyError("%d files couldn't be transferred" % len(files_remaining),
files_remaining)
def run(self):
files_copied = 0
files_transferred = 0
files_skipped = 0
transfer_funcs = {
Action.COPY: shutil.copy,
Action.MOVE: shutil.move,
}
while True:
try:
src, dst = self.file_copy_queue.get(timeout=0.1)
src, dst, act = self.queue.get(timeout=0.1)
except queue.Empty:
if self.file_copy_done.is_set():
if self.done.is_set():
break
continue
@ -72,27 +87,33 @@ class FileCopier(threading.Thread):
st_src = src.stat()
st_dst = dst.stat()
if st_dst.st_size == st_src.st_size and st_dst.st_mtime >= st_src.st_mtime:
log.info('Skipping %s; already exists', src)
log.info('SKIP %s; already exists', src)
if act == Action.MOVE:
log.debug('Deleting %s', src)
src.unlink()
files_skipped += 1
continue
log.info('Copying %s → %s', src, dst)
log.info('%s %s → %s', act.name, src, dst)
dst.parent.mkdir(parents=True, exist_ok=True)
# TODO(Sybren): when we target Py 3.6+, remove the str() calls.
shutil.copy(str(src), str(dst))
files_copied += 1
transfer = transfer_funcs[act]
transfer(str(src), str(dst))
files_transferred += 1
except Exception:
# We have to catch exceptions in a broad way, as this is running in
# a separate thread, and exceptions won't otherwise be seen.
log.exception('Error copying %s to %s', src, dst)
log.exception('Error transferring %s to %s', src, dst)
# Put the files to copy back into the queue, and abort. This allows
# the main thread to inspect the queue and see which files were not
# copied. The one we just failed (due to this exception) should also
# be reported there.
self.file_copy_queue.put((src, dst))
self.queue.put((src, dst, act))
return
if files_copied:
log.info('Copied %d files', files_copied)
if files_transferred:
log.info('Transferred %d files', files_transferred)
if files_skipped:
log.info('Skipped %d files', files_skipped)