ac3f6142b3
This will cause a nice speedup if the temporary directory happens to be on the same file system as the target directory for the BAT Pack. This can be influenced by setting an environment variable; see the documentation for [tempfile.gettempdir](https://docs.python.org/3/library/tempfile.html#tempfile.gettempdir)
308 lines
12 KiB
Python
308 lines
12 KiB
Python
import collections
|
|
import enum
|
|
import functools
|
|
import logging
|
|
import pathlib
|
|
import tempfile
|
|
|
|
import typing
|
|
|
|
from blender_asset_tracer import trace, bpathlib, blendfile
|
|
from blender_asset_tracer.cli import common
|
|
from blender_asset_tracer.trace import result
|
|
from . import queued_copy
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class PathAction(enum.Enum):
|
|
KEEP_PATH = 1
|
|
FIND_NEW_LOCATION = 2
|
|
|
|
|
|
class AssetAction:
|
|
"""All the info required to rewrite blend files and copy assets."""
|
|
|
|
def __init__(self) -> None:
|
|
self.path_action = PathAction.KEEP_PATH
|
|
self.usages = [] # type: typing.List[result.BlockUsage]
|
|
"""BlockUsage objects referring to this asset.
|
|
|
|
Those BlockUsage objects could refer to data blocks in this blend file
|
|
(if the asset is a blend file) or in another blend file.
|
|
"""
|
|
|
|
self.new_path = None # type: pathlib.Path
|
|
"""Absolute path to the asset in the BAT Pack."""
|
|
|
|
self.read_from = None # type: typing.Optional[pathlib.Path]
|
|
"""Optional path from which to read the asset.
|
|
|
|
This is used when blend files have been rewritten. It is assumed that
|
|
when this property is set, the file can be moved instead of copied.
|
|
"""
|
|
|
|
self.rewrites = [] # type: typing.List[result.BlockUsage]
|
|
"""BlockUsage objects in this asset that may require rewriting.
|
|
|
|
Empty list if this AssetAction is not for a blend file.
|
|
"""
|
|
|
|
|
|
class Packer:
|
|
def __init__(self,
|
|
blendfile: pathlib.Path,
|
|
project: pathlib.Path,
|
|
target: pathlib.Path,
|
|
noop=False) -> None:
|
|
self.blendfile = blendfile
|
|
self.project = project
|
|
self.target = target
|
|
self.noop = noop
|
|
|
|
self._shorten = functools.partial(common.shorten, self.project)
|
|
|
|
if noop:
|
|
log.warning('Running in no-op mode, only showing what will be done.')
|
|
|
|
# Filled by strategise()
|
|
self._actions = collections.defaultdict(
|
|
AssetAction) # type: typing.DefaultDict[pathlib.Path, AssetAction]
|
|
|
|
# Number of files we would copy, if not for --noop
|
|
self._file_count = 0
|
|
|
|
self._tmpdir = tempfile.TemporaryDirectory(suffix='-batpack')
|
|
self._rewrite_in = pathlib.Path(self._tmpdir.name)
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
self.close()
|
|
|
|
def close(self):
|
|
"""Clean up any temporary files."""
|
|
self._tmpdir.cleanup()
|
|
|
|
def strategise(self):
|
|
"""Determine what to do with the assets.
|
|
|
|
Places an asset into one of these categories:
|
|
- Can be copied as-is, nothing smart required.
|
|
- Blend files referring to this asset need to be rewritten.
|
|
"""
|
|
|
|
# The blendfile that we pack is generally not its own dependency, so
|
|
# we have to explicitly add it to the _packed_paths.
|
|
bfile_path = self.blendfile.absolute()
|
|
bfile_pp = self.target / bfile_path.relative_to(self.project)
|
|
|
|
act = self._actions[bfile_path]
|
|
act.path_action = PathAction.KEEP_PATH
|
|
act.new_path = bfile_pp
|
|
|
|
new_location_paths = set()
|
|
for usage in trace.deps(self.blendfile):
|
|
# Needing rewriting is not a per-asset thing, but a per-asset-per-
|
|
# blendfile thing, since different blendfiles can refer to it in
|
|
# different ways (for example with relative and absolute paths).
|
|
asset_path = usage.abspath
|
|
bfile_path = usage.block.bfile.filepath.absolute()
|
|
|
|
path_in_project = self._path_in_project(asset_path)
|
|
use_as_is = usage.asset_path.is_blendfile_relative() and path_in_project
|
|
needs_rewriting = not use_as_is
|
|
|
|
act = self._actions[asset_path]
|
|
assert isinstance(act, AssetAction)
|
|
|
|
act.usages.append(usage)
|
|
if needs_rewriting:
|
|
log.info('%s needs rewritten path to %s', bfile_path, usage.asset_path)
|
|
act.path_action = PathAction.FIND_NEW_LOCATION
|
|
new_location_paths.add(asset_path)
|
|
else:
|
|
log.debug('%s can keep using %s', bfile_path, usage.asset_path)
|
|
asset_pp = self.target / asset_path.relative_to(self.project)
|
|
act.new_path = asset_pp
|
|
|
|
self._find_new_paths(new_location_paths)
|
|
self._group_rewrites()
|
|
|
|
def _find_new_paths(self, asset_paths: typing.Set[pathlib.Path]):
|
|
"""Find new locations in the BAT Pack for the given assets."""
|
|
|
|
for path in asset_paths:
|
|
act = self._actions[path]
|
|
assert isinstance(act, AssetAction)
|
|
# Like a join, but ignoring the fact that 'path' is absolute.
|
|
act.new_path = pathlib.Path(self.target, '_outside_project', *path.parts[1:])
|
|
|
|
def _group_rewrites(self):
|
|
"""For each blend file, collect which fields need rewriting.
|
|
|
|
This ensures that the execute() step has to visit each blend file
|
|
only once.
|
|
"""
|
|
|
|
for action in self._actions.values():
|
|
if action.path_action != PathAction.FIND_NEW_LOCATION:
|
|
# This asset doesn't require a new location, so no rewriting necessary.
|
|
continue
|
|
|
|
for usage in action.usages:
|
|
bfile_path = usage.block.bfile.filepath.absolute().resolve()
|
|
self._actions[bfile_path].rewrites.append(usage)
|
|
|
|
def _path_in_project(self, path: pathlib.Path) -> bool:
|
|
try:
|
|
# MUST use resolve(), otherwise /path/to/proj/../../asset.png
|
|
# will return True (relative_to will return ../../asset.png).
|
|
path.resolve().relative_to(self.project)
|
|
except ValueError:
|
|
return False
|
|
return True
|
|
|
|
def execute(self):
|
|
"""Execute the strategy."""
|
|
assert self._actions, 'Run strategise() first'
|
|
|
|
if not self.noop:
|
|
self._rewrite_paths()
|
|
self._copy_files_to_target()
|
|
|
|
def _copy_files_to_target(self):
|
|
"""Copy all assets to the target directoy.
|
|
|
|
This creates the BAT Pack but does not yet do any path rewriting.
|
|
"""
|
|
log.debug('Executing %d copy actions', len(self._actions))
|
|
|
|
fc = queued_copy.FileCopier()
|
|
if not self.noop:
|
|
fc.start()
|
|
|
|
for asset_path, action in self._actions.items():
|
|
self._copy_asset_and_deps(asset_path, action, fc)
|
|
|
|
if self.noop:
|
|
log.info('Would copy %d files to %s', self._file_count, self.target)
|
|
return
|
|
fc.done_and_join()
|
|
|
|
def _rewrite_paths(self):
|
|
"""Rewrite paths to the new location of the assets.
|
|
|
|
Writes the rewritten blend files to a temporary location.
|
|
"""
|
|
|
|
for bfile_path, action in self._actions.items():
|
|
if not action.rewrites:
|
|
continue
|
|
|
|
assert isinstance(bfile_path, pathlib.Path)
|
|
# bfile_pp is the final path of this blend file in the BAT pack.
|
|
# It is used to determine relative paths to other blend files.
|
|
# It is *not* used for any disk I/O, since the file may not even
|
|
# exist on the local filesystem.
|
|
bfile_pp = self._actions[bfile_path].new_path
|
|
|
|
# Use tempfile to create a unique name in our temporary directoy.
|
|
# The file should be deleted when self.close() is called, and not
|
|
# when the bfile_tp object is GC'd.
|
|
bfile_tmp = tempfile.NamedTemporaryFile(dir=str(self._rewrite_in),
|
|
suffix='-' + bfile_path.name,
|
|
delete=False)
|
|
bfile_tp = pathlib.Path(bfile_tmp.name)
|
|
action.read_from = bfile_tp
|
|
log.info('Rewriting %s to %s', bfile_path, bfile_tp)
|
|
|
|
# The original blend file will have been cached, so we can use it
|
|
# to avoid re-parsing all data blocks in the to-be-rewritten file.
|
|
bfile = blendfile.open_cached(bfile_path, assert_cached=True)
|
|
bfile.copy_and_rebind(bfile_tp, mode='rb+')
|
|
|
|
for usage in action.rewrites:
|
|
assert isinstance(usage, result.BlockUsage)
|
|
asset_pp = self._actions[usage.abspath].new_path
|
|
assert isinstance(asset_pp, pathlib.Path)
|
|
|
|
log.debug(' - %s is packed at %s', usage.asset_path, asset_pp)
|
|
relpath = bpathlib.BlendPath.mkrelative(asset_pp, bfile_pp)
|
|
if relpath == usage.asset_path:
|
|
log.info(' - %s remained at %s', usage.asset_path, relpath)
|
|
continue
|
|
|
|
log.info(' - %s moved to %s', usage.asset_path, relpath)
|
|
|
|
# Find the same block in the newly copied file.
|
|
block = bfile.dereference_pointer(usage.block.addr_old)
|
|
if usage.path_full_field is None:
|
|
log.debug(' - updating field %s of block %s',
|
|
usage.path_dir_field.name.name_only, block)
|
|
reldir = bpathlib.BlendPath.mkrelative(asset_pp.parent, bfile_pp)
|
|
written = block.set(usage.path_dir_field.name.name_only, reldir)
|
|
log.debug(' - written %d bytes', written)
|
|
|
|
# BIG FAT ASSUMPTION that the filename (e.g. basename
|
|
# without path) does not change. This makes things much
|
|
# easier, as in the sequence editor the directory and
|
|
# filename fields are in different blocks. See the
|
|
# blocks2assets.scene() function for the implementation.
|
|
else:
|
|
log.debug(' - updating field %s of block %s',
|
|
usage.path_full_field.name.name_only, block)
|
|
written = block.set(usage.path_full_field.name.name_only, relpath)
|
|
log.debug(' - written %d bytes', written)
|
|
|
|
# Make sure we close the file, otherwise changes may not be
|
|
# flushed before it gets copied.
|
|
bfile.close()
|
|
|
|
def _copy_asset_and_deps(self, asset_path: pathlib.Path, action: AssetAction,
|
|
fc: queued_copy.FileCopier):
|
|
# Copy the asset itself.
|
|
packed_path = action.new_path
|
|
read_path = action.read_from or asset_path
|
|
self._send_to_target(read_path, packed_path, fc,
|
|
may_move=action.read_from is not None)
|
|
|
|
# Copy its sequence dependencies.
|
|
for usage in action.usages:
|
|
if not usage.is_sequence:
|
|
continue
|
|
|
|
first_pp = self._actions[usage.abspath].new_path
|
|
|
|
# In case of globbing, we only support globbing by filename,
|
|
# and not by directory.
|
|
assert '*' not in str(first_pp) or '*' in first_pp.name
|
|
|
|
packed_base_dir = first_pp.parent
|
|
for file_path in usage.files():
|
|
packed_path = packed_base_dir / file_path.name
|
|
# Assumption: assets in a sequence are never blend files.
|
|
self._send_to_target(file_path, packed_path, fc)
|
|
|
|
# Assumption: all data blocks using this asset use it the same way.
|
|
break
|
|
|
|
def _send_to_target(self,
|
|
asset_path: pathlib.Path,
|
|
target: pathlib.Path,
|
|
fc: queued_copy.FileCopier,
|
|
may_move=False):
|
|
if self.noop:
|
|
print('%s → %s' % (asset_path, target))
|
|
self._file_count += 1
|
|
return
|
|
|
|
verb = 'move' if may_move else 'copy'
|
|
log.debug('Queueing %s of %s', verb, asset_path)
|
|
|
|
if may_move:
|
|
fc.queue_move(asset_path, target)
|
|
else:
|
|
fc.queue_copy(asset_path, target)
|