Renamed module tracer → trace
This is consistent with the other module names.
This commit is contained in:
Sybren A. Stüvel
@@ -0,0 +1,49 @@
|
||||
import logging
|
||||
import pathlib
|
||||
import typing
|
||||
|
||||
from blender_asset_tracer import blendfile
|
||||
from . import result, blocks2assets, file2blocks
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
codes_to_skip = {
|
||||
# These blocks never have external assets:
|
||||
b'ID', b'WM', b'SN',
|
||||
|
||||
# These blocks are skipped for now, until we have proof they point to
|
||||
# assets otherwise missed:
|
||||
b'GR', b'WO', b'BR', b'LS',
|
||||
}
|
||||
|
||||
|
||||
def deps(bfilepath: pathlib.Path) -> typing.Iterator[result.BlockUsage]:
|
||||
"""Open the blend file and report its dependencies.
|
||||
|
||||
:param bfilepath: File to open.
|
||||
"""
|
||||
|
||||
log.info('opening: %s', bfilepath)
|
||||
bfile = blendfile.open_cached(bfilepath)
|
||||
|
||||
# Sort the asset-holding blocks so that we can iterate over them
|
||||
# in disk order, which is slightly faster than random order.
|
||||
ahb = asset_holding_blocks(file2blocks.iter_blocks(bfile))
|
||||
for block in sorted(ahb):
|
||||
yield from blocks2assets.iter_assets(block)
|
||||
|
||||
|
||||
def asset_holding_blocks(blocks: typing.Iterable[blendfile.BlendFileBlock]) \
|
||||
-> typing.Iterator[blendfile.BlendFileBlock]:
|
||||
"""Generator, yield data blocks that could reference external assets."""
|
||||
for block in blocks:
|
||||
assert isinstance(block, blendfile.BlendFileBlock)
|
||||
code = block.code
|
||||
|
||||
# The longer codes are either arbitrary data or data blocks that
|
||||
# don't refer to external assets. The former data blocks will be
|
||||
# visited when we hit the two-letter datablocks that use them.
|
||||
if len(code) > 2 or code in codes_to_skip:
|
||||
continue
|
||||
|
||||
yield block
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Block walkers.
|
||||
|
||||
From a Blend file data block, iter_assts() yields all the referred-to assets.
|
||||
"""
|
||||
|
||||
import functools
|
||||
import logging
|
||||
import typing
|
||||
|
||||
from blender_asset_tracer import blendfile, bpathlib, cdefs
|
||||
from blender_asset_tracer.blendfile import iterators
|
||||
from . import result, modifier_walkers
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_warned_about_types = set()
|
||||
_funcs_for_code = {}
|
||||
|
||||
|
||||
def iter_assets(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
|
||||
"""Generator, yield the assets used by this data block."""
|
||||
assert block.code != b'DATA'
|
||||
|
||||
try:
|
||||
block_reader = _funcs_for_code[block.code]
|
||||
except KeyError:
|
||||
if block.code not in _warned_about_types:
|
||||
log.debug('No reader implemented for block type %r', block.code.decode())
|
||||
_warned_about_types.add(block.code)
|
||||
return
|
||||
|
||||
log.debug('Tracing block %r', block)
|
||||
yield from block_reader(block)
|
||||
|
||||
|
||||
def dna_code(block_code: str):
|
||||
"""Decorator, marks decorated func as handler for that DNA code."""
|
||||
|
||||
assert isinstance(block_code, str)
|
||||
|
||||
def decorator(wrapped):
|
||||
_funcs_for_code[block_code.encode()] = wrapped
|
||||
return wrapped
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def skip_packed(wrapped):
|
||||
"""Decorator, skip blocks where 'packedfile' is set to true."""
|
||||
|
||||
@functools.wraps(wrapped)
|
||||
def wrapper(block: blendfile.BlendFileBlock, *args, **kwargs):
|
||||
if block.get(b'packedfile', default=False):
|
||||
return
|
||||
|
||||
yield from wrapped(block, *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
@dna_code('CF')
|
||||
def cache_file(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
|
||||
"""Cache file data blocks."""
|
||||
path, field = block.get(b'filepath', return_field=True)
|
||||
yield result.BlockUsage(block, path, path_full_field=field)
|
||||
|
||||
|
||||
@dna_code('IM')
|
||||
@skip_packed
|
||||
def image(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
|
||||
"""Image data blocks."""
|
||||
# old files miss this
|
||||
image_source = block.get(b'source', default=cdefs.IMA_SRC_FILE)
|
||||
if image_source not in {cdefs.IMA_SRC_FILE, cdefs.IMA_SRC_SEQUENCE, cdefs.IMA_SRC_MOVIE}:
|
||||
return
|
||||
|
||||
pathname, field = block.get(b'name', return_field=True)
|
||||
is_sequence = image_source == cdefs.IMA_SRC_SEQUENCE
|
||||
|
||||
yield result.BlockUsage(block, pathname, is_sequence, path_full_field=field)
|
||||
|
||||
|
||||
@dna_code('LI')
|
||||
def library(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
|
||||
"""Library data blocks."""
|
||||
path, field = block.get(b'name', return_field=True)
|
||||
yield result.BlockUsage(block, path, path_full_field=field)
|
||||
|
||||
# The 'filepath' also points to the blend file. However, this is set to the
|
||||
# absolute path of the file by Blender (see BKE_library_filepath_set). This
|
||||
# is thus not a property we have to report or rewrite.
|
||||
|
||||
|
||||
@dna_code('ME')
|
||||
def mesh(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
|
||||
"""Mesh data blocks."""
|
||||
block_external = block.get_pointer((b'ldata', b'external'), None)
|
||||
if block_external is None:
|
||||
block_external = block.get_pointer((b'fdata', b'external'), None)
|
||||
if block_external is None:
|
||||
return
|
||||
|
||||
path, field = block_external.get(b'filename', return_field=True)
|
||||
yield result.BlockUsage(block, path, path_full_field=field)
|
||||
|
||||
|
||||
@dna_code('MC')
|
||||
def movie_clip(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
|
||||
"""MovieClip data blocks."""
|
||||
path, field = block.get(b'name', return_field=True)
|
||||
# TODO: The assumption that this is not a sequence may not be true for all modifiers.
|
||||
yield result.BlockUsage(block, path, is_sequence=False, path_full_field=field)
|
||||
|
||||
|
||||
@dna_code('OB')
|
||||
def object_block(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
|
||||
"""Object data blocks."""
|
||||
# 'ob->modifiers[...].filepath'
|
||||
mods = block.get_pointer((b'modifiers', b'first'))
|
||||
for mod_idx, block_mod in enumerate(iterators.listbase(mods, next_path=(b'modifier', b'next'))):
|
||||
block_name = b'%s.modifiers[%d]' % (block.id_name, mod_idx)
|
||||
mod_type = block_mod[b'modifier', b'type']
|
||||
log.debug('Tracing modifier %s, type=%d', block_name.decode(), mod_type)
|
||||
|
||||
try:
|
||||
mod_handler = modifier_walkers.modifier_handlers[mod_type]
|
||||
except KeyError:
|
||||
continue
|
||||
yield from mod_handler(block_mod, block_name)
|
||||
|
||||
|
||||
@dna_code('SC')
|
||||
def scene(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
|
||||
"""Scene data blocks."""
|
||||
# Sequence editor is the only interesting bit.
|
||||
block_ed = block.get_pointer(b'ed')
|
||||
if block_ed is None:
|
||||
return
|
||||
|
||||
single_asset_types = {cdefs.SEQ_TYPE_MOVIE, cdefs.SEQ_TYPE_SOUND_RAM, cdefs.SEQ_TYPE_SOUND_HD}
|
||||
asset_types = single_asset_types.union({cdefs.SEQ_TYPE_IMAGE})
|
||||
|
||||
for seq, seq_type in iterators.sequencer_strips(block_ed):
|
||||
if seq_type not in asset_types:
|
||||
continue
|
||||
|
||||
seq_strip = seq.get_pointer(b'strip')
|
||||
if seq_strip is None:
|
||||
continue
|
||||
seq_stripdata = seq_strip.get_pointer(b'stripdata')
|
||||
if seq_stripdata is None:
|
||||
continue
|
||||
|
||||
dirname, dn_field = seq_strip.get(b'dir', return_field=True)
|
||||
basename, bn_field = seq_stripdata.get(b'name', return_field=True)
|
||||
asset_path = bpathlib.BlendPath(dirname) / basename
|
||||
|
||||
is_sequence = seq_type not in single_asset_types
|
||||
yield result.BlockUsage(seq_strip, asset_path,
|
||||
is_sequence=is_sequence,
|
||||
path_dir_field=dn_field,
|
||||
path_base_field=bn_field)
|
||||
|
||||
|
||||
@dna_code('SO')
|
||||
@skip_packed
|
||||
def sound(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
|
||||
"""Sound data blocks."""
|
||||
path, field = block.get(b'name', return_field=True)
|
||||
yield result.BlockUsage(block, path, path_full_field=field)
|
||||
|
||||
|
||||
@dna_code('VF')
|
||||
@skip_packed
|
||||
def vector_font(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
|
||||
"""Vector Font data blocks."""
|
||||
path, field = block.get(b'name', return_field=True)
|
||||
if path == b'<builtin>': # builtin font
|
||||
return
|
||||
yield result.BlockUsage(block, path, path_full_field=field)
|
||||
@@ -0,0 +1,231 @@
|
||||
"""Low-level functions called by file2block.
|
||||
|
||||
Those can expand data blocks and yield their dependencies (e.g. other data
|
||||
blocks necessary to render/display/work with the given data block).
|
||||
"""
|
||||
import logging
|
||||
import typing
|
||||
|
||||
from blender_asset_tracer import blendfile, cdefs
|
||||
from blender_asset_tracer.blendfile import iterators
|
||||
|
||||
# Don't warn about these types at all.
|
||||
_warned_about_types = {b'LI', b'DATA'}
|
||||
_funcs_for_code = {}
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def expand_block(block: blendfile.BlendFileBlock) -> typing.Iterator[blendfile.BlendFileBlock]:
|
||||
"""Generator, yield the data blocks used by this data block."""
|
||||
|
||||
try:
|
||||
expander = _funcs_for_code[block.code]
|
||||
except KeyError:
|
||||
if block.code not in _warned_about_types:
|
||||
log.debug('No expander implemented for block type %r', block.code.decode())
|
||||
_warned_about_types.add(block.code)
|
||||
return
|
||||
|
||||
log.debug('Expanding block %r', block)
|
||||
# Filter out falsy blocks, i.e. None values.
|
||||
# Allowing expanders to yield None makes them more consise.
|
||||
yield from filter(None, expander(block))
|
||||
|
||||
|
||||
def dna_code(block_code: str):
|
||||
"""Decorator, marks decorated func as expander for that DNA code."""
|
||||
|
||||
assert isinstance(block_code, str)
|
||||
|
||||
def decorator(wrapped):
|
||||
_funcs_for_code[block_code.encode()] = wrapped
|
||||
return wrapped
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def _expand_generic_material(block: blendfile.BlendFileBlock):
|
||||
array_len = block.get(b'totcol')
|
||||
yield from block.iter_array_of_pointers(b'mat', array_len)
|
||||
|
||||
|
||||
def _expand_generic_mtex(block: blendfile.BlendFileBlock):
|
||||
for mtex in block.iter_fixed_array_of_pointers(b'mtex'):
|
||||
yield mtex.get_pointer(b'tex')
|
||||
yield mtex.get_pointer(b'object')
|
||||
|
||||
|
||||
def _expand_generic_nodetree(block: blendfile.BlendFileBlock):
|
||||
assert block.dna_type.dna_type_id == b'bNodeTree'
|
||||
|
||||
nodes = block.get_pointer((b'nodes', b'first'))
|
||||
for node in iterators.listbase(nodes):
|
||||
if node[b'type'] == cdefs.CMP_NODE_R_LAYERS:
|
||||
continue
|
||||
yield node
|
||||
|
||||
# The 'id' property points to whatever is used by the node
|
||||
# (like the image in an image texture node).
|
||||
yield node.get_pointer(b'id')
|
||||
|
||||
|
||||
def _expand_generic_nodetree_id(block: blendfile.BlendFileBlock):
|
||||
block_ntree = block.get_pointer(b'nodetree', None)
|
||||
if block_ntree is not None:
|
||||
yield from _expand_generic_nodetree(block_ntree)
|
||||
|
||||
|
||||
def _expand_generic_animdata(block: blendfile.BlendFileBlock):
|
||||
block_adt = block.get_pointer(b'adt')
|
||||
if block_adt:
|
||||
yield block_adt.get_pointer(b'action')
|
||||
# TODO, NLA
|
||||
|
||||
|
||||
@dna_code('AR')
|
||||
def _expand_armature(block: blendfile.BlendFileBlock):
|
||||
yield from _expand_generic_animdata(block)
|
||||
|
||||
|
||||
@dna_code('CU')
|
||||
def _expand_curve(block: blendfile.BlendFileBlock):
|
||||
yield from _expand_generic_animdata(block)
|
||||
yield from _expand_generic_material(block)
|
||||
|
||||
for fieldname in (b'vfont', b'vfontb', b'vfonti', b'vfontbi',
|
||||
b'bevobj', b'taperobj', b'textoncurve'):
|
||||
yield block.get_pointer(fieldname)
|
||||
|
||||
|
||||
@dna_code('GR')
|
||||
def _expand_group(block: blendfile.BlendFileBlock):
|
||||
objects = block.get_pointer((b'gobject', b'first'))
|
||||
for item in iterators.listbase(objects):
|
||||
yield item.get_pointer(b'ob')
|
||||
|
||||
|
||||
@dna_code('LA')
|
||||
def _expand_lamp(block: blendfile.BlendFileBlock):
|
||||
yield from _expand_generic_animdata(block)
|
||||
yield from _expand_generic_nodetree_id(block)
|
||||
yield from _expand_generic_mtex(block)
|
||||
|
||||
|
||||
@dna_code('MA')
|
||||
def _expand_material(block: blendfile.BlendFileBlock):
|
||||
yield from _expand_generic_animdata(block)
|
||||
yield from _expand_generic_nodetree_id(block)
|
||||
yield from _expand_generic_mtex(block)
|
||||
|
||||
yield block.get_pointer(b'group')
|
||||
|
||||
|
||||
@dna_code('MB')
|
||||
def _expand_metaball(block: blendfile.BlendFileBlock):
|
||||
yield from _expand_generic_animdata(block)
|
||||
yield from _expand_generic_material(block)
|
||||
|
||||
|
||||
@dna_code('ME')
|
||||
def _expand_mesh(block: blendfile.BlendFileBlock):
|
||||
yield from _expand_generic_animdata(block)
|
||||
yield from _expand_generic_material(block)
|
||||
yield block.get_pointer(b'texcomesh')
|
||||
# TODO, TexFace? - it will be slow, we could simply ignore :S
|
||||
|
||||
|
||||
@dna_code('NT')
|
||||
def _expand_node_tree(block: blendfile.BlendFileBlock):
|
||||
yield from _expand_generic_animdata(block)
|
||||
yield from _expand_generic_nodetree(block)
|
||||
|
||||
|
||||
@dna_code('OB')
|
||||
def _expand_object(block: blendfile.BlendFileBlock):
|
||||
yield from _expand_generic_animdata(block)
|
||||
yield from _expand_generic_material(block)
|
||||
|
||||
yield block.get_pointer(b'data')
|
||||
|
||||
if block[b'transflag'] & cdefs.OB_DUPLIGROUP:
|
||||
yield block.get_pointer(b'dup_group')
|
||||
|
||||
yield block.get_pointer(b'proxy')
|
||||
yield block.get_pointer(b'proxy_group')
|
||||
|
||||
# 'ob->pose->chanbase[...].custom'
|
||||
block_pose = block.get_pointer(b'pose')
|
||||
if block_pose:
|
||||
assert block_pose.dna_type.dna_type_id == b'bPose'
|
||||
# sdna_index_bPoseChannel = block_pose.file.sdna_index_from_id[b'bPoseChannel']
|
||||
channels = block_pose.get_pointer((b'chanbase', b'first'))
|
||||
for pose_chan in iterators.listbase(channels):
|
||||
yield pose_chan.get_pointer(b'custom')
|
||||
|
||||
# Expand the objects 'ParticleSettings' via 'ob->particlesystem[...].part'
|
||||
# sdna_index_ParticleSystem = block.file.sdna_index_from_id.get(b'ParticleSystem')
|
||||
# if sdna_index_ParticleSystem is not None:
|
||||
psystems = block.get_pointer((b'particlesystem', b'first'))
|
||||
for psystem in iterators.listbase(psystems):
|
||||
yield psystem.get_pointer(b'part')
|
||||
|
||||
|
||||
@dna_code('PA')
|
||||
def _expand_particle_settings(block: blendfile.BlendFileBlock):
|
||||
yield from _expand_generic_animdata(block)
|
||||
yield from _expand_generic_mtex(block)
|
||||
|
||||
block_ren_as = block[b'ren_as']
|
||||
if block_ren_as == cdefs.PART_DRAW_GR:
|
||||
yield block.get_pointer(b'dup_group')
|
||||
elif block_ren_as == cdefs.PART_DRAW_OB:
|
||||
yield block.get_pointer(b'dup_ob')
|
||||
|
||||
|
||||
@dna_code('SC')
|
||||
def _expand_scene(block: blendfile.BlendFileBlock):
|
||||
yield from _expand_generic_animdata(block)
|
||||
yield from _expand_generic_nodetree_id(block)
|
||||
yield block.get_pointer(b'camera')
|
||||
yield block.get_pointer(b'world')
|
||||
yield block.get_pointer(b'set', default=None)
|
||||
yield block.get_pointer(b'clip', default=None)
|
||||
|
||||
# sdna_index_Base = block.file.sdna_index_from_id[b'Base']
|
||||
# for item in bf_utils.iter_ListBase(block.get_pointer((b'base', b'first'))):
|
||||
# yield item.get_pointer(b'object', sdna_index_refine=sdna_index_Base)
|
||||
bases = block.get_pointer((b'base', b'first'))
|
||||
for base in iterators.listbase(bases):
|
||||
yield base.get_pointer(b'object')
|
||||
|
||||
# Sequence Editor
|
||||
block_ed = block.get_pointer(b'ed')
|
||||
if not block_ed:
|
||||
return
|
||||
|
||||
strip_type_to_field = {
|
||||
cdefs.SEQ_TYPE_SCENE: b'scene',
|
||||
cdefs.SEQ_TYPE_MOVIECLIP: b'clip',
|
||||
cdefs.SEQ_TYPE_MASK: b'mask',
|
||||
cdefs.SEQ_TYPE_SOUND_RAM: b'sound',
|
||||
}
|
||||
for strip, strip_type in iterators.sequencer_strips(block_ed):
|
||||
try:
|
||||
field_name = strip_type_to_field[strip_type]
|
||||
except KeyError:
|
||||
continue
|
||||
yield strip.get_pointer(field_name)
|
||||
|
||||
|
||||
@dna_code('TE')
|
||||
def _expand_texture(block: blendfile.BlendFileBlock):
|
||||
yield from _expand_generic_animdata(block)
|
||||
yield from _expand_generic_nodetree_id(block)
|
||||
yield block.get_pointer(b'ima')
|
||||
|
||||
|
||||
@dna_code('WO')
|
||||
def _expand_world(block: blendfile.BlendFileBlock):
|
||||
yield from _expand_generic_animdata(block)
|
||||
yield from _expand_generic_nodetree_id(block)
|
||||
yield from _expand_generic_mtex(block)
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Expand data blocks.
|
||||
|
||||
The expansion process follows pointers and library links to construct the full
|
||||
set of actually-used data blocks. This set consists of all data blocks in the
|
||||
initial blend file, and all *actually linked-to* data blocks in linked
|
||||
blend files.
|
||||
"""
|
||||
import collections
|
||||
import logging
|
||||
import typing
|
||||
|
||||
from blender_asset_tracer import blendfile, bpathlib
|
||||
from . import expanders
|
||||
|
||||
_funcs_for_code = {}
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _BlockIterator:
|
||||
"""Expand blocks with dependencies from other libraries.
|
||||
|
||||
This class exists so that we have some context for the recursive expansion
|
||||
without having to pass those variables to each recursive call.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Set of (blend file Path, block address) of already-reported blocks.
|
||||
self.blocks_yielded = set()
|
||||
|
||||
# Queue of blocks to visit
|
||||
self.to_visit = collections.deque()
|
||||
|
||||
def iter_blocks(self,
|
||||
bfile: blendfile.BlendFile,
|
||||
limit_to: typing.Set[blendfile.BlendFileBlock] = frozenset(),
|
||||
) -> typing.Iterator[blendfile.BlendFileBlock]:
|
||||
"""Expand blocks with dependencies from other libraries."""
|
||||
|
||||
log.info('inspecting: %s', bfile.filepath)
|
||||
if limit_to:
|
||||
self._queue_named_blocks(bfile, limit_to)
|
||||
else:
|
||||
self._queue_all_blocks(bfile)
|
||||
|
||||
blocks_per_lib = yield from self._visit_blocks(bfile, limit_to)
|
||||
yield from self._visit_linked_blocks(blocks_per_lib)
|
||||
|
||||
def _visit_blocks(self, bfile, limit_to):
|
||||
bpath = bfile.filepath.absolute().resolve()
|
||||
root_dir = bpathlib.BlendPath(bpath.parent)
|
||||
|
||||
# Mapping from library path to data blocks to expand.
|
||||
blocks_per_lib = collections.defaultdict(set)
|
||||
|
||||
while self.to_visit:
|
||||
block = self.to_visit.popleft()
|
||||
assert isinstance(block, blendfile.BlendFileBlock)
|
||||
if (bpath, block.addr_old) in self.blocks_yielded:
|
||||
continue
|
||||
|
||||
if block.code == b'ID':
|
||||
# ID blocks represent linked-in assets. Those are the ones that
|
||||
# should be loaded from their own blend file and "expanded" to
|
||||
# the entire set of data blocks required to render them. We
|
||||
# defer the handling of those so that we can work with one
|
||||
# blend file at a time.
|
||||
lib = block.get_pointer(b'lib')
|
||||
lib_bpath = bpathlib.BlendPath(lib[b'name']).absolute(root_dir)
|
||||
blocks_per_lib[lib_bpath].add(block)
|
||||
|
||||
# The library block itself should also be reported, because it
|
||||
# represents a blend file that is a dependency as well.
|
||||
self.to_visit.append(lib)
|
||||
continue
|
||||
|
||||
if limit_to:
|
||||
# We're limiting the blocks, so we have to expand them to make
|
||||
# sure we don't miss anything. Otherwise we're yielding the
|
||||
# entire file anyway, and no expansion is necessary.
|
||||
self._queue_dependencies(block)
|
||||
self.blocks_yielded.add((bpath, block.addr_old))
|
||||
yield block
|
||||
|
||||
return blocks_per_lib
|
||||
|
||||
def _visit_linked_blocks(self, blocks_per_lib):
|
||||
# We've gone through all the blocks in this file, now open the libraries
|
||||
# and iterate over the blocks referred there.
|
||||
for lib_bpath, idblocks in blocks_per_lib.items():
|
||||
lib_path = lib_bpath.to_path()
|
||||
try:
|
||||
lib_path = lib_path.resolve()
|
||||
except FileNotFoundError:
|
||||
log.warning('Library %s does not exist', lib_path)
|
||||
continue
|
||||
|
||||
log.debug('Expanding %d blocks in %s', len(idblocks), lib_path)
|
||||
libfile = blendfile.open_cached(lib_path)
|
||||
yield from self.iter_blocks(libfile, idblocks)
|
||||
|
||||
def _queue_all_blocks(self, bfile: blendfile.BlendFile):
|
||||
log.debug('Queueing all blocks from file %s', bfile.filepath)
|
||||
# Don't bother visiting DATA blocks, as we won't know what
|
||||
# to do with them anyway.
|
||||
self.to_visit.extend(block for block in bfile.blocks
|
||||
if block.code != b'DATA')
|
||||
self._sort_queue()
|
||||
|
||||
def _queue_named_blocks(self,
|
||||
bfile: blendfile.BlendFile,
|
||||
limit_to: typing.Set[blendfile.BlendFileBlock]):
|
||||
"""Queue only the blocks referred to in limit_to.
|
||||
|
||||
:param bfile:
|
||||
:param limit_to: set of ID blocks that name the blocks to queue.
|
||||
The queued blocks are loaded from the actual blend file, and
|
||||
selected by name.
|
||||
"""
|
||||
|
||||
for to_find in limit_to:
|
||||
assert to_find.code == b'ID'
|
||||
name_to_find = to_find[b'name']
|
||||
code = name_to_find[:2]
|
||||
log.debug('Finding block %r with code %r', name_to_find, code)
|
||||
same_code = bfile.find_blocks_from_code(code)
|
||||
for block in same_code:
|
||||
if block.id_name == name_to_find:
|
||||
log.debug('Queueing %r from file %s', block, bfile.filepath)
|
||||
self.to_visit.append(block)
|
||||
self._sort_queue()
|
||||
|
||||
def _queue_dependencies(self, block: blendfile.BlendFileBlock):
|
||||
self.to_visit.extend(expanders.expand_block(block))
|
||||
self._sort_queue()
|
||||
|
||||
def _sort_queue(self):
|
||||
"""Sort the queued blocks by file and by offset.
|
||||
|
||||
This allows us to go through the blend files sequentially.
|
||||
"""
|
||||
|
||||
def sort_key(block: blendfile.BlendFileBlock):
|
||||
return block.bfile.filepath, block.file_offset
|
||||
|
||||
self.to_visit = collections.deque(sorted(self.to_visit, key=sort_key))
|
||||
|
||||
|
||||
def iter_blocks(bfile: blendfile.BlendFile) -> typing.Iterator[blendfile.BlendFileBlock]:
|
||||
"""Generator, yield all blocks in this file + required blocks in libs."""
|
||||
bi = _BlockIterator()
|
||||
yield from bi.iter_blocks(bfile)
|
||||
@@ -0,0 +1,49 @@
|
||||
import logging
|
||||
import pathlib
|
||||
import typing
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DoesNotExist(OSError):
|
||||
"""Indicates a path does not exist on the filesystem."""
|
||||
|
||||
def __init__(self, path: pathlib.Path):
|
||||
super().__init__(path)
|
||||
self.path = path
|
||||
|
||||
|
||||
def expand_sequence(path: pathlib.Path) -> typing.Iterator[pathlib.Path]:
|
||||
"""Expand a file sequence path into the actual file paths.
|
||||
|
||||
:param path: can be either a glob pattern (must contain a * character)
|
||||
or the path of the first file in the sequence.
|
||||
"""
|
||||
|
||||
if '*' in str(path): # assume it is a glob
|
||||
import glob
|
||||
log.debug('expanding glob %s', path)
|
||||
for fname in sorted(glob.glob(str(path), recursive=True)):
|
||||
yield pathlib.Path(fname)
|
||||
return
|
||||
|
||||
if not path.exists():
|
||||
raise DoesNotExist(path)
|
||||
|
||||
if path.is_dir():
|
||||
raise TypeError('path is a directory: %s' % path)
|
||||
|
||||
log.debug('expanding file sequence %s', path)
|
||||
|
||||
import string
|
||||
stem_no_digits = path.stem.rstrip(string.digits)
|
||||
if stem_no_digits == path.stem:
|
||||
# Just a single file, no digits here.
|
||||
yield path
|
||||
return
|
||||
|
||||
# Return everything start starts with 'stem_no_digits' and ends with the
|
||||
# same suffix as the first file. This may result in more files than used
|
||||
# by Blender, but at least it shouldn't miss any.
|
||||
pattern = '%s*%s' % (stem_no_digits, path.suffix)
|
||||
yield from sorted(path.parent.glob(pattern))
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Modifier handling code used in blocks2assets.py
|
||||
|
||||
The _modifier_xxx() functions all yield result.BlockUsage objects for external
|
||||
files used by the modifiers.
|
||||
"""
|
||||
import typing
|
||||
|
||||
from blender_asset_tracer import blendfile, bpathlib, cdefs
|
||||
from . import result
|
||||
|
||||
|
||||
def _modifier_filepath(modifier: blendfile.BlendFileBlock, block_name: bytes) \
|
||||
-> typing.Iterator[result.BlockUsage]:
|
||||
"""Just yield the 'filepath' field."""
|
||||
path, field = modifier.get(b'filepath', return_field=True)
|
||||
yield result.BlockUsage(modifier, path, path_full_field=field, block_name=block_name)
|
||||
|
||||
|
||||
def _modifier_ocean(modifier: blendfile.BlendFileBlock, block_name: bytes) \
|
||||
-> typing.Iterator[result.BlockUsage]:
|
||||
if not modifier[b'cached']:
|
||||
return
|
||||
|
||||
path, field = modifier.get(b'cachepath', return_field=True)
|
||||
# The path indicates the directory containing the cached files.
|
||||
yield result.BlockUsage(modifier, path, is_sequence=True, path_full_field=field,
|
||||
block_name=block_name)
|
||||
|
||||
|
||||
def _modifier_particle_system(modifier: blendfile.BlendFileBlock, block_name: bytes) \
|
||||
-> typing.Iterator[result.BlockUsage]:
|
||||
psys = modifier.get_pointer(b'psys')
|
||||
if psys is None:
|
||||
return
|
||||
|
||||
pointcache = psys.get_pointer(b'pointcache')
|
||||
if pointcache is None:
|
||||
return
|
||||
|
||||
flag = pointcache[b'flag']
|
||||
|
||||
if flag & cdefs.PTCACHE_DISK_CACHE:
|
||||
# See ptcache_path() in pointcache.c
|
||||
name, field = pointcache.get(b'name', return_field=True)
|
||||
path = b'//%b%b/%b_*%b' % (
|
||||
cdefs.PTCACHE_PATH,
|
||||
modifier.bfile.filepath.stem.encode(),
|
||||
name,
|
||||
cdefs.PTCACHE_EXT)
|
||||
bpath = bpathlib.BlendPath(path)
|
||||
yield result.BlockUsage(pointcache, bpath, path_full_field=field,
|
||||
is_sequence=True, block_name=block_name)
|
||||
|
||||
if flag & cdefs.PTCACHE_EXTERNAL:
|
||||
path, field = pointcache.get(b'path', return_field=True)
|
||||
yield result.BlockUsage(pointcache, path, path_full_field=field,
|
||||
is_sequence=True, block_name=block_name)
|
||||
|
||||
|
||||
modifier_handlers = {
|
||||
cdefs.eModifierType_ParticleSystem: _modifier_particle_system,
|
||||
cdefs.eModifierType_Ocean: _modifier_ocean,
|
||||
cdefs.eModifierType_MeshCache: _modifier_filepath,
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import functools
|
||||
import logging
|
||||
import pathlib
|
||||
import typing
|
||||
|
||||
from blender_asset_tracer import blendfile, bpathlib
|
||||
from blender_asset_tracer.blendfile import dna
|
||||
from . import file_sequence
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@functools.total_ordering
|
||||
class BlockUsage:
|
||||
"""Represents the use of an asset by a data block.
|
||||
|
||||
:ivar block_name: an identifying name for this block. Defaults to the ID
|
||||
name of the block.
|
||||
:ivar block:
|
||||
:ivar asset_path: The path of the asset, if is_sequence=False. Otherwise
|
||||
it can be either a glob pattern (must contain a * byte) or the path of
|
||||
the first file in the sequence.
|
||||
:ivar is_sequence: Indicates whether this file is alone (False), the
|
||||
first of a sequence (True, and the path points to a file), or a
|
||||
directory containing a sequence (True, and path points to a directory).
|
||||
In certain cases such files should be reported once (f.e. when
|
||||
rewriting the source field to another path), and in other cases the
|
||||
sequence should be expanded (f.e. when copying all assets to a BAT
|
||||
Pack).
|
||||
:ivar path_full_field: field containing the full path of this asset.
|
||||
:ivar path_dir_field: field containing the parent path (i.e. the
|
||||
directory) of this asset.
|
||||
:ivar path_base_field: field containing the basename of this asset.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
block: blendfile.BlendFileBlock,
|
||||
asset_path: bpathlib.BlendPath,
|
||||
is_sequence: bool = False,
|
||||
path_full_field: dna.Field = None,
|
||||
path_dir_field: dna.Field = None,
|
||||
path_base_field: dna.Field = None,
|
||||
block_name: bytes = '',
|
||||
):
|
||||
if block_name:
|
||||
self.block_name = block_name
|
||||
else:
|
||||
self.block_name = self.guess_block_name(block)
|
||||
|
||||
assert isinstance(block, blendfile.BlendFileBlock)
|
||||
assert isinstance(asset_path, (bytes, bpathlib.BlendPath)), \
|
||||
'asset_path should be BlendPath, not %r' % type(asset_path)
|
||||
|
||||
if path_full_field is None:
|
||||
assert isinstance(path_dir_field, dna.Field), \
|
||||
'path_dir_field should be dna.Field, not %r' % type(path_dir_field)
|
||||
assert isinstance(path_base_field, dna.Field), \
|
||||
'path_base_field should be dna.Field, not %r' % type(path_base_field)
|
||||
else:
|
||||
assert isinstance(path_full_field, dna.Field), \
|
||||
'path_full_field should be dna.Field, not %r' % type(path_full_field)
|
||||
|
||||
if isinstance(asset_path, bytes):
|
||||
asset_path = bpathlib.BlendPath(asset_path)
|
||||
|
||||
self.block = block
|
||||
self.asset_path = asset_path
|
||||
self.is_sequence = bool(is_sequence)
|
||||
self.path_full_field = path_full_field
|
||||
self.path_dir_field = path_dir_field
|
||||
self.path_base_field = path_base_field
|
||||
self._abspath = None # cached by __fspath__()
|
||||
|
||||
@staticmethod
|
||||
def guess_block_name(block: blendfile.BlendFileBlock) -> bytes:
|
||||
try:
|
||||
return block[b'id', b'name']
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
return block[b'name']
|
||||
except KeyError:
|
||||
pass
|
||||
return b'-unnamed-'
|
||||
|
||||
def __repr__(self):
|
||||
if self.path_full_field is None:
|
||||
field_name = self.path_dir_field.name.name_full.decode() + \
|
||||
'/' + \
|
||||
self.path_base_field.name.name_full.decode()
|
||||
else:
|
||||
field_name = self.path_full_field.name.name_full.decode()
|
||||
return '<BlockUsage name=%r type=%r field=%r asset=%r%s>' % (
|
||||
self.block_name, self.block.dna_type_name,
|
||||
field_name, self.asset_path,
|
||||
' sequence' if self.is_sequence else ''
|
||||
)
|
||||
|
||||
def files(self) -> typing.Iterator[pathlib.Path]:
|
||||
"""Determine absolute path(s) of the asset file(s).
|
||||
|
||||
A relative path is interpreted relative to the blend file referring
|
||||
to the asset. If this BlockUsage represents a sequence, the filesystem
|
||||
is inspected and the actual files in the sequence are yielded.
|
||||
|
||||
It is assumed that paths are valid UTF-8.
|
||||
"""
|
||||
|
||||
path = self.__fspath__()
|
||||
if not self.is_sequence:
|
||||
if not path.exists():
|
||||
log.warning('Path %s does not exist for %s', path, self)
|
||||
return
|
||||
yield path
|
||||
return
|
||||
|
||||
try:
|
||||
yield from file_sequence.expand_sequence(path)
|
||||
except file_sequence.DoesNotExist:
|
||||
log.warning('Path %s does not exist for %s', path, self)
|
||||
|
||||
def __fspath__(self) -> pathlib.Path:
|
||||
"""Determine the absolute path of the asset on the filesystem."""
|
||||
if self._abspath is None:
|
||||
bpath = self.block.bfile.abspath(self.asset_path)
|
||||
self._abspath = bpath.to_path().resolve()
|
||||
return self._abspath
|
||||
|
||||
abspath = property(__fspath__)
|
||||
|
||||
def __lt__(self, other: 'BlockUsage'):
|
||||
"""Allow sorting for repeatable and predictable unit tests."""
|
||||
if not isinstance(other, BlockUsage):
|
||||
raise NotImplemented()
|
||||
return self.block_name < other.block_name and self.block < other.block
|
||||
|
||||
def __eq__(self, other: 'BlockUsage'):
|
||||
if not isinstance(other, BlockUsage):
|
||||
return False
|
||||
return self.block_name == other.block_name and self.block == other.block
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.block_name, hash(self.block)))
|
||||
Reference in New Issue
Block a user