initial commit

trace/__init__.py

@@ -0,0 +1,80 @@
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+# ***** END GPL LICENCE BLOCK *****
+#
+# (c) 2018, Blender Foundation - Sybren A. Stüvel
+import logging
+import pathlib
+import typing
+
+from blender_asset_tracer import blendfile
+from . import result, blocks2assets, file2blocks, progress
+
+log = logging.getLogger(__name__)
+
+codes_to_skip = {
+    # These blocks never have external assets:
+    b'ID', b'WM', b'SN',
+
+    # These blocks are skipped for now, until we have proof they point to
+    # assets otherwise missed:
+    b'GR', b'WO', b'BR', b'LS',
+}
+
+
+def deps(bfilepath: pathlib.Path, progress_cb: typing.Optional[progress.Callback] = None) \
+        -> typing.Iterator[result.BlockUsage]:
+    """Open the blend file and report its dependencies.
+
+    :param bfilepath: File to open.
+    :param progress_cb: Progress callback object.
+    """
+
+    log.info('opening: %s', bfilepath)
+    bfile = blendfile.open_cached(bfilepath)
+
+    bi = file2blocks.BlockIterator()
+    if progress_cb:
+        bi.progress_cb = progress_cb
+
+    # Remember which block usages we've reported already, without keeping the
+    # blocks themselves in memory.
+    seen_hashes = set()  # type: typing.Set[int]
+
+    for block in asset_holding_blocks(bi.iter_blocks(bfile)):
+        for block_usage in blocks2assets.iter_assets(block):
+            usage_hash = hash(block_usage)
+            if usage_hash in seen_hashes:
+                continue
+            seen_hashes.add(usage_hash)
+            yield block_usage
+
+
+def asset_holding_blocks(blocks: typing.Iterable[blendfile.BlendFileBlock]) \
+        -> typing.Iterator[blendfile.BlendFileBlock]:
+    """Generator, yield data blocks that could reference external assets."""
+    for block in blocks:
+        assert isinstance(block, blendfile.BlendFileBlock)
+        code = block.code
+
+        # The longer codes are either arbitrary data or data blocks that
+        # don't refer to external assets. The former data blocks will be
+        # visited when we hit the two-letter datablocks that use them.
+        if len(code) > 2 or code in codes_to_skip:
+            continue
+
+        yield block

trace/blocks2assets.py

@@ -0,0 +1,210 @@
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+# ***** END GPL LICENCE BLOCK *****
+#
+# (c) 2014, Blender Foundation - Campbell Barton
+# (c) 2018, Blender Foundation - Sybren A. Stüvel
+"""Block walkers.
+
+From a Blend file data block, iter_assts() yields all the referred-to assets.
+"""
+
+import functools
+import logging
+import typing
+
+from blender_asset_tracer import blendfile, bpathlib, cdefs
+from blender_asset_tracer.blendfile import iterators
+from . import result, modifier_walkers
+
+log = logging.getLogger(__name__)
+
+_warned_about_types = set()  # type: typing.Set[bytes]
+_funcs_for_code = {}  # type: typing.Dict[bytes, typing.Callable]
+
+
+def iter_assets(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
+    """Generator, yield the assets used by this data block."""
+    assert block.code != b'DATA'
+
+    try:
+        block_reader = _funcs_for_code[block.code]
+    except KeyError:
+        if block.code not in _warned_about_types:
+            log.debug('No reader implemented for block type %r', block.code.decode())
+            _warned_about_types.add(block.code)
+        return
+
+    log.debug('Tracing block %r', block)
+    yield from block_reader(block)
+
+
+def dna_code(block_code: str):
+    """Decorator, marks decorated func as handler for that DNA code."""
+
+    assert isinstance(block_code, str)
+
+    def decorator(wrapped):
+        _funcs_for_code[block_code.encode()] = wrapped
+        return wrapped
+
+    return decorator
+
+
+def skip_packed(wrapped):
+    """Decorator, skip blocks where 'packedfile' is set to true."""
+
+    @functools.wraps(wrapped)
+    def wrapper(block: blendfile.BlendFileBlock, *args, **kwargs):
+        if block.get(b'packedfile', default=False):
+            log.debug('Datablock %r is packed; skipping', block.id_name)
+            return
+
+        yield from wrapped(block, *args, **kwargs)
+
+    return wrapper
+
+
+@dna_code('CF')
+def cache_file(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
+    """Cache file data blocks."""
+    path, field = block.get(b'filepath', return_field=True)
+    yield result.BlockUsage(block, path, path_full_field=field)
+
+
+@dna_code('IM')
+@skip_packed
+def image(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
+    """Image data blocks."""
+    # old files miss this
+    image_source = block.get(b'source', default=cdefs.IMA_SRC_FILE)
+    #print('------image_source: ', image_source)
+    #if image_source not in {cdefs.IMA_SRC_FILE, cdefs.IMA_SRC_SEQUENCE, cdefs.IMA_SRC_MOVIE}:
+    #    return
+    if image_source not in {cdefs.IMA_SRC_FILE, cdefs.IMA_SRC_SEQUENCE, cdefs.IMA_SRC_MOVIE, cdefs.IMA_SRC_TILED}:
+        return
+    pathname, field = block.get(b'name', return_field=True)
+    #is_sequence = image_source == cdefs.IMA_SRC_SEQUENCE
+
+    if image_source in {cdefs.IMA_SRC_SEQUENCE, cdefs.IMA_SRC_TILED}:
+        is_sequence = True
+    else:
+        is_sequence = False
+
+    #print('is_sequence: ', is_sequence)
+    yield result.BlockUsage(block, pathname, is_sequence, path_full_field=field)
+
+
+@dna_code('LI')
+def library(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
+    """Library data blocks."""
+    path, field = block.get(b'name', return_field=True)
+    yield result.BlockUsage(block, path, path_full_field=field)
+
+    # The 'filepath' also points to the blend file. However, this is set to the
+    # absolute path of the file by Blender (see BKE_library_filepath_set). This
+    # is thus not a property we have to report or rewrite.
+
+
+@dna_code('ME')
+def mesh(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
+    """Mesh data blocks."""
+    block_external = block.get_pointer((b'ldata', b'external'), None)
+    if block_external is None:
+        block_external = block.get_pointer((b'fdata', b'external'), None)
+    if block_external is None:
+        return
+
+    path, field = block_external.get(b'filename', return_field=True)
+    yield result.BlockUsage(block, path, path_full_field=field)
+
+
+@dna_code('MC')
+def movie_clip(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
+    """MovieClip data blocks."""
+    path, field = block.get(b'name', return_field=True)
+    # TODO: The assumption that this is not a sequence may not be true for all modifiers.
+    yield result.BlockUsage(block, path, is_sequence=False, path_full_field=field)
+
+
+@dna_code('OB')
+def object_block(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
+    """Object data blocks."""
+    ctx = modifier_walkers.ModifierContext(owner=block)
+
+    # 'ob->modifiers[...].filepath'
+    for mod_idx, block_mod in enumerate(iterators.modifiers(block)):
+        block_name = b'%s.modifiers[%d]' % (block.id_name, mod_idx)
+        mod_type = block_mod[b'modifier', b'type']
+        log.debug('Tracing modifier %s, type=%d', block_name.decode(), mod_type)
+
+        try:
+            mod_handler = modifier_walkers.modifier_handlers[mod_type]
+        except KeyError:
+            continue
+        yield from mod_handler(ctx, block_mod, block_name)
+
+
+@dna_code('SC')
+def scene(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
+    """Scene data blocks."""
+    # Sequence editor is the only interesting bit.
+    block_ed = block.get_pointer(b'ed')
+    if block_ed is None:
+        return
+
+    single_asset_types = {cdefs.SEQ_TYPE_MOVIE, cdefs.SEQ_TYPE_SOUND_RAM, cdefs.SEQ_TYPE_SOUND_HD}
+    asset_types = single_asset_types.union({cdefs.SEQ_TYPE_IMAGE})
+
+    for seq, seq_type in iterators.sequencer_strips(block_ed):
+        if seq_type not in asset_types:
+            continue
+
+        seq_strip = seq.get_pointer(b'strip')
+        if seq_strip is None:
+            continue
+        seq_stripdata = seq_strip.get_pointer(b'stripdata')
+        if seq_stripdata is None:
+            continue
+
+        dirname, dn_field = seq_strip.get(b'dir', return_field=True)
+        basename, bn_field = seq_stripdata.get(b'name', return_field=True)
+        asset_path = bpathlib.BlendPath(dirname) / basename
+
+        is_sequence = seq_type not in single_asset_types
+        yield result.BlockUsage(seq_strip, asset_path,
+                                is_sequence=is_sequence,
+                                path_dir_field=dn_field,
+                                path_base_field=bn_field)
+
+
+@dna_code('SO')
+@skip_packed
+def sound(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
+    """Sound data blocks."""
+    path, field = block.get(b'name', return_field=True)
+    yield result.BlockUsage(block, path, path_full_field=field)
+
+
+@dna_code('VF')
+@skip_packed
+def vector_font(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
+    """Vector Font data blocks."""
+    path, field = block.get(b'name', return_field=True)
+    if path == b'<builtin>':  # builtin font
+        return
+    yield result.BlockUsage(block, path, path_full_field=field)

trace/expanders.py

@@ -0,0 +1,294 @@
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+# ***** END GPL LICENCE BLOCK *****
+#
+# (c) 2014, Blender Foundation - Campbell Barton
+# (c) 2018, Blender Foundation - Sybren A. Stüvel
+"""Low-level functions called by file2block.
+
+Those can expand data blocks and yield their dependencies (e.g. other data
+blocks necessary to render/display/work with the given data block).
+"""
+import logging
+import typing
+
+from blender_asset_tracer import blendfile, cdefs
+from blender_asset_tracer.blendfile import iterators
+
+# Don't warn about these types at all.
+_warned_about_types = {b'LI', b'DATA'}
+_funcs_for_code = {}  # type: typing.Dict[bytes, typing.Callable]
+log = logging.getLogger(__name__)
+
+
+def expand_block(block: blendfile.BlendFileBlock) -> typing.Iterator[blendfile.BlendFileBlock]:
+    """Generator, yield the data blocks used by this data block."""
+
+    try:
+        expander = _funcs_for_code[block.code]
+    except KeyError:
+        if block.code not in _warned_about_types:
+            log.debug('No expander implemented for block type %r', block.code.decode())
+            _warned_about_types.add(block.code)
+        return
+
+    log.debug('Expanding block %r', block)
+    # Filter out falsy blocks, i.e. None values.
+    # Allowing expanders to yield None makes them more consise.
+    yield from filter(None, expander(block))
+
+
+def dna_code(block_code: str):
+    """Decorator, marks decorated func as expander for that DNA code."""
+
+    assert isinstance(block_code, str)
+
+    def decorator(wrapped):
+        _funcs_for_code[block_code.encode()] = wrapped
+        return wrapped
+
+    return decorator
+
+
+def _expand_generic_material(block: blendfile.BlendFileBlock):
+    array_len = block.get(b'totcol')
+    yield from block.iter_array_of_pointers(b'mat', array_len)
+
+
+def _expand_generic_mtex(block: blendfile.BlendFileBlock):
+    if not block.dna_type.has_field(b'mtex'):
+        # mtex was removed in Blender 2.8
+        return
+
+    for mtex in block.iter_fixed_array_of_pointers(b'mtex'):
+        yield mtex.get_pointer(b'tex')
+        yield mtex.get_pointer(b'object')
+
+
+def _expand_generic_nodetree(block: blendfile.BlendFileBlock):
+    assert block.dna_type.dna_type_id == b'bNodeTree'
+
+    nodes = block.get_pointer((b'nodes', b'first'))
+    for node in iterators.listbase(nodes):
+        if node[b'type'] == cdefs.CMP_NODE_R_LAYERS:
+            continue
+        yield node
+
+        # The 'id' property points to whatever is used by the node
+        # (like the image in an image texture node).
+        yield node.get_pointer(b'id')
+
+
+def _expand_generic_nodetree_id(block: blendfile.BlendFileBlock):
+    block_ntree = block.get_pointer(b'nodetree', None)
+    if block_ntree is not None:
+        yield from _expand_generic_nodetree(block_ntree)
+
+
+def _expand_generic_animdata(block: blendfile.BlendFileBlock):
+    block_adt = block.get_pointer(b'adt')
+    if block_adt:
+        yield block_adt.get_pointer(b'action')
+    # TODO, NLA
+
+
+@dna_code('AR')
+def _expand_armature(block: blendfile.BlendFileBlock):
+    yield from _expand_generic_animdata(block)
+
+
+@dna_code('CU')
+def _expand_curve(block: blendfile.BlendFileBlock):
+    yield from _expand_generic_animdata(block)
+    yield from _expand_generic_material(block)
+
+    for fieldname in (b'vfont', b'vfontb', b'vfonti', b'vfontbi',
+                      b'bevobj', b'taperobj', b'textoncurve'):
+        yield block.get_pointer(fieldname)
+
+
+@dna_code('GR')
+def _expand_group(block: blendfile.BlendFileBlock):
+    log.debug('Collection/group Block: %s (name=%s)', block, block.id_name)
+
+    objects = block.get_pointer((b'gobject', b'first'))
+    for item in iterators.listbase(objects):
+        yield item.get_pointer(b'ob')
+
+    # Recurse through child collections.
+    try:
+        children = block.get_pointer((b'children', b'first'))
+    except KeyError:
+        # 'children' was introduced in Blender 2.8 collections
+        pass
+    else:
+        for child in iterators.listbase(children):
+            subcoll = child.get_pointer(b'collection')
+            if subcoll is None:
+                continue
+
+            if subcoll.dna_type_id == b'ID':
+                # This issue happened while recursing a linked-in 'Hidden'
+                # collection in the Chimes set of the Spring project. Such
+                # collections named 'Hidden' were apparently created while
+                # converting files from Blender 2.79 to 2.80. This error
+                # isn't reproducible with just Blender 2.80.
+                yield subcoll
+                continue
+
+            log.debug('recursing into child collection %s (name=%r, type=%r)',
+                      subcoll, subcoll.id_name, subcoll.dna_type_name)
+            yield from _expand_group(subcoll)
+
+
+@dna_code('LA')
+def _expand_lamp(block: blendfile.BlendFileBlock):
+    yield from _expand_generic_animdata(block)
+    yield from _expand_generic_nodetree_id(block)
+    yield from _expand_generic_mtex(block)
+
+
+@dna_code('MA')
+def _expand_material(block: blendfile.BlendFileBlock):
+    yield from _expand_generic_animdata(block)
+    yield from _expand_generic_nodetree_id(block)
+    yield from _expand_generic_mtex(block)
+
+    try:
+        yield block.get_pointer(b'group')
+    except KeyError:
+        # Groups were removed from Blender 2.8
+        pass
+
+
+@dna_code('MB')
+def _expand_metaball(block: blendfile.BlendFileBlock):
+    yield from _expand_generic_animdata(block)
+    yield from _expand_generic_material(block)
+
+
+@dna_code('ME')
+def _expand_mesh(block: blendfile.BlendFileBlock):
+    yield from _expand_generic_animdata(block)
+    yield from _expand_generic_material(block)
+    yield block.get_pointer(b'texcomesh')
+    # TODO, TexFace? - it will be slow, we could simply ignore :S
+
+
+@dna_code('NT')
+def _expand_node_tree(block: blendfile.BlendFileBlock):
+    yield from _expand_generic_animdata(block)
+    yield from _expand_generic_nodetree(block)
+
+
+@dna_code('OB')
+def _expand_object(block: blendfile.BlendFileBlock):
+    yield from _expand_generic_animdata(block)
+    yield from _expand_generic_material(block)
+
+    yield block.get_pointer(b'data')
+
+    if block[b'transflag'] & cdefs.OB_DUPLIGROUP:
+        yield block.get_pointer(b'dup_group')
+
+    yield block.get_pointer(b'proxy')
+    yield block.get_pointer(b'proxy_group')
+
+    # 'ob->pose->chanbase[...].custom'
+    block_pose = block.get_pointer(b'pose')
+    if block_pose:
+        assert block_pose.dna_type.dna_type_id == b'bPose'
+        # sdna_index_bPoseChannel = block_pose.file.sdna_index_from_id[b'bPoseChannel']
+        channels = block_pose.get_pointer((b'chanbase', b'first'))
+        for pose_chan in iterators.listbase(channels):
+            yield pose_chan.get_pointer(b'custom')
+
+    # Expand the objects 'ParticleSettings' via 'ob->particlesystem[...].part'
+    # sdna_index_ParticleSystem = block.file.sdna_index_from_id.get(b'ParticleSystem')
+    # if sdna_index_ParticleSystem is not None:
+    psystems = block.get_pointer((b'particlesystem', b'first'))
+    for psystem in iterators.listbase(psystems):
+        yield psystem.get_pointer(b'part')
+
+    # Modifiers can also refer to other datablocks, which should also get expanded.
+    for block_mod in iterators.modifiers(block):
+        mod_type = block_mod[b'modifier', b'type']
+        # Currently only node groups are supported. If the support should expand
+        # to more types, something more intelligent than this should be made.
+        if mod_type == cdefs.eModifierType_Nodes:
+            yield block_mod.get_pointer(b'node_group')
+
+
+@dna_code('PA')
+def _expand_particle_settings(block: blendfile.BlendFileBlock):
+    yield from _expand_generic_animdata(block)
+    yield from _expand_generic_mtex(block)
+
+    block_ren_as = block[b'ren_as']
+    if block_ren_as == cdefs.PART_DRAW_GR:
+        yield block.get_pointer(b'dup_group')
+    elif block_ren_as == cdefs.PART_DRAW_OB:
+        yield block.get_pointer(b'dup_ob')
+
+
+@dna_code('SC')
+def _expand_scene(block: blendfile.BlendFileBlock):
+    yield from _expand_generic_animdata(block)
+    yield from _expand_generic_nodetree_id(block)
+    yield block.get_pointer(b'camera')
+    yield block.get_pointer(b'world')
+    yield block.get_pointer(b'set', default=None)
+    yield block.get_pointer(b'clip', default=None)
+
+    # sdna_index_Base = block.file.sdna_index_from_id[b'Base']
+    # for item in bf_utils.iter_ListBase(block.get_pointer((b'base', b'first'))):
+    #     yield item.get_pointer(b'object', sdna_index_refine=sdna_index_Base)
+    bases = block.get_pointer((b'base', b'first'))
+    for base in iterators.listbase(bases):
+        yield base.get_pointer(b'object')
+
+    # Sequence Editor
+    block_ed = block.get_pointer(b'ed')
+    if not block_ed:
+        return
+
+    strip_type_to_field = {
+        cdefs.SEQ_TYPE_SCENE: b'scene',
+        cdefs.SEQ_TYPE_MOVIECLIP: b'clip',
+        cdefs.SEQ_TYPE_MASK: b'mask',
+        cdefs.SEQ_TYPE_SOUND_RAM: b'sound',
+    }
+    for strip, strip_type in iterators.sequencer_strips(block_ed):
+        try:
+            field_name = strip_type_to_field[strip_type]
+        except KeyError:
+            continue
+        yield strip.get_pointer(field_name)
+
+
+@dna_code('TE')
+def _expand_texture(block: blendfile.BlendFileBlock):
+    yield from _expand_generic_animdata(block)
+    yield from _expand_generic_nodetree_id(block)
+    yield block.get_pointer(b'ima')
+
+
+@dna_code('WO')
+def _expand_world(block: blendfile.BlendFileBlock):
+    yield from _expand_generic_animdata(block)
+    yield from _expand_generic_nodetree_id(block)
+    yield from _expand_generic_mtex(block)

trace/file2blocks.py

@@ -0,0 +1,176 @@
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+# ***** END GPL LICENCE BLOCK *****
+#
+# (c) 2018, Blender Foundation - Sybren A. Stüvel
+"""Expand data blocks.
+
+The expansion process follows pointers and library links to construct the full
+set of actually-used data blocks. This set consists of all data blocks in the
+initial blend file, and all *actually linked-to* data blocks in linked
+blend files.
+"""
+import collections
+import logging
+import pathlib
+import queue
+import typing
+
+from blender_asset_tracer import blendfile, bpathlib
+from . import expanders, progress
+
+_funcs_for_code = {}  # type: typing.Dict[bytes, typing.Callable]
+log = logging.getLogger(__name__)
+
+
+# noinspection PyProtectedMember
+class BlockQueue(queue.PriorityQueue):
+    """PriorityQueue that sorts by filepath and file offset"""
+
+    def _put(self, item: blendfile.BlendFileBlock):
+        super()._put((item.bfile.filepath, item.file_offset, item))
+
+    def _get(self) -> blendfile.BlendFileBlock:
+        _, _, item = super()._get()
+        return item
+
+
+class BlockIterator:
+    """Expand blocks with dependencies from other libraries.
+
+    This class exists so that we have some context for the recursive expansion
+    without having to pass those variables to each recursive call.
+    """
+
+    def __init__(self) -> None:
+        # Set of (blend file Path, block address) of already-reported blocks.
+        self.blocks_yielded = set()  # type: typing.Set[typing.Tuple[pathlib.Path, int]]
+
+        # Queue of blocks to visit
+        self.to_visit = BlockQueue()
+
+        self.progress_cb = progress.Callback()
+
+    def iter_blocks(self,
+                    bfile: blendfile.BlendFile,
+                    limit_to: typing.Set[blendfile.BlendFileBlock] = set(),
+                    ) -> typing.Iterator[blendfile.BlendFileBlock]:
+        """Expand blocks with dependencies from other libraries."""
+
+        self.progress_cb.trace_blendfile(bfile.filepath)
+        log.info('inspecting: %s', bfile.filepath)
+        if limit_to:
+            self._queue_named_blocks(bfile, limit_to)
+        else:
+            self._queue_all_blocks(bfile)
+
+        blocks_per_lib = yield from self._visit_blocks(bfile, limit_to)
+        yield from self._visit_linked_blocks(blocks_per_lib)
+
+    def _visit_blocks(self, bfile, limit_to):
+        bpath = bpathlib.make_absolute(bfile.filepath)
+        root_dir = bpathlib.BlendPath(bpath.parent)
+
+        # Mapping from library path to data blocks to expand.
+        blocks_per_lib = collections.defaultdict(set)
+
+        while not self.to_visit.empty():
+            block = self.to_visit.get()
+            assert isinstance(block, blendfile.BlendFileBlock)
+            if (bpath, block.addr_old) in self.blocks_yielded:
+                continue
+
+            if block.code == b'ID':
+                # ID blocks represent linked-in assets. Those are the ones that
+                # should be loaded from their own blend file and "expanded" to
+                # the entire set of data blocks required to render them. We
+                # defer the handling of those so that we can work with one
+                # blend file at a time.
+                lib = block.get_pointer(b'lib')
+                lib_bpath = bpathlib.BlendPath(lib[b'name']).absolute(root_dir)
+                blocks_per_lib[lib_bpath].add(block)
+
+                # The library block itself should also be reported, because it
+                # represents a blend file that is a dependency as well.
+                self.to_visit.put(lib)
+                continue
+
+            if limit_to:
+                # We're limiting the blocks, so we have to expand them to make
+                # sure we don't miss anything. Otherwise we're yielding the
+                # entire file anyway, and no expansion is necessary.
+                self._queue_dependencies(block)
+            self.blocks_yielded.add((bpath, block.addr_old))
+            yield block
+
+        return blocks_per_lib
+
+    def _visit_linked_blocks(self, blocks_per_lib):
+        # We've gone through all the blocks in this file, now open the libraries
+        # and iterate over the blocks referred there.
+        for lib_bpath, idblocks in blocks_per_lib.items():
+            lib_path = bpathlib.make_absolute(lib_bpath.to_path())
+
+            #assert lib_path.exists()
+            if not lib_path.exists():
+                log.warning('Library %s does not exist', lib_path)
+                continue
+
+            log.debug('Expanding %d blocks in %s', len(idblocks), lib_path)
+            libfile = blendfile.open_cached(lib_path)
+            yield from self.iter_blocks(libfile, idblocks)
+
+    def _queue_all_blocks(self, bfile: blendfile.BlendFile):
+        log.debug('Queueing all blocks from file %s', bfile.filepath)
+        for block in bfile.blocks:
+            # Don't bother visiting DATA blocks, as we won't know what
+            # to do with them anyway.
+            if block.code == b'DATA':
+                continue
+            self.to_visit.put(block)
+
+    def _queue_named_blocks(self,
+                            bfile: blendfile.BlendFile,
+                            limit_to: typing.Set[blendfile.BlendFileBlock]):
+        """Queue only the blocks referred to in limit_to.
+
+        :param bfile:
+        :param limit_to: set of ID blocks that name the blocks to queue.
+            The queued blocks are loaded from the actual blend file, and
+            selected by name.
+        """
+
+        for to_find in limit_to:
+            assert to_find.code == b'ID'
+            name_to_find = to_find[b'name']
+            code = name_to_find[:2]
+            log.debug('Finding block %r with code %r', name_to_find, code)
+            same_code = bfile.find_blocks_from_code(code)
+            for block in same_code:
+                if block.id_name == name_to_find:
+                    log.debug('Queueing %r from file %s', block, bfile.filepath)
+                    self.to_visit.put(block)
+
+    def _queue_dependencies(self, block: blendfile.BlendFileBlock):
+        for block in expanders.expand_block(block):
+            self.to_visit.put(block)
+
+
+def iter_blocks(bfile: blendfile.BlendFile) -> typing.Iterator[blendfile.BlendFileBlock]:
+    """Generator, yield all blocks in this file + required blocks in libs."""
+    bi = BlockIterator()
+    yield from bi.iter_blocks(bfile)

trace/file_sequence.py

@@ -0,0 +1,69 @@
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+# ***** END GPL LICENCE BLOCK *****
+#
+# (c) 2018, Blender Foundation - Sybren A. Stüvel
+import logging
+import pathlib
+import typing
+
+log = logging.getLogger(__name__)
+
+
+class DoesNotExist(OSError):
+    """Indicates a path does not exist on the filesystem."""
+
+    def __init__(self, path: pathlib.Path) -> None:
+        super().__init__(path)
+        self.path = path
+
+
+def expand_sequence(path: pathlib.Path) -> typing.Iterator[pathlib.Path]:
+    """Expand a file sequence path into the actual file paths.
+
+    :param path: can be either a glob pattern (must contain a * character)
+        or the path of the first file in the sequence.
+    """
+
+    if '*' in str(path):  # assume it is a glob
+        import glob
+        log.debug('expanding glob %s', path)
+        for fname in sorted(glob.glob(str(path), recursive=True)):
+            yield pathlib.Path(fname)
+        return
+
+    if not path.exists():
+        raise DoesNotExist(path)
+
+    if path.is_dir():
+        yield path
+        return
+
+    log.debug('expanding file sequence %s', path)
+
+    import string
+    stem_no_digits = path.stem.rstrip(string.digits)
+    if stem_no_digits == path.stem:
+        # Just a single file, no digits here.
+        yield path
+        return
+
+    # Return everything start starts with 'stem_no_digits' and ends with the
+    # same suffix as the first file. This may result in more files than used
+    # by Blender, but at least it shouldn't miss any.
+    pattern = '%s*%s' % (stem_no_digits, path.suffix)
+    yield from sorted(path.parent.glob(pattern))

trace/modifier_walkers.py

@@ -0,0 +1,251 @@
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+# ***** END GPL LICENCE BLOCK *****
+#
+# (c) 2018, Blender Foundation - Sybren A. Stüvel
+"""Modifier handling code used in blocks2assets.py
+
+The modifier_xxx() functions all yield result.BlockUsage objects for external
+files used by the modifiers.
+"""
+import logging
+import typing
+
+from blender_asset_tracer import blendfile, bpathlib, cdefs
+from . import result
+
+log = logging.getLogger(__name__)
+modifier_handlers = {}  # type: typing.Dict[int, typing.Callable]
+
+
+class ModifierContext:
+    """Meta-info for modifier expansion.
+
+    Currently just contains the object on which the modifier is defined.
+    """
+    def __init__(self, owner: blendfile.BlendFileBlock) -> None:
+        assert owner.dna_type_name == 'Object'
+        self.owner = owner
+
+
+def mod_handler(dna_num: int):
+    """Decorator, marks decorated func as handler for that modifier number."""
+
+    assert isinstance(dna_num, int)
+
+    def decorator(wrapped):
+        modifier_handlers[dna_num] = wrapped
+        return wrapped
+
+    return decorator
+
+
+@mod_handler(cdefs.eModifierType_MeshCache)
+def modifier_filepath(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes) \
+        -> typing.Iterator[result.BlockUsage]:
+    """Just yield the 'filepath' field."""
+    path, field = modifier.get(b'filepath', return_field=True)
+    yield result.BlockUsage(modifier, path, path_full_field=field, block_name=block_name)
+
+
+@mod_handler(cdefs.eModifierType_MeshSequenceCache)
+def modifier_mesh_sequence_cache(ctx: ModifierContext, modifier: blendfile.BlendFileBlock,
+                                 block_name: bytes) -> typing.Iterator[result.BlockUsage]:
+    """Yield the Alembic file(s) used by this modifier"""
+    cache_file = modifier.get_pointer(b'cache_file')
+    if cache_file is None:
+        return
+
+    is_sequence = bool(cache_file[b'is_sequence'])
+    cache_block_name = cache_file.id_name
+    assert cache_block_name is not None
+
+    path, field = cache_file.get(b'filepath', return_field=True)
+    yield result.BlockUsage(cache_file, path, path_full_field=field,
+                            is_sequence=is_sequence,
+                            block_name=cache_block_name)
+
+
+@mod_handler(cdefs.eModifierType_Ocean)
+def modifier_ocean(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes) \
+        -> typing.Iterator[result.BlockUsage]:
+    if not modifier[b'cached']:
+        return
+
+    path, field = modifier.get(b'cachepath', return_field=True)
+    # The path indicates the directory containing the cached files.
+    yield result.BlockUsage(modifier, path, is_sequence=True, path_full_field=field,
+                            block_name=block_name)
+
+
+def _get_texture(prop_name: bytes, dblock: blendfile.BlendFileBlock, block_name: bytes) \
+        -> typing.Iterator[result.BlockUsage]:
+    """Yield block usages from a texture propery.
+
+    Assumes dblock[prop_name] is a texture data block.
+    """
+    if dblock is None:
+        return
+
+    tx = dblock.get_pointer(prop_name)
+    yield from _get_image(b'ima', tx, block_name)
+
+
+def _get_image(prop_name: bytes,
+               dblock: typing.Optional[blendfile.BlendFileBlock],
+               block_name: bytes) \
+        -> typing.Iterator[result.BlockUsage]:
+    """Yield block usages from an image propery.
+
+    Assumes dblock[prop_name] is an image data block.
+    """
+    if not dblock:
+        return
+
+    try:
+        ima = dblock.get_pointer(prop_name)
+    except KeyError as ex:
+        # No such property, just return.
+        log.debug('_get_image() called with non-existing property name: %s', ex)
+        return
+
+    if not ima:
+        return
+
+    path, field = ima.get(b'name', return_field=True)
+    yield result.BlockUsage(ima, path, path_full_field=field, block_name=block_name)
+
+
+@mod_handler(cdefs.eModifierType_Displace)
+@mod_handler(cdefs.eModifierType_Wave)
+def modifier_texture(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes) \
+        -> typing.Iterator[result.BlockUsage]:
+    return _get_texture(b'texture', modifier, block_name)
+
+
+@mod_handler(cdefs.eModifierType_WeightVGEdit)
+@mod_handler(cdefs.eModifierType_WeightVGMix)
+@mod_handler(cdefs.eModifierType_WeightVGProximity)
+def modifier_mask_texture(ctx: ModifierContext, modifier: blendfile.BlendFileBlock,
+                          block_name: bytes) \
+        -> typing.Iterator[result.BlockUsage]:
+    return _get_texture(b'mask_texture', modifier, block_name)
+
+
+@mod_handler(cdefs.eModifierType_UVProject)
+def modifier_image(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes) \
+        -> typing.Iterator[result.BlockUsage]:
+    yield from _get_image(b'image', modifier, block_name)
+
+
+def _walk_point_cache(ctx: ModifierContext,
+                      block_name: bytes,
+                      bfile: blendfile.BlendFile,
+                      pointcache: blendfile.BlendFileBlock,
+                      extension: bytes):
+    flag = pointcache[b'flag']
+    if flag & cdefs.PTCACHE_EXTERNAL:
+        path, field = pointcache.get(b'path', return_field=True)
+        log.info('    external cache at %s', path)
+        bpath = bpathlib.BlendPath(path)
+        yield result.BlockUsage(pointcache, bpath, path_full_field=field,
+                                is_sequence=True, block_name=block_name)
+    elif flag & cdefs.PTCACHE_DISK_CACHE:
+        # See ptcache_path() in pointcache.c
+        name, field = pointcache.get(b'name', return_field=True)
+        if not name:
+            # See ptcache_filename() in pointcache.c
+            idname = ctx.owner[b'id', b'name']
+            name = idname[2:].hex().upper().encode()
+        path = b'//%b%b/%b_*%b' % (
+            cdefs.PTCACHE_PATH,
+            bfile.filepath.stem.encode(),
+            name,
+            extension)
+        log.info('   disk cache at %s', path)
+        bpath = bpathlib.BlendPath(path)
+        yield result.BlockUsage(pointcache, bpath, path_full_field=field,
+                                is_sequence=True, block_name=block_name)
+
+
+@mod_handler(cdefs.eModifierType_ParticleSystem)
+def modifier_particle_system(ctx: ModifierContext, modifier: blendfile.BlendFileBlock,
+                             block_name: bytes) \
+        -> typing.Iterator[result.BlockUsage]:
+    psys = modifier.get_pointer(b'psys')
+    if psys is None:
+        return
+
+    pointcache = psys.get_pointer(b'pointcache')
+    if pointcache is None:
+        return
+
+    yield from _walk_point_cache(ctx, block_name, modifier.bfile, pointcache, cdefs.PTCACHE_EXT)
+
+
+@mod_handler(cdefs.eModifierType_Fluidsim)
+def modifier_fluid_sim(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes) \
+        -> typing.Iterator[result.BlockUsage]:
+    my_log = log.getChild('modifier_fluid_sim')
+
+    fss = modifier.get_pointer(b'fss')
+    if fss is None:
+        my_log.debug('Modifier %r (%r) has no fss',
+                     modifier[b'modifier', b'name'], block_name)
+        return
+
+    path, field = fss.get(b'surfdataPath', return_field=True)
+
+    # This may match more than is used by Blender, but at least it shouldn't
+    # miss any files.
+    # The 'fluidsurface' prefix is defined in source/blender/makesdna/DNA_object_fluidsim_types.h
+    bpath = bpathlib.BlendPath(path)
+    yield result.BlockUsage(fss, bpath, path_full_field=field,
+                            is_sequence=True, block_name=block_name)
+
+
+@mod_handler(cdefs.eModifierType_Smokesim)
+def modifier_smoke_sim(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes) \
+        -> typing.Iterator[result.BlockUsage]:
+    my_log = log.getChild('modifier_smoke_sim')
+
+    domain = modifier.get_pointer(b'domain')
+    if domain is None:
+        my_log.debug('Modifier %r (%r) has no domain',
+                     modifier[b'modifier', b'name'], block_name)
+        return
+
+    pointcache = domain.get_pointer(b'point_cache')
+    if pointcache is None:
+        return
+
+    format = domain.get(b'cache_file_format')
+    extensions = {
+        cdefs.PTCACHE_FILE_PTCACHE: cdefs.PTCACHE_EXT,
+        cdefs.PTCACHE_FILE_OPENVDB: cdefs.PTCACHE_EXT_VDB
+    }
+    yield from _walk_point_cache(ctx, block_name, modifier.bfile, pointcache, extensions[format])
+
+
+@mod_handler(cdefs.eModifierType_Cloth)
+def modifier_cloth(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes) \
+        -> typing.Iterator[result.BlockUsage]:
+    pointcache = modifier.get_pointer(b'point_cache')
+    if pointcache is None:
+        return
+
+    yield from _walk_point_cache(ctx, block_name, modifier.bfile, pointcache, cdefs.PTCACHE_EXT)

trace/progress.py

@@ -0,0 +1,31 @@
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+# ***** END GPL LICENCE BLOCK *****
+#
+# (c) 2018, Blender Foundation - Sybren A. Stüvel
+"""Callback class definition for BAT Tracer progress reporting.
+
+Mostly used to forward events to pack.progress.Callback.
+"""
+import pathlib
+
+
+class Callback:
+    """BAT Tracer progress reporting."""
+
+    def trace_blendfile(self, filename: pathlib.Path) -> None:
+        """Called for every blendfile opened when tracing dependencies."""

trace/result.py

@@ -0,0 +1,182 @@
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+# ***** END GPL LICENCE BLOCK *****
+#
+# (c) 2018, Blender Foundation - Sybren A. Stüvel
+import functools
+import logging
+import pathlib
+import typing
+
+from blender_asset_tracer import blendfile, bpathlib
+from blender_asset_tracer.blendfile import dna
+from . import file_sequence
+
+log = logging.getLogger(__name__)
+
+
+@functools.total_ordering
+class BlockUsage:
+    """Represents the use of an asset by a data block.
+
+    :ivar block_name: an identifying name for this block. Defaults to the ID
+        name of the block.
+    :ivar block:
+    :ivar asset_path: The path of the asset, if is_sequence=False. Otherwise
+        it can be either a glob pattern (must contain a * byte) or the path of
+        the first file in the sequence.
+    :ivar is_sequence: Indicates whether this file is alone (False), the
+        first of a sequence (True, and the path points to a file), or a
+        directory containing a sequence (True, and path points to a directory).
+        In certain cases such files should be reported once (f.e. when
+        rewriting the source field to another path), and in other cases the
+        sequence should be expanded (f.e. when copying all assets to a BAT
+        Pack).
+    :ivar path_full_field: field containing the full path of this asset.
+    :ivar path_dir_field: field containing the parent path (i.e. the
+        directory) of this asset.
+    :ivar path_base_field: field containing the basename of this asset.
+    """
+
+    def __init__(self,
+                 block: blendfile.BlendFileBlock,
+                 asset_path: bpathlib.BlendPath,
+                 is_sequence: bool = False,
+                 path_full_field: dna.Field = None,
+                 path_dir_field: dna.Field = None,
+                 path_base_field: dna.Field = None,
+                 block_name: bytes = b'',
+                 ) -> None:
+        if block_name:
+            self.block_name = block_name
+        else:
+            self.block_name = self.guess_block_name(block)
+
+        assert isinstance(block, blendfile.BlendFileBlock)
+        assert isinstance(asset_path, (bytes, bpathlib.BlendPath)), \
+            'asset_path should be BlendPath, not %r' % type(asset_path)
+
+        if path_full_field is None:
+            assert isinstance(path_dir_field, dna.Field), \
+                'path_dir_field should be dna.Field, not %r' % type(path_dir_field)
+            assert isinstance(path_base_field, dna.Field), \
+                'path_base_field should be dna.Field, not %r' % type(path_base_field)
+        else:
+            assert isinstance(path_full_field, dna.Field), \
+                'path_full_field should be dna.Field, not %r' % type(path_full_field)
+
+        if isinstance(asset_path, bytes):
+            asset_path = bpathlib.BlendPath(asset_path)
+
+        self.block = block
+        self.asset_path = asset_path
+        self.is_sequence = bool(is_sequence)
+        self.path_full_field = path_full_field
+        self.path_dir_field = path_dir_field
+        self.path_base_field = path_base_field
+
+        # cached by __fspath__()
+        self._abspath = None  # type: typing.Optional[pathlib.Path]
+
+    @staticmethod
+    def guess_block_name(block: blendfile.BlendFileBlock) -> bytes:
+        try:
+            return block[b'id', b'name']
+        except KeyError:
+            pass
+        try:
+            return block[b'name']
+        except KeyError:
+            pass
+        return b'-unnamed-'
+
+    def __repr__(self):
+        if self.path_full_field is None:
+            field_name = self.path_dir_field.name.name_full.decode() + \
+                '/' + \
+                self.path_base_field.name.name_full.decode()
+        else:
+            field_name = self.path_full_field.name.name_full.decode()
+        return '<BlockUsage name=%r type=%r field=%r asset=%r%s>' % (
+            self.block_name, self.block.dna_type_name,
+            field_name, self.asset_path,
+            ' sequence' if self.is_sequence else ''
+        )
+
+    def files(self) -> typing.Iterator[pathlib.Path]:
+        """Determine absolute path(s) of the asset file(s).
+
+        A relative path is interpreted relative to the blend file referring
+        to the asset. If this BlockUsage represents a sequence, the filesystem
+        is inspected and the actual files in the sequence are yielded.
+
+        It is assumed that paths are valid UTF-8.
+        """
+
+        path = self.__fspath__()
+        if not self.is_sequence:
+            if not path.exists():
+                log.warning('Path %s does not exist for %s', path, self)
+                return
+            yield path
+            return
+
+        try:
+            yield from file_sequence.expand_sequence(path)
+        except file_sequence.DoesNotExist:
+            log.warning('Path %s does not exist for %s', path, self)
+
+    def __fspath__(self) -> pathlib.Path:
+        """Determine the absolute path of the asset on the filesystem."""
+        if self._abspath is None:
+            bpath = self.block.bfile.abspath(self.asset_path)
+            log.info('Resolved %s rel to %s -> %s',
+                     self.asset_path, self.block.bfile.filepath, bpath)
+
+            as_path = pathlib.Path(bpath.to_path())
+
+            # Windows cannot make a path that has a glob pattern in it absolute.
+            # Since globs are generally only on the filename part, we take that off,
+            # make the parent directory absolute, then put the filename back.
+            try:
+                abs_parent = bpathlib.make_absolute(as_path.parent)
+            except FileNotFoundError:
+                self._abspath = as_path
+            else:
+                self._abspath = abs_parent / as_path.name
+
+            log.info('Resolving %s rel to %s -> %s',
+                     self.asset_path, self.block.bfile.filepath, self._abspath)
+        else:
+            log.info('Reusing abspath %s', self._abspath)
+        return self._abspath
+
+    abspath = property(__fspath__)
+
+    def __lt__(self, other: 'BlockUsage'):
+        """Allow sorting for repeatable and predictable unit tests."""
+        if not isinstance(other, BlockUsage):
+            raise NotImplemented()
+        return self.block_name < other.block_name and self.block < other.block
+
+    def __eq__(self, other: object):
+        if not isinstance(other, BlockUsage):
+            return False
+        return self.block_name == other.block_name and self.block == other.block
+
+    def __hash__(self):
+        return hash((self.block_name, hash(self.block)))