Added recursion into library blend files.

blender_asset_tracer/blendfile/__init__.py

@@ -30,6 +30,7 @@ import tempfile
 import typing
 
 from . import exceptions, dna_io, dna, header
+from blender_asset_tracer import bpathlib
 
 log = logging.getLogger(__name__)
 
@@ -277,6 +278,21 @@ class BlendFile:
 
         return structs, sdna_index_from_id
 
+    def abspath(self, relpath: bpathlib.BlendPath) -> bpathlib.BlendPath:
+        """Construct an absolute path from a blendfile-relative path."""
+
+        if relpath.is_absolute():
+            return relpath
+
+        bfile_dir = self.filepath.absolute().parent
+        root = bpathlib.BlendPath(bfile_dir)
+        abspath = relpath.absolute(root)
+
+        my_log = self.log.getChild('abspath')
+        my_log.info('Resolved %s relative to %s to %s', relpath, self.filepath, abspath)
+
+        return abspath
+
 
 class BlendFileBlock:
     """

blender_asset_tracer/bpathlib.py

@@ -5,6 +5,7 @@ or vice versa.
 """
 
 import os.path
+import pathlib
 import string
 
 
@@ -12,6 +13,8 @@ class BlendPath(bytes):
     """A path within Blender is always stored as bytes."""
 
     def __new__(cls, path):
+        if isinstance(path, pathlib.Path):
+            path = str(path)  # handle as string, which is encoded to bytes below.
         if isinstance(path, str):
             # As a convenience, when a string is given, interpret as UTF-8.
             return bytes.__new__(cls, path.encode('utf-8'))

blender_asset_tracer/tracer/__init__.py

@@ -2,25 +2,53 @@ import logging
 import pathlib
 import typing
 
-from blender_asset_tracer import blendfile
+from blender_asset_tracer import blendfile, bpathlib
 from . import result, block_walkers
 
 log = logging.getLogger(__name__)
 
 codes_to_skip = {
-    b'ID', b'WM', b'SN',  # These blocks never have external assets.
+    # These blocks never have external assets:
+    b'ID', b'WM', b'SN',
+
+    # These blocks are skipped for now, until we have proof they point to
+    # assets otherwise missed:
+    b'GR', b'WO', b'BR', b'LS',
 }
 
 
-def deps(bfilepath: pathlib.Path) -> typing.Iterator[result.BlockUsage]:
+def deps(bfilepath: pathlib.Path, recursive=False) -> typing.Iterator[result.BlockUsage]:
-    """Open the blend file and report its dependencies."""
+    """Open the blend file and report its dependencies.
+
+    :param bfilepath: File to open.
+    :param recursive: Also report dependencies inside linked blend files.
+    """
     log.info('Tracing %s', bfilepath)
 
     with blendfile.BlendFile(bfilepath) as bfile:
         for block in asset_holding_blocks(bfile):
             yield from block_walkers.from_block(block)
 
-        # TODO: handle library blocks for recursion.
+            if recursive and block.code == b'LI':
+                yield from _recurse_deps(block)
+
+
+def _recurse_deps(lib_block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
+    """Call deps() on the file linked from the library block."""
+    if lib_block.code != b'LI':
+        raise ValueError('Expected LI block, not %r' % lib_block)
+
+    relpath = bpathlib.BlendPath(lib_block[b'name'])
+    abspath = lib_block.bfile.abspath(relpath)
+
+    # Convert bytes to pathlib.Path object so we have a nice interface to work with.
+    # This assumes the path is encoded in UTF-8.
+    path = pathlib.Path(abspath.decode())
+    if not path.exists():
+        log.warning('Linked blend file %s (%s) does not exist; skipping.', relpath, path)
+        return
+
+    yield from deps(path, recursive=True)
 
 
 def asset_holding_blocks(bfile: blendfile.BlendFile) -> typing.Iterator[blendfile.BlendFileBlock]:

tests/abstract_test.py

@@ -7,7 +7,9 @@ class AbstractBlendFileTest(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.blendfiles = pathlib.Path(__file__).with_name('blendfiles')
-        logging.basicConfig(level=logging.INFO)
+        logging.basicConfig(
+            format='%(asctime)-15s %(levelname)8s %(name)s %(message)s',
+            level=logging.INFO)
 
     def setUp(self):
         self.bf = None

tests/test_tracer.py

@@ -1,7 +1,9 @@
 import collections
 import logging
+import typing
 
 from blender_asset_tracer import tracer, blendfile
+from blender_asset_tracer.blendfile import dna
 from abstract_test import AbstractBlendFileTest
 
 # Mimicks a BlockUsage, but without having to set the block to an expected value.
@@ -34,8 +36,8 @@ class AssetHoldingBlocksTest(AbstractTracerTest):
             # shouldn't be yielded.
             self.assertEqual(2, len(block.code))
 
-            # Library blocks should not be yielded either.
+            # World blocks should not yielded either.
-            self.assertNotEqual(b'LI', block.code)
+            self.assertNotEqual(b'WO', block.code)
 
             # Do some arbitrary tests that convince us stuff is read well.
             if block.code == b'SC':
@@ -57,26 +59,35 @@ class AssetHoldingBlocksTest(AbstractTracerTest):
         # The numbers here are taken from whatever the code does now; I didn't
         # count the actual blocks in the actual blend file.
         self.assertEqual(965, len(self.bf.blocks))
-        self.assertEqual(37, blocks_seen)
+        self.assertEqual(4, blocks_seen)
 
 
 class DepsTest(AbstractTracerTest):
+    @staticmethod
+    def field_name(field: dna.Field) -> typing.Optional[str]:
+        if field is None:
+            return None
+        return field.name.name_full.decode()
+
+    def assert_deps(self, blend_fname, expects: dict, recursive=False):
+        for dep in tracer.deps(self.blendfiles / blend_fname, recursive=recursive):
+            actual_type = dep.block.dna_type.dna_type_id.decode()
+            actual_full_field = self.field_name(dep.path_full_field)
+            actual_dirname = self.field_name(dep.path_dir_field)
+            actual_basename = self.field_name(dep.path_base_field)
+
+            actual = Expect(actual_type, actual_full_field, actual_dirname, actual_basename,
+                            dep.asset_path, dep.is_sequence)
 
-    def assert_deps(self, blend_fname, expects):
-        for dep in tracer.deps(self.blendfiles / blend_fname):
             exp = expects[dep.block_name]
-            self.assertEqual(exp.type, dep.block.dna_type.dna_type_id.decode())
+            if isinstance(exp, set):
-            self.assertEqual(exp.asset_path, dep.asset_path)
+                self.assertIn(actual, exp, msg='for block %s' % dep.block_name)
-            self.assertEqual(exp.is_sequence, dep.is_sequence)
+                exp.discard(actual)
-
+                if not exp:
-            if exp.full_field is not None:
+                    # Don't leave empty sets in expects.
-                self.assertEqual(exp.full_field, dep.path_full_field.name.name_full.decode())
+                    del expects[dep.block_name]
-            if exp.dirname_field is not None:
+            else:
-                self.assertEqual(exp.dirname_field, dep.path_dir_field.name.name_full.decode())
+                self.assertEqual(exp, actual, msg='for block %s' % dep.block_name)
-            if exp.basename_field is not None:
-                self.assertEqual(exp.basename_field, dep.path_base_field.name.name_full.decode())
-
-            # Each expectation should be seen only once.
                 del expects[dep.block_name]
 
         # All expected uses should have been seen.
@@ -161,3 +172,27 @@ class DepsTest(AbstractTracerTest):
         self.assert_deps('linked_cube.blend', {
             b'LILib': Expect('Library', 'name[1024]', None, None, b'//basic_file.blend', False),
         })
+
+    def test_deps_recursive(self):
+        self.assert_deps('doubly_linked.blend', {
+            b'LILib': {
+                # From doubly_linked.blend
+                Expect('Library', 'name[1024]', None, None, b'//linked_cube.blend', False),
+
+                # From linked_cube.blend
+                Expect('Library', 'name[1024]', None, None, b'//basic_file.blend', False),
+            },
+            b'LILib.002': Expect('Library', 'name[1024]', None, None,
+                                 b'//material_textures.blend', False),
+
+            # From material_texture.blend
+            b'IMbrick_dotted_04-bump': Expect(
+                'Image', 'name[1024]', None, None,
+                b'//textures/Bricks/brick_dotted_04-bump.jpg', False),
+            b'IMbrick_dotted_04-color': Expect(
+                'Image', 'name[1024]', None, None,
+                b'//textures/Bricks/brick_dotted_04-color.jpg', False),
+            b'IMbuildings_roof_04-color': Expect(
+                'Image', 'name[1024]', None, None,
+                b'//textures/Textures/Buildings/buildings_roof_04-color.png', False),
+        }, recursive=True)