Start of dependency tracer work.

blender_asset_tracer/bpathlib.py

@@ -0,0 +1,62 @@
+"""Blender path support.
+
+Does not use pathlib, because we may have to handle POSIX paths on Windows
+or vice versa.
+"""
+
+import os.path
+import string
+
+
+class BlendPath(bytes):
+    """A path within Blender is always stored as bytes."""
+
+    def __new__(cls, path):
+        if isinstance(path, str):
+            # As a convenience, when a string is given, interpret as UTF-8.
+            return bytes.__new__(cls, path.encode('utf-8'))
+        return bytes.__new__(cls, path)
+
+    def __str__(self) -> str:
+        """Decodes the path as UTF-8, replacing undecodable bytes.
+
+        Undecodable bytes are ignored so this function can be safely used
+        for reporting.
+        """
+        return self.decode('utf8', errors='replace')
+
+    def is_blendfile_relative(self) -> bool:
+        return self[:2] == b'//'
+
+    def is_absolute(self) -> bool:
+        if self.is_blendfile_relative():
+            return False
+        if self[0:1] == b'/':
+            return True
+
+        # Windows style path starting with drive letter.
+        if (len(self) >= 3 and
+                (self.decode('utf8'))[0] in string.ascii_letters and
+                self[1:2] == b':' and
+                self[2:3] in {b'\\', b'/'}):
+            return True
+
+        return False
+
+    def absolute(self, root: bytes=None) -> 'BlendPath':
+        """Determine absolute path.
+
+        :param root: root directory to compute paths relative to.
+            For blendfile-relative paths, root should be the directory
+            containing the blendfile. If not given, blendfile-relative
+            paths cause a ValueError but filesystem-relative paths are
+            resolved based on the current working directory.
+        """
+        if self.is_absolute():
+            return self
+
+        if self.is_blendfile_relative():
+            my_relpath = self[2:]  # strip off leading //
+        else:
+            my_relpath = self
+        return BlendPath(os.path.join(root, my_relpath))

blender_asset_tracer/tracer/__init__.py

@@ -0,0 +1,37 @@
+import logging
+import pathlib
+import typing
+
+from blender_asset_tracer import bpathlib, blendfile
+from . import result, blocks
+
+log = logging.getLogger(__name__)
+
+codes_to_skip = {
+    b'LI',  # Library blocks we handle after the blend file itself.
+    b'ID', b'WM', b'SN',  # These blocks never have external assets.
+}
+
+
+def deps(bfilepath: pathlib.Path) -> typing.Iterator[result.BlockUsage]:
+    log.info('Tracing %s', bfilepath)
+
+    with blendfile.BlendFile(bfilepath) as bfile:
+        for block in asset_holding_blocks(bfile):
+            yield from blocks.from_block(block)
+
+        # TODO: handle library blocks for recursion.
+
+
+def asset_holding_blocks(bfile: blendfile.BlendFile) -> typing.Iterator[blendfile.BlendFileBlock]:
+    for block in bfile.blocks:
+        assert isinstance(block, blendfile.BlendFileBlock)
+        code = block.code
+
+        # The longer codes are either arbitrary data or data blocks that
+        # don't refer to external assets. The former data blocks will be
+        # visited when we hit the two-letter datablocks that use them.
+        if len(code) > 2 or code in codes_to_skip:
+            continue
+
+        yield block

blender_asset_tracer/tracer/blocks.py

@@ -0,0 +1,49 @@
+import logging
+import sys
+import typing
+
+from blender_asset_tracer import blendfile
+from . import result, cdefs
+
+log = logging.getLogger(__name__)
+
+_warned_about_types = set()
+
+
+class NoReaderImplemented(NotImplementedError):
+    """There is no reader implementation for a specific block code."""
+
+    def __init__(self, message: str, code: bytes):
+        super().__init__(message)
+        self.code = code
+
+
+def from_block(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
+    assert block.code != b'DATA'
+
+    module = sys.modules[__name__]
+    funcname = '_from_block_' + block.code.decode().lower()
+    try:
+        block_reader = getattr(module, funcname)
+    except AttributeError:
+        if block.code not in _warned_about_types:
+            log.warning('No reader implemented for block type %r', block.code.decode())
+            _warned_about_types.add(block.code)
+        return
+
+    yield from block_reader(block)
+
+
+def _from_block_im(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
+    # old files miss this
+    image_source = block.get(b'source', default=cdefs.IMA_SRC_FILE)
+    if image_source not in {cdefs.IMA_SRC_FILE, cdefs.IMA_SRC_SEQUENCE, cdefs.IMA_SRC_MOVIE}:
+        return
+    if block[b'packedfile']:
+        return
+
+    pathname, field = block.get(b'name', return_field=True)
+
+    # TODO: the receiver should inspect the 'source' property too, and if set
+    # to cdefs.IMA_SRC_SEQUENCE yield the entire sequence of files.
+    yield result.BlockUsage(block, field, pathname)

blender_asset_tracer/tracer/cdefs.py

@@ -0,0 +1,47 @@
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+# ***** END GPL LICENCE BLOCK *****
+
+"""Constants defined in C."""
+
+# DNA_sequence_types.h (Sequence.type)
+SEQ_TYPE_IMAGE = 0
+SEQ_TYPE_META = 1
+SEQ_TYPE_SCENE = 2
+SEQ_TYPE_MOVIE = 3
+SEQ_TYPE_SOUND_RAM = 4
+SEQ_TYPE_SOUND_HD = 5
+SEQ_TYPE_MOVIECLIP = 6
+SEQ_TYPE_MASK = 7
+SEQ_TYPE_EFFECT = 8
+
+IMA_SRC_FILE = 1
+IMA_SRC_SEQUENCE = 2
+IMA_SRC_MOVIE = 3
+
+# DNA_modifier_types.h
+eModifierType_MeshCache = 46
+
+# DNA_particle_types.h
+PART_DRAW_OB = 7
+PART_DRAW_GR = 8
+
+# DNA_object_types.h
+# Object.transflag
+OB_DUPLIGROUP = 1 << 8
+
+CACHE_LIBRARY_SOURCE_CACHE = 1

blender_asset_tracer/tracer/result.py

@@ -0,0 +1,35 @@
+
+
+class BlockUsage:
+    """Represents the use of an asset by a data block.
+
+    :ivar block:
+    :type block: blendfile.BlendFileBlock
+    :ivar field:
+    :type field: dna.Field
+    :ivar asset_path:
+    :type asset_path: bpathlib.BlendPath
+    """
+
+    def __init__(self, block, field, asset_path):
+        self.block_idname = block[b'id', b'name']
+
+        from blender_asset_tracer import blendfile, bpathlib
+        from blender_asset_tracer.blendfile import dna
+
+        assert isinstance(block, blendfile.BlendFileBlock)
+        assert isinstance(field, dna.Field), 'field should be dna.Field, not %r' % type(field)
+        assert isinstance(asset_path, (bytes, bpathlib.BlendPath)), \
+            'asset_path should be BlendPath, not %r' % type(asset_path)
+
+        if isinstance(asset_path, bytes):
+            asset_path = bpathlib.BlendPath(asset_path)
+
+        self.block = block
+        self.field = field
+        self.asset_path = asset_path
+
+    def __repr__(self):
+        return '<BlockUsage name=%r type=%r field=%r asset=%r>' % (
+            self.block_idname, self.block.dna_type_name,
+            self.field.name.name_full.decode(), self.asset_path)

tests/test_bpathlib.py

@@ -0,0 +1,44 @@
+import unittest
+
+from blender_asset_tracer.bpathlib import BlendPath
+
+
+class BlendPathTest(unittest.TestCase):
+    def test_string_path(self):
+        p = BlendPath('//some/file.blend')
+        self.assertEqual(b'//some/file.blend', p)
+
+        p = BlendPath(r'C:\some\file.blend')
+        self.assertEqual(b'C:\\some\\file.blend', p)
+
+    def test_is_absolute(self):
+        self.assertFalse(BlendPath('//some/file.blend').is_absolute())
+        self.assertTrue(BlendPath('/some/file.blend').is_absolute())
+        self.assertTrue(BlendPath('C:/some/file.blend').is_absolute())
+        self.assertFalse(BlendPath('some/file.blend').is_absolute())
+
+        self.assertFalse(BlendPath(b'//some/file.blend').is_absolute())
+        self.assertTrue(BlendPath(b'/some/file.blend').is_absolute())
+        self.assertTrue(BlendPath(b'C:/some/file.blend').is_absolute())
+        self.assertFalse(BlendPath(b'some/file.blend').is_absolute())
+
+    def test_is_blendfile_relative(self):
+        self.assertTrue(BlendPath('//some/file.blend').is_blendfile_relative())
+        self.assertFalse(BlendPath('/some/file.blend').is_blendfile_relative())
+        self.assertFalse(BlendPath('C:/some/file.blend').is_blendfile_relative())
+        self.assertFalse(BlendPath('some/file.blend').is_blendfile_relative())
+
+        self.assertTrue(BlendPath(b'//some/file.blend').is_blendfile_relative())
+        self.assertFalse(BlendPath(b'/some/file.blend').is_blendfile_relative())
+        self.assertFalse(BlendPath(b'C:/some/file.blend').is_blendfile_relative())
+        self.assertFalse(BlendPath(b'some/file.blend').is_blendfile_relative())
+
+    def test_make_absolute(self):
+        self.assertEqual(b'/root/to/some/file.blend',
+                         BlendPath(b'//some/file.blend').absolute(b'/root/to'))
+        self.assertEqual(b'/root/to/some/file.blend',
+                         BlendPath(b'some/file.blend').absolute(b'/root/to'))
+        self.assertEqual(b'/root/to/../some/file.blend',
+                         BlendPath(b'../some/file.blend').absolute(b'/root/to'))
+        self.assertEqual(b'/shared/some/file.blend',
+                         BlendPath(b'/shared/some/file.blend').absolute(b'/root/to'))

tests/test_tracer.py

@@ -0,0 +1,66 @@
+import logging
+
+from abstract_test import AbstractBlendFileTest
+
+from blender_asset_tracer import tracer, blendfile
+
+
+class AbstractTracerTest(AbstractBlendFileTest):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        logging.basicConfig(level=logging.INFO)
+        logging.getLogger('blender_asset_tracer.tracer').setLevel(logging.DEBUG)
+
+
+class AssetHoldingBlocksTest(AbstractTracerTest):
+    def setUp(self):
+        self.bf = blendfile.BlendFile(self.blendfiles / 'basic_file.blend')
+
+    def test_simple_file(self):
+        # This file should not depend on external assets.
+        blocks_seen = 0
+        seen_scene = seen_ob = False
+
+        for block in tracer.asset_holding_blocks(self.bf):
+            assert isinstance(block, blendfile.BlendFileBlock)
+            blocks_seen += 1
+
+            # The four-letter-code blocks don't refer to assets, so they
+            # shouldn't be yielded.
+            self.assertEqual(2, len(block.code))
+
+            # Library blocks should not be yielded either.
+            self.assertNotEqual(b'LI', block.code)
+
+            # Do some arbitrary tests that convince us stuff is read well.
+            if block.code == b'SC':
+                seen_scene = True
+                self.assertEqual(b'SCScene', block[b'id', b'name'])
+                continue
+
+            if block.code == b'OB':
+                seen_ob = True
+                self.assertEqual('OBümlaut', block.get((b'id', b'name'), as_str=True))
+                continue
+
+        self.assertTrue(seen_scene)
+        self.assertTrue(seen_ob)
+
+        # Many of the data blocks are skipped, because asset_holding_blocks() only
+        # yields top-level, directly-understandable blocks.
+        #
+        # The numbers here are taken from whatever the code does now; I didn't
+        # count the actual blocks in the actual blend file.
+        self.assertEqual(965, len(self.bf.blocks))
+        self.assertEqual(37, blocks_seen)
+
+
+class DepsTest(AbstractBlendFileTest):
+    def test_no_deps(self):
+        for dep in tracer.deps(self.blendfiles / 'basic_file.blend'):
+            self.fail(dep)
+
+    def test_ob_mat_texture(self):
+        for dep in tracer.deps(self.blendfiles / 'material_textures.blend'):
+            self.fail(repr(dep))