diff --git a/blender_asset_tracer/bpathlib.py b/blender_asset_tracer/bpathlib.py new file mode 100644 index 0000000..228d482 --- /dev/null +++ b/blender_asset_tracer/bpathlib.py @@ -0,0 +1,62 @@ +"""Blender path support. + +Does not use pathlib, because we may have to handle POSIX paths on Windows +or vice versa. +""" + +import os.path +import string + + +class BlendPath(bytes): + """A path within Blender is always stored as bytes.""" + + def __new__(cls, path): + if isinstance(path, str): + # As a convenience, when a string is given, interpret as UTF-8. + return bytes.__new__(cls, path.encode('utf-8')) + return bytes.__new__(cls, path) + + def __str__(self) -> str: + """Decodes the path as UTF-8, replacing undecodable bytes. + + Undecodable bytes are ignored so this function can be safely used + for reporting. + """ + return self.decode('utf8', errors='replace') + + def is_blendfile_relative(self) -> bool: + return self[:2] == b'//' + + def is_absolute(self) -> bool: + if self.is_blendfile_relative(): + return False + if self[0:1] == b'/': + return True + + # Windows style path starting with drive letter. + if (len(self) >= 3 and + (self.decode('utf8'))[0] in string.ascii_letters and + self[1:2] == b':' and + self[2:3] in {b'\\', b'/'}): + return True + + return False + + def absolute(self, root: bytes=None) -> 'BlendPath': + """Determine absolute path. + + :param root: root directory to compute paths relative to. + For blendfile-relative paths, root should be the directory + containing the blendfile. If not given, blendfile-relative + paths cause a ValueError but filesystem-relative paths are + resolved based on the current working directory. + """ + if self.is_absolute(): + return self + + if self.is_blendfile_relative(): + my_relpath = self[2:] # strip off leading // + else: + my_relpath = self + return BlendPath(os.path.join(root, my_relpath)) diff --git a/blender_asset_tracer/tracer/__init__.py b/blender_asset_tracer/tracer/__init__.py new file mode 100644 index 0000000..5f0f89a --- /dev/null +++ b/blender_asset_tracer/tracer/__init__.py @@ -0,0 +1,37 @@ +import logging +import pathlib +import typing + +from blender_asset_tracer import bpathlib, blendfile +from . import result, blocks + +log = logging.getLogger(__name__) + +codes_to_skip = { + b'LI', # Library blocks we handle after the blend file itself. + b'ID', b'WM', b'SN', # These blocks never have external assets. +} + + +def deps(bfilepath: pathlib.Path) -> typing.Iterator[result.BlockUsage]: + log.info('Tracing %s', bfilepath) + + with blendfile.BlendFile(bfilepath) as bfile: + for block in asset_holding_blocks(bfile): + yield from blocks.from_block(block) + + # TODO: handle library blocks for recursion. + + +def asset_holding_blocks(bfile: blendfile.BlendFile) -> typing.Iterator[blendfile.BlendFileBlock]: + for block in bfile.blocks: + assert isinstance(block, blendfile.BlendFileBlock) + code = block.code + + # The longer codes are either arbitrary data or data blocks that + # don't refer to external assets. The former data blocks will be + # visited when we hit the two-letter datablocks that use them. + if len(code) > 2 or code in codes_to_skip: + continue + + yield block diff --git a/blender_asset_tracer/tracer/blocks.py b/blender_asset_tracer/tracer/blocks.py new file mode 100644 index 0000000..9cef305 --- /dev/null +++ b/blender_asset_tracer/tracer/blocks.py @@ -0,0 +1,49 @@ +import logging +import sys +import typing + +from blender_asset_tracer import blendfile +from . import result, cdefs + +log = logging.getLogger(__name__) + +_warned_about_types = set() + + +class NoReaderImplemented(NotImplementedError): + """There is no reader implementation for a specific block code.""" + + def __init__(self, message: str, code: bytes): + super().__init__(message) + self.code = code + + +def from_block(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]: + assert block.code != b'DATA' + + module = sys.modules[__name__] + funcname = '_from_block_' + block.code.decode().lower() + try: + block_reader = getattr(module, funcname) + except AttributeError: + if block.code not in _warned_about_types: + log.warning('No reader implemented for block type %r', block.code.decode()) + _warned_about_types.add(block.code) + return + + yield from block_reader(block) + + +def _from_block_im(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]: + # old files miss this + image_source = block.get(b'source', default=cdefs.IMA_SRC_FILE) + if image_source not in {cdefs.IMA_SRC_FILE, cdefs.IMA_SRC_SEQUENCE, cdefs.IMA_SRC_MOVIE}: + return + if block[b'packedfile']: + return + + pathname, field = block.get(b'name', return_field=True) + + # TODO: the receiver should inspect the 'source' property too, and if set + # to cdefs.IMA_SRC_SEQUENCE yield the entire sequence of files. + yield result.BlockUsage(block, field, pathname) diff --git a/blender_asset_tracer/tracer/cdefs.py b/blender_asset_tracer/tracer/cdefs.py new file mode 100644 index 0000000..2b86889 --- /dev/null +++ b/blender_asset_tracer/tracer/cdefs.py @@ -0,0 +1,47 @@ +# ***** BEGIN GPL LICENSE BLOCK ***** +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# ***** END GPL LICENCE BLOCK ***** + +"""Constants defined in C.""" + +# DNA_sequence_types.h (Sequence.type) +SEQ_TYPE_IMAGE = 0 +SEQ_TYPE_META = 1 +SEQ_TYPE_SCENE = 2 +SEQ_TYPE_MOVIE = 3 +SEQ_TYPE_SOUND_RAM = 4 +SEQ_TYPE_SOUND_HD = 5 +SEQ_TYPE_MOVIECLIP = 6 +SEQ_TYPE_MASK = 7 +SEQ_TYPE_EFFECT = 8 + +IMA_SRC_FILE = 1 +IMA_SRC_SEQUENCE = 2 +IMA_SRC_MOVIE = 3 + +# DNA_modifier_types.h +eModifierType_MeshCache = 46 + +# DNA_particle_types.h +PART_DRAW_OB = 7 +PART_DRAW_GR = 8 + +# DNA_object_types.h +# Object.transflag +OB_DUPLIGROUP = 1 << 8 + +CACHE_LIBRARY_SOURCE_CACHE = 1 diff --git a/blender_asset_tracer/tracer/result.py b/blender_asset_tracer/tracer/result.py new file mode 100644 index 0000000..a0a70e6 --- /dev/null +++ b/blender_asset_tracer/tracer/result.py @@ -0,0 +1,35 @@ + + +class BlockUsage: + """Represents the use of an asset by a data block. + + :ivar block: + :type block: blendfile.BlendFileBlock + :ivar field: + :type field: dna.Field + :ivar asset_path: + :type asset_path: bpathlib.BlendPath + """ + + def __init__(self, block, field, asset_path): + self.block_idname = block[b'id', b'name'] + + from blender_asset_tracer import blendfile, bpathlib + from blender_asset_tracer.blendfile import dna + + assert isinstance(block, blendfile.BlendFileBlock) + assert isinstance(field, dna.Field), 'field should be dna.Field, not %r' % type(field) + assert isinstance(asset_path, (bytes, bpathlib.BlendPath)), \ + 'asset_path should be BlendPath, not %r' % type(asset_path) + + if isinstance(asset_path, bytes): + asset_path = bpathlib.BlendPath(asset_path) + + self.block = block + self.field = field + self.asset_path = asset_path + + def __repr__(self): + return '' % ( + self.block_idname, self.block.dna_type_name, + self.field.name.name_full.decode(), self.asset_path) diff --git a/tests/blendfiles/material_textures.blend b/tests/blendfiles/material_textures.blend new file mode 100644 index 0000000..85e0328 Binary files /dev/null and b/tests/blendfiles/material_textures.blend differ diff --git a/tests/blendfiles/textures/Bricks/brick_dotted_04-bump.jpg b/tests/blendfiles/textures/Bricks/brick_dotted_04-bump.jpg new file mode 100644 index 0000000..e86fa7d Binary files /dev/null and b/tests/blendfiles/textures/Bricks/brick_dotted_04-bump.jpg differ diff --git a/tests/blendfiles/textures/Bricks/brick_dotted_04-color.jpg b/tests/blendfiles/textures/Bricks/brick_dotted_04-color.jpg new file mode 100644 index 0000000..eec7773 Binary files /dev/null and b/tests/blendfiles/textures/Bricks/brick_dotted_04-color.jpg differ diff --git a/tests/blendfiles/textures/Bricks/brick_dotted_04-specular.jpg b/tests/blendfiles/textures/Bricks/brick_dotted_04-specular.jpg new file mode 100644 index 0000000..de9cbfd Binary files /dev/null and b/tests/blendfiles/textures/Bricks/brick_dotted_04-specular.jpg differ diff --git a/tests/blendfiles/textures/Textures/Buildings/buildings_roof_04-color.png b/tests/blendfiles/textures/Textures/Buildings/buildings_roof_04-color.png new file mode 100644 index 0000000..1d024cf Binary files /dev/null and b/tests/blendfiles/textures/Textures/Buildings/buildings_roof_04-color.png differ diff --git a/tests/test_bpathlib.py b/tests/test_bpathlib.py new file mode 100644 index 0000000..be6def6 --- /dev/null +++ b/tests/test_bpathlib.py @@ -0,0 +1,44 @@ +import unittest + +from blender_asset_tracer.bpathlib import BlendPath + + +class BlendPathTest(unittest.TestCase): + def test_string_path(self): + p = BlendPath('//some/file.blend') + self.assertEqual(b'//some/file.blend', p) + + p = BlendPath(r'C:\some\file.blend') + self.assertEqual(b'C:\\some\\file.blend', p) + + def test_is_absolute(self): + self.assertFalse(BlendPath('//some/file.blend').is_absolute()) + self.assertTrue(BlendPath('/some/file.blend').is_absolute()) + self.assertTrue(BlendPath('C:/some/file.blend').is_absolute()) + self.assertFalse(BlendPath('some/file.blend').is_absolute()) + + self.assertFalse(BlendPath(b'//some/file.blend').is_absolute()) + self.assertTrue(BlendPath(b'/some/file.blend').is_absolute()) + self.assertTrue(BlendPath(b'C:/some/file.blend').is_absolute()) + self.assertFalse(BlendPath(b'some/file.blend').is_absolute()) + + def test_is_blendfile_relative(self): + self.assertTrue(BlendPath('//some/file.blend').is_blendfile_relative()) + self.assertFalse(BlendPath('/some/file.blend').is_blendfile_relative()) + self.assertFalse(BlendPath('C:/some/file.blend').is_blendfile_relative()) + self.assertFalse(BlendPath('some/file.blend').is_blendfile_relative()) + + self.assertTrue(BlendPath(b'//some/file.blend').is_blendfile_relative()) + self.assertFalse(BlendPath(b'/some/file.blend').is_blendfile_relative()) + self.assertFalse(BlendPath(b'C:/some/file.blend').is_blendfile_relative()) + self.assertFalse(BlendPath(b'some/file.blend').is_blendfile_relative()) + + def test_make_absolute(self): + self.assertEqual(b'/root/to/some/file.blend', + BlendPath(b'//some/file.blend').absolute(b'/root/to')) + self.assertEqual(b'/root/to/some/file.blend', + BlendPath(b'some/file.blend').absolute(b'/root/to')) + self.assertEqual(b'/root/to/../some/file.blend', + BlendPath(b'../some/file.blend').absolute(b'/root/to')) + self.assertEqual(b'/shared/some/file.blend', + BlendPath(b'/shared/some/file.blend').absolute(b'/root/to')) diff --git a/tests/test_tracer.py b/tests/test_tracer.py new file mode 100644 index 0000000..3f29a60 --- /dev/null +++ b/tests/test_tracer.py @@ -0,0 +1,66 @@ +import logging + +from abstract_test import AbstractBlendFileTest + +from blender_asset_tracer import tracer, blendfile + + +class AbstractTracerTest(AbstractBlendFileTest): + @classmethod + def setUpClass(cls): + super().setUpClass() + logging.basicConfig(level=logging.INFO) + logging.getLogger('blender_asset_tracer.tracer').setLevel(logging.DEBUG) + + +class AssetHoldingBlocksTest(AbstractTracerTest): + def setUp(self): + self.bf = blendfile.BlendFile(self.blendfiles / 'basic_file.blend') + + def test_simple_file(self): + # This file should not depend on external assets. + blocks_seen = 0 + seen_scene = seen_ob = False + + for block in tracer.asset_holding_blocks(self.bf): + assert isinstance(block, blendfile.BlendFileBlock) + blocks_seen += 1 + + # The four-letter-code blocks don't refer to assets, so they + # shouldn't be yielded. + self.assertEqual(2, len(block.code)) + + # Library blocks should not be yielded either. + self.assertNotEqual(b'LI', block.code) + + # Do some arbitrary tests that convince us stuff is read well. + if block.code == b'SC': + seen_scene = True + self.assertEqual(b'SCScene', block[b'id', b'name']) + continue + + if block.code == b'OB': + seen_ob = True + self.assertEqual('OBümlaut', block.get((b'id', b'name'), as_str=True)) + continue + + self.assertTrue(seen_scene) + self.assertTrue(seen_ob) + + # Many of the data blocks are skipped, because asset_holding_blocks() only + # yields top-level, directly-understandable blocks. + # + # The numbers here are taken from whatever the code does now; I didn't + # count the actual blocks in the actual blend file. + self.assertEqual(965, len(self.bf.blocks)) + self.assertEqual(37, blocks_seen) + + +class DepsTest(AbstractBlendFileTest): + def test_no_deps(self): + for dep in tracer.deps(self.blendfiles / 'basic_file.blend'): + self.fail(dep) + + def test_ob_mat_texture(self): + for dep in tracer.deps(self.blendfiles / 'material_textures.blend'): + self.fail(repr(dep))