diff --git a/.gitignore b/.gitignore index abe64b1..3712a08 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ __pycache__ /*.egg-info/ /.cache /.pytest_cache -/.coverage +.coverage /tests/blendfiles/cache_ocean/ /tests/blendfiles/T53562/blendcache_bam_pack_bug/ diff --git a/README.md b/README.md index e7f8142..cc332a0 100644 --- a/README.md +++ b/README.md @@ -6,3 +6,19 @@ Blender Asset Tracer, a.k.a. BAT🦇, is the replacement of [BAM](https://develo and [blender-file](https://developer.blender.org/source/blender-file/) Development is driven by choices explained in [T54125](https://developer.blender.org/T54125). + + +## Paths + +There are two object types used to represent file paths. Those are strictly separated. + +1. `bpathlib.BlendPath` represents a path as stored in a blend file. It consists of bytes, and is + blendfile-relative when it starts with `//`. It can represent any path from any OS Blender + supports, and as such should be used carefully. +2. `pathlib.Path` represents an actual path, possibly on the local filesystem of the computer + running BAT. Any filesystem operation (such as checking whether it exists) must be done using a + `pathlib.Path`. + +When it is necessary to interpret a `bpathlib.BlendPath` as a real path instead of a sequence of +bytes, BAT assumes it is encoded as UTF-8. This is not necessarily true, and possibly using the +local filesystem encoding might be better, but that is also no guarantee of correctness. diff --git a/blender_asset_tracer/cli/__init__.py b/blender_asset_tracer/cli/__init__.py new file mode 100644 index 0000000..327d7db --- /dev/null +++ b/blender_asset_tracer/cli/__init__.py @@ -0,0 +1,48 @@ +"""Commandline entry points.""" + +import argparse +import logging + +from . import common, pack, list_deps + + +def cli_main(): + parser = argparse.ArgumentParser(description='BAT: Blender Asset Tracer') + + # func is set by subparsers to indicate which function to run. + parser.set_defaults(func=None, + loglevel=logging.WARNING) + loggroup = parser.add_mutually_exclusive_group() + loggroup.add_argument('-v', '--verbose', dest='loglevel', + action='store_const', const=logging.INFO, + help='Log INFO level and higher') + loggroup.add_argument('-d', '--debug', dest='loglevel', + action='store_const', const=logging.DEBUG, + help='Log everything') + loggroup.add_argument('-q', '--quiet', dest='loglevel', + action='store_const', const=logging.ERROR, + help='Log at ERROR level and higher') + subparsers = parser.add_subparsers(help='sub-command help') + + pack.add_parser(subparsers) + list_deps.add_parser(subparsers) + + args = parser.parse_args() + config_logging(args) + + from blender_asset_tracer import __version__ + log = logging.getLogger(__name__) + log.debug('Running BAT version %s', __version__) + + if not args.func: + parser.error('No subcommand was given') + return args.func(args) + + +def config_logging(args): + """Configures the logging system based on CLI arguments.""" + + logging.basicConfig( + level=args.loglevel, + format='%(asctime)-15s %(levelname)8s %(name)-40s %(message)s', + ) diff --git a/blender_asset_tracer/cli/common.py b/blender_asset_tracer/cli/common.py new file mode 100644 index 0000000..3f9118a --- /dev/null +++ b/blender_asset_tracer/cli/common.py @@ -0,0 +1,16 @@ +"""Common functionality for CLI parsers.""" + + +def add_flag(argparser, flag_name: str, **kwargs): + """Add a CLI argument for the flag. + + The flag defaults to False, and when present on the CLI stores True. + """ + + argparser.add_argument('-%s' % flag_name[0], + '--%s' % flag_name, + default=False, + action='store_true', + **kwargs) + + diff --git a/blender_asset_tracer/cli/list_deps.py b/blender_asset_tracer/cli/list_deps.py new file mode 100644 index 0000000..e50e174 --- /dev/null +++ b/blender_asset_tracer/cli/list_deps.py @@ -0,0 +1,36 @@ +"""List dependencies of a blend file.""" +import logging +import pathlib + +from . import common + +log = logging.getLogger(__name__) + + +def add_parser(subparsers): + """Add argparser for this subcommand.""" + + parser = subparsers.add_parser('list', help=__doc__) + parser.set_defaults(func=cli_list) + parser.add_argument('blendfile', type=pathlib.Path) + common.add_flag(parser, 'recursive', help='Also report dependencies of dependencies') + common.add_flag(parser, 'json', help='Output as JSON instead of human-readable text') + + +def cli_list(args): + from blender_asset_tracer import tracer + + bpath = args.blendfile + if not bpath.exists(): + log.fatal('File %s does not exist', args.blendfile) + return 3 + + reported_files = set() + for usage in tracer.deps(bpath, recursive=args.recursive): + for path in usage.files(): + path = path.resolve() + if path in reported_files: + log.debug('Already reported %s', path) + continue + print(path) + reported_files.add(path) diff --git a/blender_asset_tracer/cli/pack.py b/blender_asset_tracer/cli/pack.py new file mode 100644 index 0000000..ba6b017 --- /dev/null +++ b/blender_asset_tracer/cli/pack.py @@ -0,0 +1,12 @@ +"""Create a BAT-pack for the given blend file.""" + + +def add_parser(subparsers): + """Add argparser for this subcommand.""" + + parser = subparsers.add_parser('pack', help=__doc__) + parser.set_defaults(func=cli_pack) + + +def cli_pack(args): + raise NotImplementedError('bat pack not implemented yet') diff --git a/blender_asset_tracer/tracer/__init__.py b/blender_asset_tracer/tracer/__init__.py index 186f049..bade862 100644 --- a/blender_asset_tracer/tracer/__init__.py +++ b/blender_asset_tracer/tracer/__init__.py @@ -17,38 +17,63 @@ codes_to_skip = { } +class _Tracer: + """Trace dependencies with protection against infinite loops. + + Don't use this directly, use the function deps(...) instead. + """ + def __init__(self): + self.seen_files = set() + + def deps(self, bfilepath: pathlib.Path, recursive=False) -> typing.Iterator[result.BlockUsage]: + """Open the blend file and report its dependencies. + + :param bfilepath: File to open. + :param recursive: Also report dependencies inside linked blend files. + """ + log.info('Tracing %s', bfilepath) + self.seen_files.add(bfilepath) + + with blendfile.BlendFile(bfilepath) as bfile: + for block in asset_holding_blocks(bfile): + yield from block_walkers.from_block(block) + + if recursive and block.code == b'LI': + yield from self._recurse_deps(block) + + def _recurse_deps(self, lib_block: blendfile.BlendFileBlock) \ + -> typing.Iterator[result.BlockUsage]: + """Call deps() on the file linked from the library block.""" + if lib_block.code != b'LI': + raise ValueError('Expected LI block, not %r' % lib_block) + + relpath = bpathlib.BlendPath(lib_block[b'name']) + abspath = lib_block.bfile.abspath(relpath) + + # Convert bytes to pathlib.Path object so we have a nice interface to work with. + # This assumes the path is encoded in UTF-8. + path = pathlib.Path(abspath.decode()).resolve() + if not path.exists(): + log.warning('Linked blend file %s (%s) does not exist; skipping.', relpath, path) + return + + # Avoid infinite recursion. + if path in self.seen_files: + log.debug('ignoring file, already seen %s', path) + return + + yield from self.deps(path, recursive=True) + + def deps(bfilepath: pathlib.Path, recursive=False) -> typing.Iterator[result.BlockUsage]: """Open the blend file and report its dependencies. :param bfilepath: File to open. :param recursive: Also report dependencies inside linked blend files. """ - log.info('Tracing %s', bfilepath) - with blendfile.BlendFile(bfilepath) as bfile: - for block in asset_holding_blocks(bfile): - yield from block_walkers.from_block(block) - - if recursive and block.code == b'LI': - yield from _recurse_deps(block) - - -def _recurse_deps(lib_block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]: - """Call deps() on the file linked from the library block.""" - if lib_block.code != b'LI': - raise ValueError('Expected LI block, not %r' % lib_block) - - relpath = bpathlib.BlendPath(lib_block[b'name']) - abspath = lib_block.bfile.abspath(relpath) - - # Convert bytes to pathlib.Path object so we have a nice interface to work with. - # This assumes the path is encoded in UTF-8. - path = pathlib.Path(abspath.decode()) - if not path.exists(): - log.warning('Linked blend file %s (%s) does not exist; skipping.', relpath, path) - return - - yield from deps(path, recursive=True) + tracer = _Tracer() + yield from tracer.deps(bfilepath, recursive=recursive) def asset_holding_blocks(bfile: blendfile.BlendFile) -> typing.Iterator[blendfile.BlendFileBlock]: diff --git a/blender_asset_tracer/tracer/file_sequence.py b/blender_asset_tracer/tracer/file_sequence.py new file mode 100644 index 0000000..54816a4 --- /dev/null +++ b/blender_asset_tracer/tracer/file_sequence.py @@ -0,0 +1,51 @@ +import logging +import pathlib +import typing + +log = logging.getLogger(__name__) + + +class DoesNotExist(OSError): + """Indicates a path does not exist on the filesystem.""" + + def __init__(self, path: pathlib.Path): + super().__init__(path) + self.path = path + + +def expand_sequence(path: pathlib.Path) -> typing.Iterator[pathlib.Path]: + """Expand a file sequence path into the actual file paths. + + :param path: can be either a glob pattern (must contain a * character), + a directory, or the path of the first file in the sequence. + """ + + if '*' in str(path): # assume it is a glob + import glob + log.debug('expanding glob %s', path) + for fname in sorted(glob.glob(str(path), recursive=True)): + yield pathlib.Path(fname) + return + + if not path.exists(): + raise DoesNotExist(path) + + if path.is_dir(): + log.debug('expanding directory %s', path) + yield from sorted(path.rglob('*')) + return + + log.debug('expanding file sequence %s', path) + + import string + stem_no_digits = path.stem.rstrip(string.digits) + if stem_no_digits == path.stem: + # Just a single file, no digits here. + yield path + return + + # Return everything start starts with 'stem_no_digits' and ends with the + # same suffix as the first file. This may result in more files than used + # by Blender, but at least it shouldn't miss any. + pattern = '%s*%s' % (stem_no_digits, path.suffix) + yield from sorted(path.parent.glob(pattern)) diff --git a/blender_asset_tracer/tracer/result.py b/blender_asset_tracer/tracer/result.py index 1e0e51f..f65b544 100644 --- a/blender_asset_tracer/tracer/result.py +++ b/blender_asset_tracer/tracer/result.py @@ -1,5 +1,12 @@ +import logging +import pathlib +import typing + from blender_asset_tracer import blendfile, bpathlib from blender_asset_tracer.blendfile import dna +from . import file_sequence + +log = logging.getLogger(__name__) class BlockUsage: @@ -79,3 +86,27 @@ class BlockUsage: self.path_full_field.name.name_full.decode(), self.asset_path, ' sequence' if self.is_sequence else '' ) + + def files(self) -> typing.Iterator[pathlib.Path]: + """Determine absolute path(s) of the asset file(s). + + A relative path is interpreted relative to the blend file referring + to the asset. If this BlockUsage represents a sequence, the filesystem + is inspected and the actual files in the sequence are yielded. + + It is assumed that paths are valid UTF-8. + """ + + bpath = self.block.bfile.abspath(self.asset_path) + path = pathlib.Path(bpath.decode()) + if not self.is_sequence: + if not path.exists(): + log.warning('Path %s does not exist for %s', path, self) + return + yield path + return + + try: + yield from file_sequence.expand_sequence(path) + except file_sequence.DoesNotExist: + log.warning('Path %s does not exist for %s', path, self) diff --git a/setup.py b/setup.py index f270874..2a882ad 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ setup( }, entry_points={ 'console_scripts': [ - # 'bf = bam.cli:main', + 'bat = blender_asset_tracer.cli:cli_main', ], }, zip_safe=True, diff --git a/tests/blendfiles/recursive_dependency_1.blend b/tests/blendfiles/recursive_dependency_1.blend new file mode 100644 index 0000000..e683ea2 Binary files /dev/null and b/tests/blendfiles/recursive_dependency_1.blend differ diff --git a/tests/blendfiles/recursive_dependency_2.blend b/tests/blendfiles/recursive_dependency_2.blend new file mode 100644 index 0000000..32cb77e Binary files /dev/null and b/tests/blendfiles/recursive_dependency_2.blend differ diff --git a/tests/test_tracer.py b/tests/test_tracer.py index ae309c4..31b0d8a 100644 --- a/tests/test_tracer.py +++ b/tests/test_tracer.py @@ -1,5 +1,6 @@ import collections import logging +import sys import typing from blender_asset_tracer import tracer, blendfile @@ -203,3 +204,15 @@ class DepsTest(AbstractTracerTest): 'PointCache', 'name[64]', None, None, b'//blendcache_bam_pack_bug/particles_*.bphys', True), }) + + def test_recursion_loop(self): + infinite_bfile = self.blendfiles / 'recursive_dependency_1.blend' + + reclim = sys.getrecursionlimit() + try: + sys.setrecursionlimit(80) + # This should finish without hitting the recursion limit. + for _ in tracer.deps(infinite_bfile, recursive=True): + pass + finally: + sys.setrecursionlimit(reclim) diff --git a/tests/test_tracer_file_sequence.py b/tests/test_tracer_file_sequence.py new file mode 100644 index 0000000..47f1730 --- /dev/null +++ b/tests/test_tracer_file_sequence.py @@ -0,0 +1,39 @@ +from abstract_test import AbstractBlendFileTest + +from blender_asset_tracer.tracer import file_sequence + + +class ExpandFileSequenceTest(AbstractBlendFileTest): + def setUp(self): + super().setUp() + self.imgseq = [self.blendfiles / ('imgseq/%06d.png' % num) + for num in range(210, 215)] + + def test_glob(self): + path = self.blendfiles / 'imgseq/*.png' + actual = list(file_sequence.expand_sequence(path)) + self.assertEqual(self.imgseq, actual) + + def test_directory(self): + path = self.blendfiles / 'imgseq' + actual = list(file_sequence.expand_sequence(path)) + expected = self.imgseq + [self.blendfiles / 'imgseq/LICENSE.txt'] + self.assertEqual(expected, actual) + + def test_first_file(self): + path = self.blendfiles / 'imgseq/000210.png' + actual = list(file_sequence.expand_sequence(path)) + self.assertEqual(self.imgseq, actual) + + def test_nonexistent(self): + path = self.blendfiles / 'nonexistant' + with self.assertRaises(file_sequence.DoesNotExist) as raises: + for result in file_sequence.expand_sequence(path): + self.fail('unexpected result %r' % result) + + self.assertEqual(path, raises.exception.path) + + def test_non_sequence_file(self): + path = self.blendfiles / 'imgseq/LICENSE.txt' + actual = list(file_sequence.expand_sequence(path)) + self.assertEqual([path], actual)