Beginning of CLI interface + protection against infinite recursion.

.gitignore

@@ -4,7 +4,7 @@ __pycache__
 /*.egg-info/
 /.cache
 /.pytest_cache
-/.coverage
+.coverage
 
 /tests/blendfiles/cache_ocean/
 /tests/blendfiles/T53562/blendcache_bam_pack_bug/

README.md

@@ -6,3 +6,19 @@ Blender Asset Tracer, a.k.a. BAT🦇, is the replacement of [BAM](https://develo
 and [blender-file](https://developer.blender.org/source/blender-file/)
 
 Development is driven by choices explained in [T54125](https://developer.blender.org/T54125).
+
+
+## Paths
+
+There are two object types used to represent file paths. Those are strictly separated.
+
+1. `bpathlib.BlendPath` represents a path as stored in a blend file. It consists of bytes, and is
+   blendfile-relative when it starts with `//`. It can represent any path from any OS Blender
+   supports, and as such should be used carefully.
+2. `pathlib.Path` represents an actual path, possibly on the local filesystem of the computer
+   running BAT. Any filesystem operation (such as checking whether it exists) must be done using a
+   `pathlib.Path`.
+
+When it is necessary to interpret a `bpathlib.BlendPath` as a real path instead of a sequence of
+bytes, BAT assumes it is encoded as UTF-8. This is not necessarily true, and possibly using the
+local filesystem encoding might be better, but that is also no guarantee of correctness.

blender_asset_tracer/cli/__init__.py

@@ -0,0 +1,48 @@
+"""Commandline entry points."""
+
+import argparse
+import logging
+
+from . import common, pack, list_deps
+
+
+def cli_main():
+    parser = argparse.ArgumentParser(description='BAT: Blender Asset Tracer')
+
+    # func is set by subparsers to indicate which function to run.
+    parser.set_defaults(func=None,
+                        loglevel=logging.WARNING)
+    loggroup = parser.add_mutually_exclusive_group()
+    loggroup.add_argument('-v', '--verbose', dest='loglevel',
+                          action='store_const', const=logging.INFO,
+                          help='Log INFO level and higher')
+    loggroup.add_argument('-d', '--debug', dest='loglevel',
+                          action='store_const', const=logging.DEBUG,
+                          help='Log everything')
+    loggroup.add_argument('-q', '--quiet', dest='loglevel',
+                          action='store_const', const=logging.ERROR,
+                          help='Log at ERROR level and higher')
+    subparsers = parser.add_subparsers(help='sub-command help')
+
+    pack.add_parser(subparsers)
+    list_deps.add_parser(subparsers)
+
+    args = parser.parse_args()
+    config_logging(args)
+
+    from blender_asset_tracer import __version__
+    log = logging.getLogger(__name__)
+    log.debug('Running BAT version %s', __version__)
+
+    if not args.func:
+        parser.error('No subcommand was given')
+    return args.func(args)
+
+
+def config_logging(args):
+    """Configures the logging system based on CLI arguments."""
+
+    logging.basicConfig(
+        level=args.loglevel,
+        format='%(asctime)-15s %(levelname)8s %(name)-40s %(message)s',
+    )

blender_asset_tracer/cli/common.py

@@ -0,0 +1,16 @@
+"""Common functionality for CLI parsers."""
+
+
+def add_flag(argparser, flag_name: str, **kwargs):
+    """Add a CLI argument for the flag.
+
+    The flag defaults to False, and when present on the CLI stores True.
+    """
+
+    argparser.add_argument('-%s' % flag_name[0],
+                           '--%s' % flag_name,
+                           default=False,
+                           action='store_true',
+                           **kwargs)
+
+

blender_asset_tracer/cli/list_deps.py

@@ -0,0 +1,36 @@
+"""List dependencies of a blend file."""
+import logging
+import pathlib
+
+from . import common
+
+log = logging.getLogger(__name__)
+
+
+def add_parser(subparsers):
+    """Add argparser for this subcommand."""
+
+    parser = subparsers.add_parser('list', help=__doc__)
+    parser.set_defaults(func=cli_list)
+    parser.add_argument('blendfile', type=pathlib.Path)
+    common.add_flag(parser, 'recursive', help='Also report dependencies of dependencies')
+    common.add_flag(parser, 'json', help='Output as JSON instead of human-readable text')
+
+
+def cli_list(args):
+    from blender_asset_tracer import tracer
+
+    bpath = args.blendfile
+    if not bpath.exists():
+        log.fatal('File %s does not exist', args.blendfile)
+        return 3
+
+    reported_files = set()
+    for usage in tracer.deps(bpath, recursive=args.recursive):
+        for path in usage.files():
+            path = path.resolve()
+            if path in reported_files:
+                log.debug('Already reported %s', path)
+                continue
+            print(path)
+            reported_files.add(path)

blender_asset_tracer/cli/pack.py

@@ -0,0 +1,12 @@
+"""Create a BAT-pack for the given blend file."""
+
+
+def add_parser(subparsers):
+    """Add argparser for this subcommand."""
+
+    parser = subparsers.add_parser('pack', help=__doc__)
+    parser.set_defaults(func=cli_pack)
+
+
+def cli_pack(args):
+    raise NotImplementedError('bat pack not implemented yet')

blender_asset_tracer/tracer/__init__.py

@@ -17,38 +17,63 @@ codes_to_skip = {
 }
 
 
+class _Tracer:
+    """Trace dependencies with protection against infinite loops.
+
+    Don't use this directly, use the function deps(...) instead.
+    """
+    def __init__(self):
+        self.seen_files = set()
+
+    def deps(self, bfilepath: pathlib.Path, recursive=False) -> typing.Iterator[result.BlockUsage]:
+        """Open the blend file and report its dependencies.
+
+        :param bfilepath: File to open.
+        :param recursive: Also report dependencies inside linked blend files.
+        """
+        log.info('Tracing %s', bfilepath)
+        self.seen_files.add(bfilepath)
+
+        with blendfile.BlendFile(bfilepath) as bfile:
+            for block in asset_holding_blocks(bfile):
+                yield from block_walkers.from_block(block)
+
+                if recursive and block.code == b'LI':
+                    yield from self._recurse_deps(block)
+
+    def _recurse_deps(self, lib_block: blendfile.BlendFileBlock) \
+            -> typing.Iterator[result.BlockUsage]:
+        """Call deps() on the file linked from the library block."""
+        if lib_block.code != b'LI':
+            raise ValueError('Expected LI block, not %r' % lib_block)
+
+        relpath = bpathlib.BlendPath(lib_block[b'name'])
+        abspath = lib_block.bfile.abspath(relpath)
+
+        # Convert bytes to pathlib.Path object so we have a nice interface to work with.
+        # This assumes the path is encoded in UTF-8.
+        path = pathlib.Path(abspath.decode()).resolve()
+        if not path.exists():
+            log.warning('Linked blend file %s (%s) does not exist; skipping.', relpath, path)
+            return
+
+        # Avoid infinite recursion.
+        if path in self.seen_files:
+            log.debug('ignoring file, already seen %s', path)
+            return
+
+        yield from self.deps(path, recursive=True)
+
+
 def deps(bfilepath: pathlib.Path, recursive=False) -> typing.Iterator[result.BlockUsage]:
     """Open the blend file and report its dependencies.
 
     :param bfilepath: File to open.
     :param recursive: Also report dependencies inside linked blend files.
     """
-    log.info('Tracing %s', bfilepath)
 
-    with blendfile.BlendFile(bfilepath) as bfile:
+    tracer = _Tracer()
-        for block in asset_holding_blocks(bfile):
+    yield from tracer.deps(bfilepath, recursive=recursive)
-            yield from block_walkers.from_block(block)
-
-            if recursive and block.code == b'LI':
-                yield from _recurse_deps(block)
-
-
-def _recurse_deps(lib_block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
-    """Call deps() on the file linked from the library block."""
-    if lib_block.code != b'LI':
-        raise ValueError('Expected LI block, not %r' % lib_block)
-
-    relpath = bpathlib.BlendPath(lib_block[b'name'])
-    abspath = lib_block.bfile.abspath(relpath)
-
-    # Convert bytes to pathlib.Path object so we have a nice interface to work with.
-    # This assumes the path is encoded in UTF-8.
-    path = pathlib.Path(abspath.decode())
-    if not path.exists():
-        log.warning('Linked blend file %s (%s) does not exist; skipping.', relpath, path)
-        return
-
-    yield from deps(path, recursive=True)
 
 
 def asset_holding_blocks(bfile: blendfile.BlendFile) -> typing.Iterator[blendfile.BlendFileBlock]:

blender_asset_tracer/tracer/file_sequence.py

@@ -0,0 +1,51 @@
+import logging
+import pathlib
+import typing
+
+log = logging.getLogger(__name__)
+
+
+class DoesNotExist(OSError):
+    """Indicates a path does not exist on the filesystem."""
+
+    def __init__(self, path: pathlib.Path):
+        super().__init__(path)
+        self.path = path
+
+
+def expand_sequence(path: pathlib.Path) -> typing.Iterator[pathlib.Path]:
+    """Expand a file sequence path into the actual file paths.
+
+    :param path: can be either a glob pattern (must contain a * character),
+        a directory, or the path of the first file in the sequence.
+    """
+
+    if '*' in str(path):  # assume it is a glob
+        import glob
+        log.debug('expanding glob %s', path)
+        for fname in sorted(glob.glob(str(path), recursive=True)):
+            yield pathlib.Path(fname)
+        return
+
+    if not path.exists():
+        raise DoesNotExist(path)
+
+    if path.is_dir():
+        log.debug('expanding directory %s', path)
+        yield from sorted(path.rglob('*'))
+        return
+
+    log.debug('expanding file sequence %s', path)
+
+    import string
+    stem_no_digits = path.stem.rstrip(string.digits)
+    if stem_no_digits == path.stem:
+        # Just a single file, no digits here.
+        yield path
+        return
+
+    # Return everything start starts with 'stem_no_digits' and ends with the
+    # same suffix as the first file. This may result in more files than used
+    # by Blender, but at least it shouldn't miss any.
+    pattern = '%s*%s' % (stem_no_digits, path.suffix)
+    yield from sorted(path.parent.glob(pattern))

blender_asset_tracer/tracer/result.py

@@ -1,5 +1,12 @@
+import logging
+import pathlib
+import typing
+
 from blender_asset_tracer import blendfile, bpathlib
 from blender_asset_tracer.blendfile import dna
+from . import file_sequence
+
+log = logging.getLogger(__name__)
 
 
 class BlockUsage:
@@ -79,3 +86,27 @@ class BlockUsage:
             self.path_full_field.name.name_full.decode(), self.asset_path,
             ' sequence' if self.is_sequence else ''
         )
+
+    def files(self) -> typing.Iterator[pathlib.Path]:
+        """Determine absolute path(s) of the asset file(s).
+
+        A relative path is interpreted relative to the blend file referring
+        to the asset. If this BlockUsage represents a sequence, the filesystem
+        is inspected and the actual files in the sequence are yielded.
+
+        It is assumed that paths are valid UTF-8.
+        """
+
+        bpath = self.block.bfile.abspath(self.asset_path)
+        path = pathlib.Path(bpath.decode())
+        if not self.is_sequence:
+            if not path.exists():
+                log.warning('Path %s does not exist for %s', path, self)
+                return
+            yield path
+            return
+
+        try:
+            yield from file_sequence.expand_sequence(path)
+        except file_sequence.DoesNotExist:
+            log.warning('Path %s does not exist for %s', path, self)

setup.py

@@ -36,7 +36,7 @@ setup(
         },
     entry_points={
         'console_scripts': [
-            # 'bf = bam.cli:main',
+            'bat = blender_asset_tracer.cli:cli_main',
         ],
     },
     zip_safe=True,

tests/test_tracer.py

@@ -1,5 +1,6 @@
 import collections
 import logging
+import sys
 import typing
 
 from blender_asset_tracer import tracer, blendfile
@@ -203,3 +204,15 @@ class DepsTest(AbstractTracerTest):
                 'PointCache', 'name[64]', None, None,
                 b'//blendcache_bam_pack_bug/particles_*.bphys', True),
         })
+
+    def test_recursion_loop(self):
+        infinite_bfile = self.blendfiles / 'recursive_dependency_1.blend'
+
+        reclim = sys.getrecursionlimit()
+        try:
+            sys.setrecursionlimit(80)
+            # This should finish without hitting the recursion limit.
+            for _ in tracer.deps(infinite_bfile, recursive=True):
+                pass
+        finally:
+            sys.setrecursionlimit(reclim)

tests/test_tracer_file_sequence.py

@@ -0,0 +1,39 @@
+from abstract_test import AbstractBlendFileTest
+
+from blender_asset_tracer.tracer import file_sequence
+
+
+class ExpandFileSequenceTest(AbstractBlendFileTest):
+    def setUp(self):
+        super().setUp()
+        self.imgseq = [self.blendfiles / ('imgseq/%06d.png' % num)
+                       for num in range(210, 215)]
+
+    def test_glob(self):
+        path = self.blendfiles / 'imgseq/*.png'
+        actual = list(file_sequence.expand_sequence(path))
+        self.assertEqual(self.imgseq, actual)
+
+    def test_directory(self):
+        path = self.blendfiles / 'imgseq'
+        actual = list(file_sequence.expand_sequence(path))
+        expected = self.imgseq + [self.blendfiles / 'imgseq/LICENSE.txt']
+        self.assertEqual(expected, actual)
+
+    def test_first_file(self):
+        path = self.blendfiles / 'imgseq/000210.png'
+        actual = list(file_sequence.expand_sequence(path))
+        self.assertEqual(self.imgseq, actual)
+
+    def test_nonexistent(self):
+        path = self.blendfiles / 'nonexistant'
+        with self.assertRaises(file_sequence.DoesNotExist) as raises:
+            for result in file_sequence.expand_sequence(path):
+                self.fail('unexpected result %r' % result)
+
+        self.assertEqual(path, raises.exception.path)
+
+    def test_non_sequence_file(self):
+        path = self.blendfiles / 'imgseq/LICENSE.txt'
+        actual = list(file_sequence.expand_sequence(path))
+        self.assertEqual([path], actual)