From 89bcad5cdc9d63b992d00c6242fa8e4b95b695de Mon Sep 17 00:00:00 2001 From: Joseph HENRY Date: Thu, 10 Sep 2026 13:38:47 +0200 Subject: [PATCH] feature: pack folder --- CHANGELOG.md | 6 + blender_asset_tracer/__init__.py | 1 + blender_asset_tracer/cli/__init__.py | 1 + blender_asset_tracer/cli/pack.py | 156 +++++++++- blender_asset_tracer/operators.py | 119 +++++++- blender_asset_tracer/pack/__init__.py | 16 +- blender_asset_tracer/pack/folder.py | 227 ++++++++++++++ docs/cli.rst | 44 +++ tests/test_pack_folder.py | 424 ++++++++++++++++++++++++++ 9 files changed, 978 insertions(+), 16 deletions(-) create mode 100644 blender_asset_tracer/pack/folder.py create mode 100644 tests/test_pack_folder.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3095705..14052e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ This file logs the changes that are actually interesting to users (new features, changed functionality, fixed bugs). +# Unreleased + +- New `bat pack-folder` subcommand: scan a directory tree for blend files, trace the dependencies of all of them, and pack the whole lot into one directory or ZIP, storing shared assets only once. Backup files (`.blend1`, …) are skipped, and `--latest-only` reduces each `name_vNNN.blend` version series to its highest version (variants like `name_vNNN_light.blend` count as their own series). +- New "BAT - Pack Folder" operator in the add-on (`File > External Data > BAT`), offering the same folder scan and options. +- Fix packing a blend file that is both packed itself and referenced by another packed blend file through an absolute path: it is now packed at its own place in the pack, and the referring file is rewritten to point there, instead of the file being packed into `_outside_project`. This also affects `bat pack-sequence`. + # Version 1.21 (2025-11-24) - Require Python version 3.11 or newer. Versions up to Python 3.14 are supported. diff --git a/blender_asset_tracer/__init__.py b/blender_asset_tracer/__init__.py index 9b60cef..6b22db9 100644 --- a/blender_asset_tracer/__init__.py +++ b/blender_asset_tracer/__init__.py @@ -63,6 +63,7 @@ if _HAS_BPY: operators.BAT_OT_export_zip, operators.BAT_OT_scan_sequence, operators.BAT_OT_sequence_pack, + operators.BAT_OT_folder_pack, ) def register(): diff --git a/blender_asset_tracer/cli/__init__.py b/blender_asset_tracer/cli/__init__.py index 5fe640c..e9c2857 100644 --- a/blender_asset_tracer/cli/__init__.py +++ b/blender_asset_tracer/cli/__init__.py @@ -80,6 +80,7 @@ def cli_main(): blocks.add_parser(subparsers) pack.add_parser(subparsers) pack.add_sequence_parser(subparsers) + pack.add_folder_parser(subparsers) list_deps.add_parser(subparsers) version.add_parser(subparsers) diff --git a/blender_asset_tracer/cli/pack.py b/blender_asset_tracer/cli/pack.py index a5e9d2b..d664256 100755 --- a/blender_asset_tracer/cli/pack.py +++ b/blender_asset_tracer/cli/pack.py @@ -26,6 +26,7 @@ import typing import blender_asset_tracer.pack.transfer from blender_asset_tracer import pack, bpathlib +from blender_asset_tracer.pack import folder as folder_scan log = logging.getLogger(__name__) @@ -166,6 +167,102 @@ def add_sequence_parser(subparsers): ) +def add_folder_parser(subparsers): + """Add argparser for the pack-folder subcommand.""" + + parser = subparsers.add_parser( + "pack-folder", + help="Pack every blend file in a directory tree, together with their " + "dependencies, deduplicating shared assets.", + ) + parser.set_defaults(func=cli_pack_folder) + parser.add_argument( + "folder", + type=pathlib.Path, + help="Directory to scan recursively for blend files.", + ) + parser.add_argument( + "-t", + "--target", + type=str, + required=True, + help="Target directory or ZIP file.", + ) + parser.add_argument( + "-p", + "--project", + type=pathlib.Path, + help="Root directory of your project. Defaults to the scanned folder, " + "so that the pack mirrors the folder's own structure and assets from " + "outside it end up in '_outside_project'.", + ) + parser.add_argument( + "-n", + "--noop", + default=False, + action="store_true", + help="Don't copy files, just show what would be done.", + ) + parser.add_argument( + "-e", + "--exclude", + nargs="*", + default="", + help="Space-separated list of glob patterns to exclude.", + ) + parser.add_argument( + "-c", + "--compress", + default=False, + action="store_true", + help="Compress blend files while copying.", + ) + parser.add_argument( + "-r", + "--relative-only", + default=False, + action="store_true", + help="Only pack assets referred to with a relative path.", + ) + parser.add_argument( + "--keep-hierarchy", + default=False, + action="store_true", + help="Preserve the full filesystem directory hierarchy in the pack, " + "instead of packing relative to the scanned folder.", + ) + parser.add_argument( + "--latest-only", + default=False, + action="store_true", + help="Of every 'name_vNNN.blend' version series in a directory, only " + "pack the highest version. Variants like 'name_vNNN_light.blend' form " + "their own series; files without a version are always packed.", + ) + parser.add_argument( + "--follow-symlinks", + default=False, + action="store_true", + help="Descend into symlinked directories while scanning.", + ) + parser.add_argument( + "--ignore-dir", + nargs="*", + default=list(folder_scan.DEFAULT_IGNORE_DIRS), + metavar="GLOB", + help="Globs of directory names to skip while scanning (default: %s)." + % " ".join(folder_scan.DEFAULT_IGNORE_DIRS), + ) + parser.add_argument( + "--no-validate", + dest="validate", + default=True, + action="store_false", + help="Don't check that each '.blend' file really is a blend file. " + "Faster, but a non-blend file will abort the pack.", + ) + + def derive_common_project(bpaths: typing.List[pathlib.Path]) -> pathlib.Path: """Derive common project directory from multiple blend file paths. @@ -202,25 +299,58 @@ def cli_pack_sequence(args): log.critical("No target specified. Use -t/--target.") sys.exit(3) - bpaths, ppath, tpath = paths_from_cli(args) + run_packer(args, *paths_from_cli(args)) - with create_packer(args, bpaths, ppath, tpath) as packer: - packer.strategise() - try: - packer.execute() - except blender_asset_tracer.pack.transfer.FileTransferError as ex: - log.error( - "%d files couldn't be copied, starting with %s", - len(ex.files_remaining), - ex.files_remaining[0], - ) - raise SystemExit(1) + +def cli_pack_folder(args): + """CLI entry point for pack-folder subcommand.""" + fpath = args.folder + if not fpath.exists(): + log.critical("Folder %s does not exist", fpath) + sys.exit(3) + if not fpath.is_dir(): + log.critical("%s is not a directory", fpath) + sys.exit(3) + fpath = bpathlib.make_absolute(fpath) + + log.info("Scanning %s for blend files", fpath) + scan = folder_scan.find_blend_files( + fpath, + follow_symlinks=args.follow_symlinks, + ignore_dirs=args.ignore_dir, + latest_only=args.latest_only, + validate=args.validate, + ) + + for dirpath, reason in scan.unreadable_dirs: + log.error("Could not scan %s: %s", dirpath, reason) + if scan.skipped: + log.warning("Skipped %d file(s) while scanning", len(scan.skipped)) + if not scan.blendfiles: + log.critical("No blend files found in %s", fpath) + sys.exit(3) + log.warning("Found %d blend files in %s", len(scan.blendfiles), fpath) + + # Synthesize args to reuse paths_from_cli and create_packer. + args.blendfile = None + args.sequence = scan.blendfiles + if args.project is None: + args.project = fpath + + run_packer(args, *paths_from_cli(args)) def cli_pack(args): if args.sequence: log.warning("--sequence on 'pack' is deprecated. Use 'bat pack-sequence -t TARGET FILE...' instead.") - bpaths, ppath, tpath = paths_from_cli(args) + + run_packer(args, *paths_from_cli(args)) + + +def run_packer( + args, bpaths: typing.List[pathlib.Path], ppath: pathlib.Path, tpath: str +) -> None: + """Create the packer for these paths, and run it.""" with create_packer(args, bpaths, ppath, tpath) as packer: packer.strategise() diff --git a/blender_asset_tracer/operators.py b/blender_asset_tracer/operators.py index e45ea7a..77b1c6f 100644 --- a/blender_asset_tracer/operators.py +++ b/blender_asset_tracer/operators.py @@ -1,5 +1,4 @@ import os -import re import sys import subprocess import tempfile @@ -10,11 +9,12 @@ import bpy from bpy.types import Operator, PropertyGroup from bpy_extras.io_utils import ExportHelper +from blender_asset_tracer.pack import folder as folder_scan from blender_asset_tracer.pack import zipped, progress # Matches filenames like scene_v02.blend (single integer version only). # Does NOT match: _v02.1.blend, _v2-1.blend, or files without _vNN suffix. -VERSION_RE = re.compile(r'_v(\d+)\.blend$', re.IGNORECASE) +VERSION_RE = folder_scan.VERSION_RE class BlenderProgressCallback(progress.Callback): @@ -489,12 +489,127 @@ class BAT_OT_sequence_pack(Operator, ExportHelper): return {"FINISHED"} +class BAT_OT_folder_pack(Operator, ExportHelper): + """Pack every blend file of a folder, with their dependencies, into one zip""" + + bl_idname = "bat.folder_pack" + bl_label = "BAT - Pack Folder" + filename_ext = ".zip" + + filter_glob: bpy.props.StringProperty(default="*.zip", options={'HIDDEN'}) + + folder: bpy.props.StringProperty( + name="Folder", + description="Folder to scan recursively for blend files. " + "It is also used as project root: assets from outside it are packed " + "into '_outside_project'", + subtype='DIR_PATH', + ) + + latest_only: bpy.props.BoolProperty( + name="Latest Versions Only", + description="Of every 'name_vNNN.blend' version series in a folder, " + "only pack the highest version. Variants like 'name_vNNN_light.blend' " + "form their own series; files without a version are always packed", + default=False, + ) + + keep_hierarchy: bpy.props.BoolProperty( + name="Keep Hierarchy", + description="Preserve the full filesystem directory hierarchy in the " + "zip, instead of packing relative to the scanned folder", + default=False, + ) + + follow_symlinks: bpy.props.BoolProperty( + name="Follow Symlinks", + description="Descend into symlinked directories while scanning", + default=False, + ) + + def invoke(self, context, event): + if not self.folder and bpy.data.is_saved: + self.folder = str(Path(bpy.data.filepath).parent) + if self.folder: + folder_name = Path(self.folder.rstrip("/\\")).name + self.filepath = folder_name + "_bat_pack.zip" + return super().invoke(context, event) + + def draw(self, context): + layout = self.layout + layout.prop(self, "folder") + layout.prop(self, "latest_only") + layout.prop(self, "keep_hierarchy") + layout.prop(self, "follow_symlinks") + + def execute(self, context): + from blender_asset_tracer.pack.zipped import ZipPacker + + if not self.folder: + self.report({"ERROR"}, "No folder given") + return {"CANCELLED"} + + project = Path(self.folder) + if not project.is_dir(): + self.report({"ERROR"}, "%s is not a directory" % project) + return {"CANCELLED"} + + self.report({"INFO"}, "Scanning %s for blend files..." % project) + scan = folder_scan.find_blend_files( + project, + follow_symlinks=self.follow_symlinks, + latest_only=self.latest_only, + ) + + if scan.unreadable_dirs: + self.report({"WARNING"}, + "%d folder(s) could not be scanned" % len(scan.unreadable_dirs)) + if not scan.blendfiles: + self.report({"ERROR"}, "No blend files found in %s" % project) + return {"CANCELLED"} + + target = bpy.path.ensure_ext(self.filepath, ".zip") + self.report({"INFO"}, "Packing %d blend files..." % len(scan.blendfiles)) + + wm = context.window_manager + progress_cb = BlenderProgressCallback(wm) + + try: + with ZipPacker( + scan.blendfiles, + project, + target, + keep_hierarchy=self.keep_hierarchy, + ) as packer: + packer.progress_cb = progress_cb + packer.strategise() + packer.execute() + except Exception as ex: + import traceback + traceback.print_exc() + self.report({"ERROR"}, "Packing failed: %s" % str(ex)) + return {"CANCELLED"} + finally: + try: + wm.progress_end() + except Exception: + pass + + with zipfile.ZipFile(target) as inzip: + inzip.testzip() + + self.report({"INFO"}, "Written to %s" % target) + open_folder(Path(target).parent) + return {"FINISHED"} + + def menu_func(self, context): layout = self.layout layout.separator() layout.operator(ExportBatPack.bl_idname) filepath = layout.operator(BAT_OT_export_zip.bl_idname) layout.operator(BAT_OT_sequence_pack.bl_idname) + layout.operator(BAT_OT_folder_pack.bl_idname) try: prefs = bpy.context.preferences.addons["blender_asset_tracer"].preferences diff --git a/blender_asset_tracer/pack/__init__.py b/blender_asset_tracer/pack/__init__.py index b75f28c..3b832da 100755 --- a/blender_asset_tracer/pack/__init__.py +++ b/blender_asset_tracer/pack/__init__.py @@ -139,6 +139,7 @@ class Packer: ) # type: typing.DefaultDict[pathlib.Path, AssetAction] self.missing_files = set() # type: typing.Set[pathlib.Path] self._new_location_paths = set() # type: typing.Set[pathlib.Path] + self._top_level_paths = set() # type: typing.Set[pathlib.Path] self._output_path = None # type: typing.Optional[pathlib.PurePath] self._output_paths = [] # type: typing.List[pathlib.PurePath] @@ -254,6 +255,7 @@ class Packer: self._progress_cb.pack_start() self._new_location_paths = set() + self._top_level_paths = set() self._output_paths = [] for bf in self.blendfiles: @@ -272,8 +274,14 @@ class Packer: self._output_paths.append(bfile_pp) act = self._actions[bfile_path] - act.path_action = PathAction.KEEP_PATH + # This file may already have been seen as a dependency of another + # blend file we're packing. In that case it needs to keep its + # FIND_NEW_LOCATION action, so that the file referring to it is + # rewritten; only its location in the pack is decided here. + if act.path_action != PathAction.FIND_NEW_LOCATION: + act.path_action = PathAction.KEEP_PATH act.new_path = bfile_pp + self._top_level_paths.add(bfile_path) self._check_aborted() for usage in trace.deps(bf, self._progress_cb): @@ -371,6 +379,12 @@ class Packer: act = self._actions[path] assert isinstance(act, AssetAction) + if path in self._top_level_paths: + # This is one of the blend files we're packing, so it already + # has its place in the pack. Files referring to it are still + # rewritten, they'll just point at that place. + continue + relpath = bpathlib.strip_root(path) if self.keep_hierarchy: act.new_path = pathlib.Path(self._target_path, relpath) diff --git a/blender_asset_tracer/pack/folder.py b/blender_asset_tracer/pack/folder.py new file mode 100644 index 0000000..7a09fa1 --- /dev/null +++ b/blender_asset_tracer/pack/folder.py @@ -0,0 +1,227 @@ +# ***** BEGIN GPL LICENSE BLOCK ***** +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# ***** END GPL LICENCE BLOCK ***** +"""Scan a directory tree for blend files, to feed a folder-wide BAT pack. + +This module deliberately has no Blender dependencies, so that it can be used +both from the commandline (``bat pack-folder``) and from the add-on. +""" +import fnmatch +import logging +import os +import pathlib +import re +import typing + +from blender_asset_tracer.blendfile import magic_compression + +log = logging.getLogger(__name__) + +BLENDFILE_SUFFIX = ".blend" + +# Matches filenames like scene_v02.blend (single integer version only). +# Does NOT match: _v02.1.blend, _v2-1.blend, or files without _vNN suffix. +VERSION_RE = re.compile(r"_v(\d+)\.blend$", re.IGNORECASE) + +# Splits a versioned filename into the parts that identify its version series, +# so that scene_v02.blend, scene_v02_light.blend and scene_v02_work.blend are +# each versioned independently. The greedy prefix makes the *last* '_vNNN' of +# the name the version. +VERSION_SERIES_RE = re.compile( + r"^(?P.*)_v(?P\d+)(?P.*)\.blend$", re.IGNORECASE +) + +# Directory name globs that are skipped by default while scanning. +DEFAULT_IGNORE_DIRS = (".*", "__pycache__") + + +class ScanResult(typing.NamedTuple): + """The outcome of scanning a directory tree for blend files.""" + + blendfiles: typing.List[pathlib.Path] + """Blend files to pack, in deterministic (sorted) order.""" + + skipped: typing.List[typing.Tuple[pathlib.Path, str]] + """Files that were found but not packed, as (path, reason) tuples.""" + + unreadable_dirs: typing.List[typing.Tuple[pathlib.Path, str]] + """Directories that could not be scanned, as (path, error) tuples.""" + + +def find_blend_files( + folder: pathlib.Path, + *, + follow_symlinks: bool = False, + ignore_dirs: typing.Sequence[str] = DEFAULT_IGNORE_DIRS, + latest_only: bool = False, + validate: bool = True, +) -> ScanResult: + """Recursively find the blend files in ``folder``. + + Backup files (``.blend1``, ``.blend2``, …) are never returned, as only the + ``.blend`` suffix is matched. + + :param folder: Directory to scan. + :param follow_symlinks: Whether to descend into symlinked directories. + Already-visited directories are skipped, so symlink loops are safe. + :param ignore_dirs: Globs of directory names to skip entirely. + :param latest_only: Only keep the highest ``_vNNN`` file of each series. + :param validate: Check that each file actually starts with a blend file + magic (optionally compressed). Files that don't are reported as + skipped, instead of aborting the entire pack later on. + """ + blendfiles = [] # type: typing.List[pathlib.Path] + skipped = [] # type: typing.List[typing.Tuple[pathlib.Path, str]] + unreadable_dirs = [] # type: typing.List[typing.Tuple[pathlib.Path, str]] + + def on_error(ex: OSError) -> None: + path = pathlib.Path(ex.filename) if ex.filename else folder + reason = ex.strerror or str(ex) + log.warning("Unable to scan %s: %s", path, reason) + unreadable_dirs.append((path, reason)) + + visited_dirs = set() # type: typing.Set[typing.Tuple[int, int]] + if follow_symlinks: + _first_visit(folder, visited_dirs) + + for dirpath_str, dirnames, filenames in os.walk( + str(folder), onerror=on_error, followlinks=follow_symlinks + ): + dirpath = pathlib.Path(dirpath_str) + + # Prune ignored directories, and keep the walk order deterministic. + subdirs = sorted( + dirname for dirname in dirnames if not _matches_any(dirname, ignore_dirs) + ) + if follow_symlinks: + subdirs = [ + dirname + for dirname in subdirs + if _first_visit(dirpath / dirname, visited_dirs) + ] + dirnames[:] = subdirs + + for filename in sorted(filenames): + path = dirpath / filename + if path.suffix.lower() != BLENDFILE_SUFFIX: + continue + + if validate: + reason = why_not_a_blendfile(path) + if reason is not None: + log.warning("Skipping %s: %s", path, reason) + skipped.append((path, reason)) + continue + + blendfiles.append(path) + + if latest_only: + blendfiles, superseded = keep_latest_versions(blendfiles) + for path in superseded: + log.debug("Skipping %s: superseded by a newer version", path) + skipped.extend((path, "superseded by a newer version") for path in superseded) + + return ScanResult( + blendfiles=blendfiles, skipped=skipped, unreadable_dirs=unreadable_dirs + ) + + +def why_not_a_blendfile(path: pathlib.Path) -> typing.Optional[str]: + """Return None when the file looks like a blend file, else why it doesn't.""" + + try: + with path.open("rb") as fileobj: + compression = magic_compression.find_compression_type(fileobj) + except OSError as ex: + return "unable to read: %s" % (ex.strerror or ex) + + if compression == magic_compression.Compression.UNRECOGNISED: + return "not a blend file" + return None + + +def keep_latest_versions( + blendfiles: typing.Iterable[pathlib.Path], +) -> typing.Tuple[typing.List[pathlib.Path], typing.List[pathlib.Path]]: + """Reduce each ``name_vNNN.blend`` series to its highest version. + + Files whose name contains no ``_vNNN`` are always kept. Versions are + compared per directory and per name, so identically-named files in + different directories don't shadow each other, and variants such as + ``shot_v003_light.blend`` form their own series. + + :return: (files to keep, superseded files), both sorted. + """ + # Maps (directory, name without version) to the best (version, path) so far. + best = ( + {} + ) # type: typing.Dict[typing.Tuple[pathlib.Path, str, str], typing.Tuple[int, pathlib.Path]] + kept = [] # type: typing.List[pathlib.Path] + superseded = [] # type: typing.List[pathlib.Path] + + for path in blendfiles: + match = VERSION_SERIES_RE.match(path.name) + if match is None: + kept.append(path) + continue + + key = ( + path.parent, + match.group("prefix").lower(), + match.group("suffix").lower(), + ) + version = int(match.group("version")) + previous = best.get(key) + + if previous is None: + best[key] = (version, path) + elif version > previous[0]: + best[key] = (version, path) + superseded.append(previous[1]) + else: + superseded.append(path) + + kept.extend(path for _, path in best.values()) + kept.sort() + superseded.sort() + return kept, superseded + + +def _matches_any(name: str, patterns: typing.Sequence[str]) -> bool: + return any(fnmatch.fnmatch(name, pattern) for pattern in patterns) + + +def _first_visit( + dirpath: pathlib.Path, visited: typing.Set[typing.Tuple[int, int]] +) -> bool: + """Return True when this directory wasn't walked before. + + Used to keep ``follow_symlinks`` from looping forever. + """ + try: + stat = dirpath.stat() + except OSError as ex: + log.warning("Unable to inspect %s: %s", dirpath, ex.strerror or ex) + return False + + key = (stat.st_dev, stat.st_ino) + if key in visited: + log.warning("Not descending into %s again (symlink loop?)", dirpath) + return False + + visited.add(key) + return True diff --git a/docs/cli.rst b/docs/cli.rst index 2f00347..91c3a6f 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -83,3 +83,47 @@ The optional arguments influence the manner of packing:: to exclude. For more information see the chapter :ref:`packing`. + + +Pack folder +----------- + +The ``bat pack-folder`` command scans a directory tree, collects every +``.blend`` file in it, traces the dependencies of all of them, and packs the +whole lot into a single target. Assets used by more than one blend file are +only stored once:: + + bat pack-folder [-h] -t TARGET [-p PROJECT] [-n] [-e [EXCLUDE ...]] [-c] + [-r] [--keep-hierarchy] [--latest-only] [--follow-symlinks] + [--ignore-dir [GLOB ...]] [--no-validate] folder + +For example, to zip up a project's work directory:: + + bat pack-folder /z/projects/garde_a_vue/work -t /tmp/gav_work.zip --latest-only + +Backup files (``.blend1``, ``.blend2``, …) are never packed. By default the +scanned folder also acts as the project root, so the pack mirrors the folder's +own structure and assets referenced from outside it end up in +``_outside_project``. Use ``--keep-hierarchy`` to mirror the full filesystem +paths instead. + +The options specific to this subcommand are:: + + -t TARGET, --target TARGET + Target directory or ZIP file (required). + --latest-only Of every 'name_vNNN.blend' version series in a + directory, only pack the highest version. Variants + like 'name_vNNN_light.blend' form their own series; + files without a version are always packed. + --follow-symlinks Descend into symlinked directories while scanning. + Already-visited directories are skipped, so symlink + loops are safe. + --ignore-dir [GLOB ...] + Globs of directory names to skip while scanning + (default: .* __pycache__). + --no-validate Don't check that each '.blend' file really is a + blend file. Faster, but a non-blend file will abort + the pack. + +The other options (``-p``, ``-n``, ``-e``, ``-c``, ``-r``, ``--keep-hierarchy``) +mean the same as for ``bat pack``. diff --git a/tests/test_pack_folder.py b/tests/test_pack_folder.py new file mode 100644 index 0000000..c3e5216 --- /dev/null +++ b/tests/test_pack_folder.py @@ -0,0 +1,424 @@ +"""Tests for the folder scanner and the 'bat pack-folder' subcommand.""" +import argparse +import pathlib +import shutil +import zipfile +from shutil import copyfile + +import pytest + +from blender_asset_tracer import blendfile +from blender_asset_tracer.pack import folder as folder_scan + +BLENDFILES = pathlib.Path(__file__).with_name("blendfiles") + + +def make_blend(path: pathlib.Path) -> pathlib.Path: + """Create a file that passes the blend file magic check.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"BLENDER-v303RENDH" + b"\0" * 32) + return path + + +def names(paths, relative_to: pathlib.Path): + return sorted(str(p.relative_to(relative_to)) for p in paths) + + +@pytest.fixture(autouse=True) +def close_cached_blendfiles(): + yield + blendfile.close_all_cached() + + +class TestFindBlendFiles: + def test_finds_files_recursively_in_sorted_order(self, tmp_path): + make_blend(tmp_path / "b.blend") + make_blend(tmp_path / "a.blend") + make_blend(tmp_path / "sub" / "deep" / "c.blend") + + scan = folder_scan.find_blend_files(tmp_path) + + assert names(scan.blendfiles, tmp_path) == [ + "a.blend", + "b.blend", + "sub/deep/c.blend", + ] + assert scan.skipped == [] + assert scan.unreadable_dirs == [] + + def test_ignores_backup_and_other_files(self, tmp_path): + make_blend(tmp_path / "scene.blend") + make_blend(tmp_path / "scene.blend1") + make_blend(tmp_path / "scene.blend2") + (tmp_path / "notes.txt").write_text("hi") + + scan = folder_scan.find_blend_files(tmp_path) + + assert names(scan.blendfiles, tmp_path) == ["scene.blend"] + # Non-.blend files aren't even considered, so they're not 'skipped'. + assert scan.skipped == [] + + def test_uppercase_suffix_is_found(self, tmp_path): + make_blend(tmp_path / "SCENE.BLEND") + + scan = folder_scan.find_blend_files(tmp_path) + + assert names(scan.blendfiles, tmp_path) == ["SCENE.BLEND"] + + def test_hidden_directories_are_pruned_by_default(self, tmp_path): + make_blend(tmp_path / "scene.blend") + make_blend(tmp_path / ".git" / "hidden.blend") + make_blend(tmp_path / "__pycache__" / "cached.blend") + + scan = folder_scan.find_blend_files(tmp_path) + + assert names(scan.blendfiles, tmp_path) == ["scene.blend"] + + def test_custom_ignore_dirs(self, tmp_path): + make_blend(tmp_path / "scene.blend") + make_blend(tmp_path / "backup_2024" / "old.blend") + make_blend(tmp_path / ".git" / "hidden.blend") + + scan = folder_scan.find_blend_files(tmp_path, ignore_dirs=["backup_*"]) + + # Only the custom pattern is applied, so '.git' is no longer pruned. + assert names(scan.blendfiles, tmp_path) == [".git/hidden.blend", "scene.blend"] + + def test_non_blendfile_content_is_skipped(self, tmp_path): + make_blend(tmp_path / "good.blend") + (tmp_path / "bad.blend").write_bytes(b"this is not a blend file") + + scan = folder_scan.find_blend_files(tmp_path) + + assert names(scan.blendfiles, tmp_path) == ["good.blend"] + assert len(scan.skipped) == 1 + skipped_path, reason = scan.skipped[0] + assert skipped_path == tmp_path / "bad.blend" + assert reason == "not a blend file" + + def test_validation_can_be_disabled(self, tmp_path): + make_blend(tmp_path / "good.blend") + (tmp_path / "bad.blend").write_bytes(b"this is not a blend file") + + scan = folder_scan.find_blend_files(tmp_path, validate=False) + + assert names(scan.blendfiles, tmp_path) == ["bad.blend", "good.blend"] + assert scan.skipped == [] + + def test_compressed_blendfile_is_accepted(self, tmp_path): + copyfile( + str(BLENDFILES / "basic_file_compressed.blend"), + str(tmp_path / "compressed.blend"), + ) + + scan = folder_scan.find_blend_files(tmp_path) + + assert names(scan.blendfiles, tmp_path) == ["compressed.blend"] + + def test_unreadable_directory_is_reported(self, tmp_path): + make_blend(tmp_path / "scene.blend") + bad_dir = tmp_path / "locked" + make_blend(bad_dir / "hidden.blend") + bad_dir.chmod(0o000) + + try: + scan = folder_scan.find_blend_files(tmp_path) + finally: + bad_dir.chmod(0o755) + + assert names(scan.blendfiles, tmp_path) == ["scene.blend"] + assert len(scan.unreadable_dirs) == 1 + assert scan.unreadable_dirs[0][0] == bad_dir + + def test_symlinked_dir_skipped_by_default(self, tmp_path): + make_blend(tmp_path / "root" / "scene.blend") + make_blend(tmp_path / "elsewhere" / "linked.blend") + (tmp_path / "root" / "link").symlink_to(tmp_path / "elsewhere") + + scan = folder_scan.find_blend_files(tmp_path / "root") + + assert names(scan.blendfiles, tmp_path / "root") == ["scene.blend"] + + def test_symlinked_dir_followed_on_request(self, tmp_path): + make_blend(tmp_path / "root" / "scene.blend") + make_blend(tmp_path / "elsewhere" / "linked.blend") + (tmp_path / "root" / "link").symlink_to(tmp_path / "elsewhere") + + scan = folder_scan.find_blend_files(tmp_path / "root", follow_symlinks=True) + + assert names(scan.blendfiles, tmp_path / "root") == [ + "link/linked.blend", + "scene.blend", + ] + + def test_symlink_loop_does_not_hang(self, tmp_path): + root = tmp_path / "root" + make_blend(root / "scene.blend") + (root / "loop").symlink_to(root) + + scan = folder_scan.find_blend_files(root, follow_symlinks=True) + + assert names(scan.blendfiles, root) == ["scene.blend"] + + +class TestLatestOnly: + def test_keeps_highest_version(self, tmp_path): + for version in (0, 3, 12, 7): + make_blend(tmp_path / ("shot_v%03d.blend" % version)) + + scan = folder_scan.find_blend_files(tmp_path, latest_only=True) + + assert names(scan.blendfiles, tmp_path) == ["shot_v012.blend"] + assert len(scan.skipped) == 3 + assert all(reason == "superseded by a newer version" for _, reason in scan.skipped) + + def test_keeps_unversioned_files(self, tmp_path): + make_blend(tmp_path / "shot_v001.blend") + make_blend(tmp_path / "shot_v002.blend") + make_blend(tmp_path / "shot.blend") + make_blend(tmp_path / "no_version_here.blend") + + scan = folder_scan.find_blend_files(tmp_path, latest_only=True) + + assert names(scan.blendfiles, tmp_path) == [ + "no_version_here.blend", + "shot.blend", + "shot_v002.blend", + ] + + def test_variants_are_their_own_series(self, tmp_path): + for name in ( + "shot_v001.blend", + "shot_v003.blend", + "shot_v001_light.blend", + "shot_v002_light.blend", + "shot_v005_work.blend", + ): + make_blend(tmp_path / name) + + scan = folder_scan.find_blend_files(tmp_path, latest_only=True) + + assert names(scan.blendfiles, tmp_path) == [ + "shot_v002_light.blend", + "shot_v003.blend", + "shot_v005_work.blend", + ] + + def test_last_version_marker_wins(self, tmp_path): + make_blend(tmp_path / "char_v001_rig_v002.blend") + make_blend(tmp_path / "char_v001_rig_v007.blend") + make_blend(tmp_path / "char_v002_rig_v001.blend") + + scan = folder_scan.find_blend_files(tmp_path, latest_only=True) + + assert names(scan.blendfiles, tmp_path) == [ + "char_v001_rig_v007.blend", + "char_v002_rig_v001.blend", + ] + + def test_versions_are_grouped_per_directory_and_name(self, tmp_path): + make_blend(tmp_path / "anim" / "shot_v001.blend") + make_blend(tmp_path / "anim" / "shot_v002.blend") + make_blend(tmp_path / "light" / "shot_v001.blend") + make_blend(tmp_path / "light" / "other_v005.blend") + + scan = folder_scan.find_blend_files(tmp_path, latest_only=True) + + assert names(scan.blendfiles, tmp_path) == [ + "anim/shot_v002.blend", + "light/other_v005.blend", + "light/shot_v001.blend", + ] + + def test_case_insensitive_version_match(self, tmp_path): + make_blend(tmp_path / "SHOT_V001.BLEND") + make_blend(tmp_path / "SHOT_V002.BLEND") + + scan = folder_scan.find_blend_files(tmp_path, latest_only=True) + + assert names(scan.blendfiles, tmp_path) == ["SHOT_V002.BLEND"] + + +def folder_parser(): + from blender_asset_tracer.cli import pack as cli_pack + + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + cli_pack.add_folder_parser(subparsers) + return parser + + +class TestCLIFolderParsing: + def test_requires_target(self): + with pytest.raises(SystemExit): + folder_parser().parse_args(["pack-folder", "/some/folder"]) + + def test_requires_folder(self): + with pytest.raises(SystemExit): + folder_parser().parse_args(["pack-folder", "-t", "out.zip"]) + + def test_defaults(self): + args = folder_parser().parse_args( + ["pack-folder", "/some/folder", "-t", "out.zip"] + ) + assert args.folder == pathlib.Path("/some/folder") + assert args.target == "out.zip" + assert args.project is None + assert args.latest_only is False + assert args.keep_hierarchy is False + assert args.follow_symlinks is False + assert args.validate is True + assert args.ignore_dir == list(folder_scan.DEFAULT_IGNORE_DIRS) + + def test_flags(self): + args = folder_parser().parse_args([ + "pack-folder", "/some/folder", "-t", "out.zip", + "--latest-only", "--keep-hierarchy", "--no-validate", + "--ignore-dir", "backup*", "old*", + ]) + assert args.latest_only is True + assert args.keep_hierarchy is True + assert args.validate is False + assert args.ignore_dir == ["backup*", "old*"] + + +def run_pack_folder(argv): + args = folder_parser().parse_args(argv) + args.func(args) + + +def make_linked_pair(folder: pathlib.Path) -> None: + """Copy linked_cube.blend + the basic_file.blend it links to into `folder`.""" + folder.mkdir(parents=True, exist_ok=True) + for name in ("linked_cube.blend", "basic_file.blend"): + copyfile(str(BLENDFILES / name), str(folder / name)) + + +def make_absolute_link(blendpath: pathlib.Path, target: pathlib.Path) -> None: + """Rewrite the library path of `blendpath` to the absolute path of `target`.""" + bfile = blendfile.BlendFile(blendpath, mode="r+b") + try: + library = bfile.code_index[b"LI"][0] + abspath = str(target).encode() + library[b"filepath"] = abspath + library[b"name"] = abspath + finally: + bfile.close() + blendfile.close_all_cached() + + +class TestPackFolderEndToEnd: + def test_packs_folder_to_directory(self, tmp_path): + source = tmp_path / "project" + make_linked_pair(source) + target = tmp_path / "pack" + + run_pack_folder(["pack-folder", str(source), "-t", str(target)]) + + assert (target / "linked_cube.blend").exists() + assert (target / "basic_file.blend").exists() + assert (target / "pack-info.txt").exists() + # Everything was inside the project, so nothing is packed outside it. + assert not (target / "_outside_project").exists() + + info = (target / "pack-info.txt").read_text() + assert "2 blend files" in info + + def test_packs_folder_to_zip(self, tmp_path): + source = tmp_path / "project" + make_linked_pair(source) + target = tmp_path / "pack.zip" + + run_pack_folder(["pack-folder", str(source), "-t", str(target)]) + + with zipfile.ZipFile(str(target)) as inzip: + assert inzip.testzip() is None + arcnames = set(inzip.namelist()) + + assert "linked_cube.blend" in arcnames + assert "basic_file.blend" in arcnames + assert "pack-info.txt" in arcnames + + def test_nested_blendfiles_keep_their_relative_place(self, tmp_path): + source = tmp_path / "project" + make_linked_pair(source / "shots" / "sq01") + target = tmp_path / "pack" + + run_pack_folder(["pack-folder", str(source), "-t", str(target)]) + + assert (target / "shots" / "sq01" / "linked_cube.blend").exists() + assert (target / "shots" / "sq01" / "basic_file.blend").exists() + + def test_dependency_that_is_also_packed_stays_in_place(self, tmp_path): + """A blend file linked by absolute path is packed only at its own spot.""" + source = tmp_path / "project" + make_linked_pair(source) + make_absolute_link(source / "linked_cube.blend", source / "basic_file.blend") + target = tmp_path / "pack" + + run_pack_folder(["pack-folder", str(source), "-t", str(target)]) + + assert (target / "linked_cube.blend").exists() + assert (target / "basic_file.blend").exists() + # The absolute path was rewritten to the in-pack file, rather than + # duplicating basic_file.blend into _outside_project. + assert not (target / "_outside_project").exists() + + def test_latest_only_through_cli(self, tmp_path): + source = tmp_path / "project" + make_linked_pair(source) + copyfile( + str(BLENDFILES / "basic_file.blend"), str(source / "extra_v001.blend") + ) + copyfile( + str(BLENDFILES / "basic_file.blend"), str(source / "extra_v002.blend") + ) + target = tmp_path / "pack" + + run_pack_folder(["pack-folder", str(source), "-t", str(target), "--latest-only"]) + + assert (target / "extra_v002.blend").exists() + assert not (target / "extra_v001.blend").exists() + + def test_missing_folder_exits(self, tmp_path): + with pytest.raises(SystemExit) as exc: + run_pack_folder([ + "pack-folder", str(tmp_path / "nope"), "-t", str(tmp_path / "pack") + ]) + assert exc.value.code == 3 + + def test_file_instead_of_folder_exits(self, tmp_path): + somefile = make_blend(tmp_path / "scene.blend") + with pytest.raises(SystemExit) as exc: + run_pack_folder( + ["pack-folder", str(somefile), "-t", str(tmp_path / "pack")] + ) + assert exc.value.code == 3 + + def test_empty_folder_exits(self, tmp_path): + source = tmp_path / "empty" + source.mkdir() + with pytest.raises(SystemExit) as exc: + run_pack_folder( + ["pack-folder", str(source), "-t", str(tmp_path / "pack")] + ) + assert exc.value.code == 3 + + def test_exclude_globs_are_honoured(self, tmp_path): + source = tmp_path / "project" + source.mkdir() + shutil.copytree( + str(BLENDFILES / "textures"), str(source / "textures"), dirs_exist_ok=True + ) + copyfile( + str(BLENDFILES / "material_textures.blend"), + str(source / "material_textures.blend"), + ) + target = tmp_path / "pack" + + run_pack_folder( + ["pack-folder", str(source), "-t", str(target), "-e", "*.jpg"] + ) + + assert (target / "material_textures.blend").exists() + assert list(target.rglob("*.jpg")) == []