autour_de_minuit/blender-asset-tracer
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

feature: pack folder

Browse Source
This commit is contained in:
joseph.henry
2026-09-10 13:38:47 +02:00
parent 6998242b58
commit 89bcad5cdc
9 changed files with 978 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
blender_asset_tracer/__init__.py
+1
View File
@@ -63,6 +63,7 @@ if _HAS_BPY:
operators.BAT_OT_export_zip,
operators.BAT_OT_scan_sequence,
operators.BAT_OT_sequence_pack,
operators.BAT_OT_folder_pack,
)
def register():
blender_asset_tracer/cli/__init__.py
+1
View File
@@ -80,6 +80,7 @@ def cli_main():
blocks.add_parser(subparsers)
pack.add_parser(subparsers)
pack.add_sequence_parser(subparsers)
pack.add_folder_parser(subparsers)
list_deps.add_parser(subparsers)
version.add_parser(subparsers)
blender_asset_tracer/cli/pack.py
+143 -13
View File
@@ -26,6 +26,7 @@ import typing
import blender_asset_tracer.pack.transfer
from blender_asset_tracer import pack, bpathlib
from blender_asset_tracer.pack import folder as folder_scan
log = logging.getLogger(__name__)
@@ -166,6 +167,102 @@ def add_sequence_parser(subparsers):
)
def add_folder_parser(subparsers):
"""Add argparser for the pack-folder subcommand."""
parser = subparsers.add_parser(
"pack-folder",
help="Pack every blend file in a directory tree, together with their "
"dependencies, deduplicating shared assets.",
)
parser.set_defaults(func=cli_pack_folder)
parser.add_argument(
"folder",
type=pathlib.Path,
help="Directory to scan recursively for blend files.",
)
parser.add_argument(
"-t",
"--target",
type=str,
required=True,
help="Target directory or ZIP file.",
)
parser.add_argument(
"-p",
"--project",
type=pathlib.Path,
help="Root directory of your project. Defaults to the scanned folder, "
"so that the pack mirrors the folder's own structure and assets from "
"outside it end up in '_outside_project'.",
)
parser.add_argument(
"-n",
"--noop",
default=False,
action="store_true",
help="Don't copy files, just show what would be done.",
)
parser.add_argument(
"-e",
"--exclude",
nargs="*",
default="",
help="Space-separated list of glob patterns to exclude.",
)
parser.add_argument(
"-c",
"--compress",
default=False,
action="store_true",
help="Compress blend files while copying.",
)
parser.add_argument(
"-r",
"--relative-only",
default=False,
action="store_true",
help="Only pack assets referred to with a relative path.",
)
parser.add_argument(
"--keep-hierarchy",
default=False,
action="store_true",
help="Preserve the full filesystem directory hierarchy in the pack, "
"instead of packing relative to the scanned folder.",
)
parser.add_argument(
"--latest-only",
default=False,
action="store_true",
help="Of every 'name_vNNN.blend' version series in a directory, only "
"pack the highest version. Variants like 'name_vNNN_light.blend' form "
"their own series; files without a version are always packed.",
)
parser.add_argument(
"--follow-symlinks",
default=False,
action="store_true",
help="Descend into symlinked directories while scanning.",
)
parser.add_argument(
"--ignore-dir",
nargs="*",
default=list(folder_scan.DEFAULT_IGNORE_DIRS),
metavar="GLOB",
help="Globs of directory names to skip while scanning (default: %s)."
% " ".join(folder_scan.DEFAULT_IGNORE_DIRS),
)
parser.add_argument(
"--no-validate",
dest="validate",
default=True,
action="store_false",
help="Don't check that each '.blend' file really is a blend file. "
"Faster, but a non-blend file will abort the pack.",
)
def derive_common_project(bpaths: typing.List[pathlib.Path]) -> pathlib.Path:
"""Derive common project directory from multiple blend file paths.
@@ -202,25 +299,58 @@ def cli_pack_sequence(args):
log.critical("No target specified. Use -t/--target.")
sys.exit(3)
bpaths, ppath, tpath = paths_from_cli(args)
run_packer(args, *paths_from_cli(args))
with create_packer(args, bpaths, ppath, tpath) as packer:
packer.strategise()
try:
packer.execute()
except blender_asset_tracer.pack.transfer.FileTransferError as ex:
log.error(
"%d files couldn't be copied, starting with %s",
len(ex.files_remaining),
ex.files_remaining[0],
)
raise SystemExit(1)
def cli_pack_folder(args):
"""CLI entry point for pack-folder subcommand."""
fpath = args.folder
if not fpath.exists():
log.critical("Folder %s does not exist", fpath)
sys.exit(3)
if not fpath.is_dir():
log.critical("%s is not a directory", fpath)
sys.exit(3)
fpath = bpathlib.make_absolute(fpath)
log.info("Scanning %s for blend files", fpath)
scan = folder_scan.find_blend_files(
fpath,
follow_symlinks=args.follow_symlinks,
ignore_dirs=args.ignore_dir,
latest_only=args.latest_only,
validate=args.validate,
)
for dirpath, reason in scan.unreadable_dirs:
log.error("Could not scan %s: %s", dirpath, reason)
if scan.skipped:
log.warning("Skipped %d file(s) while scanning", len(scan.skipped))
if not scan.blendfiles:
log.critical("No blend files found in %s", fpath)
sys.exit(3)
log.warning("Found %d blend files in %s", len(scan.blendfiles), fpath)
# Synthesize args to reuse paths_from_cli and create_packer.
args.blendfile = None
args.sequence = scan.blendfiles
if args.project is None:
args.project = fpath
run_packer(args, *paths_from_cli(args))
def cli_pack(args):
if args.sequence:
log.warning("--sequence on 'pack' is deprecated. Use 'bat pack-sequence -t TARGET FILE...' instead.")
bpaths, ppath, tpath = paths_from_cli(args)
run_packer(args, *paths_from_cli(args))
def run_packer(
args, bpaths: typing.List[pathlib.Path], ppath: pathlib.Path, tpath: str
) -> None:
"""Create the packer for these paths, and run it."""
with create_packer(args, bpaths, ppath, tpath) as packer:
packer.strategise()
blender_asset_tracer/operators.py
+117 -2
View File
@@ -1,5 +1,4 @@
import os
import re
import sys
import subprocess
import tempfile
@@ -10,11 +9,12 @@ import bpy
from bpy.types import Operator, PropertyGroup
from bpy_extras.io_utils import ExportHelper
from blender_asset_tracer.pack import folder as folder_scan
from blender_asset_tracer.pack import zipped, progress
# Matches filenames like scene_v02.blend (single integer version only).
# Does NOT match: _v02.1.blend, _v2-1.blend, or files without _vNN suffix.
VERSION_RE = re.compile(r'_v(\d+)\.blend$', re.IGNORECASE)
VERSION_RE = folder_scan.VERSION_RE
class BlenderProgressCallback(progress.Callback):
@@ -489,12 +489,127 @@ class BAT_OT_sequence_pack(Operator, ExportHelper):
return {"FINISHED"}
class BAT_OT_folder_pack(Operator, ExportHelper):
"""Pack every blend file of a folder, with their dependencies, into one zip"""
bl_idname = "bat.folder_pack"
bl_label = "BAT - Pack Folder"
filename_ext = ".zip"
filter_glob: bpy.props.StringProperty(default="*.zip", options={'HIDDEN'})
folder: bpy.props.StringProperty(
name="Folder",
description="Folder to scan recursively for blend files. "
"It is also used as project root: assets from outside it are packed "
"into '_outside_project'",
subtype='DIR_PATH',
)
latest_only: bpy.props.BoolProperty(
name="Latest Versions Only",
description="Of every 'name_vNNN.blend' version series in a folder, "
"only pack the highest version. Variants like 'name_vNNN_light.blend' "
"form their own series; files without a version are always packed",
default=False,
)
keep_hierarchy: bpy.props.BoolProperty(
name="Keep Hierarchy",
description="Preserve the full filesystem directory hierarchy in the "
"zip, instead of packing relative to the scanned folder",
default=False,
)
follow_symlinks: bpy.props.BoolProperty(
name="Follow Symlinks",
description="Descend into symlinked directories while scanning",
default=False,
)
def invoke(self, context, event):
if not self.folder and bpy.data.is_saved:
self.folder = str(Path(bpy.data.filepath).parent)
if self.folder:
folder_name = Path(self.folder.rstrip("/\\")).name
self.filepath = folder_name + "_bat_pack.zip"
return super().invoke(context, event)
def draw(self, context):
layout = self.layout
layout.prop(self, "folder")
layout.prop(self, "latest_only")
layout.prop(self, "keep_hierarchy")
layout.prop(self, "follow_symlinks")
def execute(self, context):
from blender_asset_tracer.pack.zipped import ZipPacker
if not self.folder:
self.report({"ERROR"}, "No folder given")
return {"CANCELLED"}
project = Path(self.folder)
if not project.is_dir():
self.report({"ERROR"}, "%s is not a directory" % project)
return {"CANCELLED"}
self.report({"INFO"}, "Scanning %s for blend files..." % project)
scan = folder_scan.find_blend_files(
project,
follow_symlinks=self.follow_symlinks,
latest_only=self.latest_only,
)
if scan.unreadable_dirs:
self.report({"WARNING"},
"%d folder(s) could not be scanned" % len(scan.unreadable_dirs))
if not scan.blendfiles:
self.report({"ERROR"}, "No blend files found in %s" % project)
return {"CANCELLED"}
target = bpy.path.ensure_ext(self.filepath, ".zip")
self.report({"INFO"}, "Packing %d blend files..." % len(scan.blendfiles))
wm = context.window_manager
progress_cb = BlenderProgressCallback(wm)
try:
with ZipPacker(
scan.blendfiles,
project,
target,
keep_hierarchy=self.keep_hierarchy,
) as packer:
packer.progress_cb = progress_cb
packer.strategise()
packer.execute()
except Exception as ex:
import traceback
traceback.print_exc()
self.report({"ERROR"}, "Packing failed: %s" % str(ex))
return {"CANCELLED"}
finally:
try:
wm.progress_end()
except Exception:
pass
with zipfile.ZipFile(target) as inzip:
inzip.testzip()
self.report({"INFO"}, "Written to %s" % target)
open_folder(Path(target).parent)
return {"FINISHED"}
def menu_func(self, context):
layout = self.layout
layout.separator()
layout.operator(ExportBatPack.bl_idname)
filepath = layout.operator(BAT_OT_export_zip.bl_idname)
layout.operator(BAT_OT_sequence_pack.bl_idname)
layout.operator(BAT_OT_folder_pack.bl_idname)
try:
prefs = bpy.context.preferences.addons["blender_asset_tracer"].preferences
blender_asset_tracer/pack/__init__.py
+15 -1
View File
@@ -139,6 +139,7 @@ class Packer:
) # type: typing.DefaultDict[pathlib.Path, AssetAction]
self.missing_files = set() # type: typing.Set[pathlib.Path]
self._new_location_paths = set() # type: typing.Set[pathlib.Path]
self._top_level_paths = set() # type: typing.Set[pathlib.Path]
self._output_path = None # type: typing.Optional[pathlib.PurePath]
self._output_paths = [] # type: typing.List[pathlib.PurePath]
@@ -254,6 +255,7 @@ class Packer:
self._progress_cb.pack_start()
self._new_location_paths = set()
self._top_level_paths = set()
self._output_paths = []
for bf in self.blendfiles:
@@ -272,8 +274,14 @@ class Packer:
self._output_paths.append(bfile_pp)
act = self._actions[bfile_path]
act.path_action = PathAction.KEEP_PATH
# This file may already have been seen as a dependency of another
# blend file we're packing. In that case it needs to keep its
# FIND_NEW_LOCATION action, so that the file referring to it is
# rewritten; only its location in the pack is decided here.
if act.path_action != PathAction.FIND_NEW_LOCATION:
act.path_action = PathAction.KEEP_PATH
act.new_path = bfile_pp
self._top_level_paths.add(bfile_path)
self._check_aborted()
for usage in trace.deps(bf, self._progress_cb):
@@ -371,6 +379,12 @@ class Packer:
act = self._actions[path]
assert isinstance(act, AssetAction)
if path in self._top_level_paths:
# This is one of the blend files we're packing, so it already
# has its place in the pack. Files referring to it are still
# rewritten, they'll just point at that place.
continue
relpath = bpathlib.strip_root(path)
if self.keep_hierarchy:
act.new_path = pathlib.Path(self._target_path, relpath)
blender_asset_tracer/pack/folder.py
+227
View File
@@ -0,0 +1,227 @@
# ***** BEGIN GPL LICENSE BLOCK *****
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# ***** END GPL LICENCE BLOCK *****
"""Scan a directory tree for blend files, to feed a folder-wide BAT pack.
This module deliberately has no Blender dependencies, so that it can be used
both from the commandline (``bat pack-folder``) and from the add-on.
"""
import fnmatch
import logging
import os
import pathlib
import re
import typing
from blender_asset_tracer.blendfile import magic_compression
log = logging.getLogger(__name__)
BLENDFILE_SUFFIX = ".blend"
# Matches filenames like scene_v02.blend (single integer version only).
# Does NOT match: _v02.1.blend, _v2-1.blend, or files without _vNN suffix.
VERSION_RE = re.compile(r"_v(\d+)\.blend$", re.IGNORECASE)
# Splits a versioned filename into the parts that identify its version series,
# so that scene_v02.blend, scene_v02_light.blend and scene_v02_work.blend are
# each versioned independently. The greedy prefix makes the *last* '_vNNN' of
# the name the version.
VERSION_SERIES_RE = re.compile(
r"^(?P<prefix>.*)_v(?P<version>\d+)(?P<suffix>.*)\.blend$", re.IGNORECASE
)
# Directory name globs that are skipped by default while scanning.
DEFAULT_IGNORE_DIRS = (".*", "__pycache__")
class ScanResult(typing.NamedTuple):
"""The outcome of scanning a directory tree for blend files."""
blendfiles: typing.List[pathlib.Path]
"""Blend files to pack, in deterministic (sorted) order."""
skipped: typing.List[typing.Tuple[pathlib.Path, str]]
"""Files that were found but not packed, as (path, reason) tuples."""
unreadable_dirs: typing.List[typing.Tuple[pathlib.Path, str]]
"""Directories that could not be scanned, as (path, error) tuples."""
def find_blend_files(
folder: pathlib.Path,
*,
follow_symlinks: bool = False,
ignore_dirs: typing.Sequence[str] = DEFAULT_IGNORE_DIRS,
latest_only: bool = False,
validate: bool = True,
) -> ScanResult:
"""Recursively find the blend files in ``folder``.
Backup files (``.blend1``, ``.blend2``, …) are never returned, as only the
``.blend`` suffix is matched.
:param folder: Directory to scan.
:param follow_symlinks: Whether to descend into symlinked directories.
Already-visited directories are skipped, so symlink loops are safe.
:param ignore_dirs: Globs of directory names to skip entirely.
:param latest_only: Only keep the highest ``_vNNN`` file of each series.
:param validate: Check that each file actually starts with a blend file
magic (optionally compressed). Files that don't are reported as
skipped, instead of aborting the entire pack later on.
"""
blendfiles = [] # type: typing.List[pathlib.Path]
skipped = [] # type: typing.List[typing.Tuple[pathlib.Path, str]]
unreadable_dirs = [] # type: typing.List[typing.Tuple[pathlib.Path, str]]
def on_error(ex: OSError) -> None:
path = pathlib.Path(ex.filename) if ex.filename else folder
reason = ex.strerror or str(ex)
log.warning("Unable to scan %s: %s", path, reason)
unreadable_dirs.append((path, reason))
visited_dirs = set() # type: typing.Set[typing.Tuple[int, int]]
if follow_symlinks:
_first_visit(folder, visited_dirs)
for dirpath_str, dirnames, filenames in os.walk(
str(folder), onerror=on_error, followlinks=follow_symlinks
):
dirpath = pathlib.Path(dirpath_str)
# Prune ignored directories, and keep the walk order deterministic.
subdirs = sorted(
dirname for dirname in dirnames if not _matches_any(dirname, ignore_dirs)
)
if follow_symlinks:
subdirs = [
dirname
for dirname in subdirs
if _first_visit(dirpath / dirname, visited_dirs)
]
dirnames[:] = subdirs
for filename in sorted(filenames):
path = dirpath / filename
if path.suffix.lower() != BLENDFILE_SUFFIX:
continue
if validate:
reason = why_not_a_blendfile(path)
if reason is not None:
log.warning("Skipping %s: %s", path, reason)
skipped.append((path, reason))
continue
blendfiles.append(path)
if latest_only:
blendfiles, superseded = keep_latest_versions(blendfiles)
for path in superseded:
log.debug("Skipping %s: superseded by a newer version", path)
skipped.extend((path, "superseded by a newer version") for path in superseded)
return ScanResult(
blendfiles=blendfiles, skipped=skipped, unreadable_dirs=unreadable_dirs
)
def why_not_a_blendfile(path: pathlib.Path) -> typing.Optional[str]:
"""Return None when the file looks like a blend file, else why it doesn't."""
try:
with path.open("rb") as fileobj:
compression = magic_compression.find_compression_type(fileobj)
except OSError as ex:
return "unable to read: %s" % (ex.strerror or ex)
if compression == magic_compression.Compression.UNRECOGNISED:
return "not a blend file"
return None
def keep_latest_versions(
blendfiles: typing.Iterable[pathlib.Path],
) -> typing.Tuple[typing.List[pathlib.Path], typing.List[pathlib.Path]]:
"""Reduce each ``name_vNNN.blend`` series to its highest version.
Files whose name contains no ``_vNNN`` are always kept. Versions are
compared per directory and per name, so identically-named files in
different directories don't shadow each other, and variants such as
``shot_v003_light.blend`` form their own series.
:return: (files to keep, superseded files), both sorted.
"""
# Maps (directory, name without version) to the best (version, path) so far.
best = (
{}
) # type: typing.Dict[typing.Tuple[pathlib.Path, str, str], typing.Tuple[int, pathlib.Path]]
kept = [] # type: typing.List[pathlib.Path]
superseded = [] # type: typing.List[pathlib.Path]
for path in blendfiles:
match = VERSION_SERIES_RE.match(path.name)
if match is None:
kept.append(path)
continue
key = (
path.parent,
match.group("prefix").lower(),
match.group("suffix").lower(),
)
version = int(match.group("version"))
previous = best.get(key)
if previous is None:
best[key] = (version, path)
elif version > previous[0]:
best[key] = (version, path)
superseded.append(previous[1])
else:
superseded.append(path)
kept.extend(path for _, path in best.values())
kept.sort()
superseded.sort()
return kept, superseded
def _matches_any(name: str, patterns: typing.Sequence[str]) -> bool:
return any(fnmatch.fnmatch(name, pattern) for pattern in patterns)
def _first_visit(
dirpath: pathlib.Path, visited: typing.Set[typing.Tuple[int, int]]
) -> bool:
"""Return True when this directory wasn't walked before.
Used to keep ``follow_symlinks`` from looping forever.
"""
try:
stat = dirpath.stat()
except OSError as ex:
log.warning("Unable to inspect %s: %s", dirpath, ex.strerror or ex)
return False
key = (stat.st_dev, stat.st_ino)
if key in visited:
log.warning("Not descending into %s again (symlink loop?)", dirpath)
return False
visited.add(key)
return True