pack-sequence #1

Merged
joseph.henry merged 2 commits from pack-sequence into main 2026-03-24 17:04:57 +01:00
6 changed files with 441 additions and 81 deletions
Showing only changes of commit 05d97cdeda - Show all commits

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
.venv
*.pyc
*.blend[1-9]
__pycache__

View File

@ -58,18 +58,42 @@ if _HAS_BPY:
classes = (
preferences.BATPreferences,
operators.BAT_SequenceFileEntry,
operators.ExportBatPack,
operators.BAT_OT_export_zip,
operators.BAT_OT_scan_sequence,
operators.BAT_OT_sequence_pack,
)
def register():
for cls in classes:
bpy.utils.register_class(cls)
bpy.types.TOPBAR_MT_file_external_data.append(operators.menu_func)
bpy.types.WindowManager.bat_sequence_template = bpy.props.EnumProperty(
name="Studio Template",
description="Folder convention used to find published blend files",
items=operators.STUDIO_TEMPLATE_ITEMS,
)
bpy.types.WindowManager.bat_sequence_task = bpy.props.EnumProperty(
name="Task",
description="Which task folder to scan (for templates that require it)",
items=operators.TASK_CHOICE_ITEMS,
)
bpy.types.WindowManager.bat_sequence_dir = bpy.props.StringProperty(
name="Sequence Directory",
description="Root folder of the sequence last scanned for published blend files",
)
bpy.types.WindowManager.bat_sequence_files = bpy.props.CollectionProperty(
type=operators.BAT_SequenceFileEntry,
)
def unregister():
bpy.types.TOPBAR_MT_file_external_data.remove(operators.menu_func)
for cls in classes:
del bpy.types.WindowManager.bat_sequence_files
del bpy.types.WindowManager.bat_sequence_dir
del bpy.types.WindowManager.bat_sequence_task
del bpy.types.WindowManager.bat_sequence_template
for cls in reversed(classes):
bpy.utils.unregister_class(cls)
if __name__ == "__main__":

View File

@ -19,6 +19,7 @@
# (c) 2018, Blender Foundation - Sybren A. Stüvel
"""Create a BAT-pack for the given blend file."""
import logging
import os
import pathlib
import sys
import typing
@ -34,7 +35,8 @@ def add_parser(subparsers):
parser = subparsers.add_parser("pack", help=__doc__)
parser.set_defaults(func=cli_pack)
parser.add_argument("blendfile", type=pathlib.Path, help="The Blend file to pack.")
parser.add_argument("blendfile", nargs='?', type=pathlib.Path, default=None,
help="The Blend file to pack (omit when using --sequence).")
parser.add_argument(
"target",
type=str,
@ -97,12 +99,20 @@ def add_parser(subparsers):
"path structure under the target directory. Paths in blend files are "
"rewritten to relative paths within this structure.",
)
parser.add_argument(
"--sequence",
nargs="+",
type=pathlib.Path,
metavar="BLENDFILE",
help="Pack multiple blend files together, deduplicating shared dependencies. "
"All listed blend files and their dependencies are packed into the target.",
)
def cli_pack(args):
bpath, ppath, tpath = paths_from_cli(args)
bpaths, ppath, tpath = paths_from_cli(args)
with create_packer(args, bpath, ppath, tpath) as packer:
with create_packer(args, bpaths, ppath, tpath) as packer:
packer.strategise()
try:
packer.execute()
@ -116,9 +126,12 @@ def cli_pack(args):
def create_packer(
args, bpath: pathlib.Path, ppath: pathlib.Path, target: str
args, bpaths: typing.List[pathlib.Path], ppath: pathlib.Path, target: str
) -> pack.Packer:
if target.startswith("s3:/"):
if len(bpaths) > 1:
raise ValueError("S3 uploader does not support --sequence")
if args.noop:
raise ValueError("S3 uploader does not support no-op.")
@ -131,13 +144,16 @@ def create_packer(
if args.keep_hierarchy:
raise ValueError("S3 uploader does not support the --keep-hierarchy option")
packer = create_s3packer(bpath, ppath, pathlib.PurePosixPath(target))
packer = create_s3packer(bpaths[0], ppath, pathlib.PurePosixPath(target))
elif (
target.startswith("shaman+http:/")
or target.startswith("shaman+https:/")
or target.startswith("shaman:/")
):
if len(bpaths) > 1:
raise ValueError("Shaman uploader does not support --sequence")
if args.noop:
raise ValueError("Shaman uploader does not support no-op.")
@ -154,7 +170,7 @@ def create_packer(
"Shaman uploader does not support the --keep-hierarchy option"
)
packer = create_shamanpacker(bpath, ppath, target)
packer = create_shamanpacker(bpaths[0], ppath, target)
elif target.lower().endswith(".zip"):
from blender_asset_tracer.pack import zipped
@ -163,12 +179,12 @@ def create_packer(
raise ValueError("ZIP packer does not support on-the-fly compression")
packer = zipped.ZipPacker(
bpath, ppath, target, noop=args.noop, relative_only=args.relative_only,
bpaths, ppath, target, noop=args.noop, relative_only=args.relative_only,
keep_hierarchy=args.keep_hierarchy,
)
else:
packer = pack.Packer(
bpath,
bpaths,
ppath,
target,
noop=args.noop,
@ -222,24 +238,42 @@ def create_shamanpacker(
)
def paths_from_cli(args) -> typing.Tuple[pathlib.Path, pathlib.Path, str]:
"""Return paths to blendfile, project, and pack target.
def paths_from_cli(args) -> typing.Tuple[typing.List[pathlib.Path], pathlib.Path, str]:
"""Return paths to blendfile(s), project, and pack target.
Calls sys.exit() if anything is wrong.
"""
bpath = args.blendfile
if not bpath.exists():
log.critical("File %s does not exist", bpath)
# Collect blend files from positional and --sequence arguments.
bpaths = [] # type: typing.List[pathlib.Path]
if args.blendfile is not None:
bpaths.append(args.blendfile)
if args.sequence:
bpaths.extend(args.sequence)
if not bpaths:
log.critical("No blend file specified. Provide a positional blendfile or use --sequence.")
sys.exit(3)
if bpath.is_dir():
log.critical("%s is a directory, should be a blend file")
sys.exit(3)
bpath = bpathlib.make_absolute(bpath)
# Deduplicate preserving order, in case the same file appears both as
# positional arg and in --sequence.
bpaths = list(dict.fromkeys(bpaths))
# Validate each blend file and make absolute.
for i, bpath in enumerate(bpaths):
if not bpath.exists():
log.critical("File %s does not exist", bpath)
sys.exit(3)
if bpath.is_dir():
log.critical("%s is a directory, should be a blend file", bpath)
sys.exit(3)
bpaths[i] = bpathlib.make_absolute(bpath)
tpath = args.target
if args.project is None:
ppath = bpathlib.make_absolute(bpath).parent
if len(bpaths) == 1:
ppath = bpaths[0].parent
else:
ppath = pathlib.Path(os.path.commonpath([p.parent for p in bpaths]))
log.warning("No project path given, using %s", ppath)
else:
ppath = bpathlib.make_absolute(args.project)
@ -256,18 +290,20 @@ def paths_from_cli(args) -> typing.Tuple[pathlib.Path, pathlib.Path, str]:
)
ppath = ppath.parent
try:
bpath.relative_to(ppath)
except ValueError:
log.critical(
"Project directory %s does not contain blend file %s",
args.project,
bpath.absolute(),
)
sys.exit(5)
for bpath in bpaths:
try:
bpath.relative_to(ppath)
except ValueError:
log.critical(
"Project directory %s does not contain blend file %s",
ppath,
bpath,
)
sys.exit(5)
log.info("Blend file to pack: %s", bpath)
for bpath in bpaths:
log.info("Blend file to pack: %s", bpath)
log.info("Project path: %s", ppath)
log.info("Pack will be created in: %s", tpath)
return bpath, ppath, tpath
return bpaths, ppath, tpath

View File

@ -1,4 +1,5 @@
import os
import re
import sys
import subprocess
import tempfile
@ -6,10 +7,64 @@ import zipfile
from pathlib import Path
import bpy
from bpy.types import Operator
from bpy.types import Operator, PropertyGroup
from bpy_extras.io_utils import ExportHelper
from blender_asset_tracer.pack import zipped
from blender_asset_tracer.pack import zipped, progress
VERSION_RE = re.compile(r'_v(\d+)\.blend$', re.IGNORECASE)
class BlenderProgressCallback(progress.Callback):
"""Progress callback that updates Blender's wm.progress cursor indicator."""
def __init__(self, wm):
self._wm = wm
self._assets_traced = 0
self._total_bytes = 0
self._transferred_bytes = 0
self._phase = "init"
def pack_start(self):
self._phase = "trace"
self._assets_traced = 0
self._wm.progress_begin(0, 1000)
self._wm.progress_update(0)
print("[BAT] Tracing dependencies...")
def trace_asset(self, filename):
self._assets_traced += 1
# During tracing we don't know the total, so just increment slowly
# Cap at 400 (40% of 1000) to leave room for the transfer phase
val = min(self._assets_traced, 400)
self._wm.progress_update(val)
if self._assets_traced % 20 == 0:
print("[BAT] Traced %d assets..." % self._assets_traced)
def transfer_file(self, src, dst):
if self._phase != "transfer":
self._phase = "transfer"
print("[BAT] Transferring files...")
def transfer_progress(self, total_bytes, transferred_bytes):
self._total_bytes = total_bytes
self._transferred_bytes = transferred_bytes
if total_bytes > 0:
# Transfer phase: map to 400-1000 range
pct = transferred_bytes / total_bytes
val = 400 + int(pct * 600)
self._wm.progress_update(val)
def pack_done(self, output_blendfile, missing_files):
self._wm.progress_update(1000)
self._wm.progress_end()
print("[BAT] Pack complete! (%d assets traced)" % self._assets_traced)
if missing_files:
print("[BAT] Warning: %d missing files" % len(missing_files))
def pack_aborted(self, reason):
self._wm.progress_end()
print("[BAT] Pack aborted: %s" % reason)
class ExportBatPack(Operator, ExportHelper):
@ -25,17 +80,16 @@ class ExportBatPack(Operator, ExportHelper):
outfname = bpy.path.ensure_ext(self.filepath, ".zip")
self.report({"INFO"}, "Executing ZipPacker ...")
progress_cb = BlenderProgressCallback(context.window_manager)
with zipped.ZipPacker(
Path(bpy.data.filepath),
Path(bpy.data.filepath).parent,
str(self.filepath),
) as packer:
print("[BAT] Strategising (tracing dependencies)...")
packer.progress_cb = progress_cb
packer.strategise()
print(f"[BAT] Found {len(packer._actions)} assets to process")
print("[BAT] Executing (rewriting paths and copying files)...")
packer.execute()
print("[BAT] Packing complete!")
self.report({"INFO"}, "Packing successful!")
with zipfile.ZipFile(str(self.filepath)) as inzip:
@ -126,13 +180,12 @@ class BAT_OT_export_zip(Operator, ExportHelper):
self.report({"INFO"}, "Packing with hierarchy...")
progress_cb = BlenderProgressCallback(context.window_manager)
with packer_cls(bfile, project, target, keep_hierarchy=True) as packer:
print("[BAT] Strategising (tracing dependencies)...")
packer.progress_cb = progress_cb
packer.strategise()
print(f"[BAT] Found {len(packer._actions)} assets to process")
print("[BAT] Executing (rewriting paths and copying files)...")
packer.execute()
print("[BAT] Packing complete!")
if self.use_zip:
with zipfile.ZipFile(target) as inzip:
@ -149,11 +202,236 @@ class BAT_OT_export_zip(Operator, ExportHelper):
return {"FINISHED"}
class BAT_SequenceFileEntry(PropertyGroup):
filepath: bpy.props.StringProperty(name="File Path")
enabled: bpy.props.BoolProperty(name="Enabled", default=True)
shot_name: bpy.props.StringProperty(name="Shot")
# Studio templates define where to find published blend files.
# Each template is a list of path segments (case-insensitive) to walk from each
# shot directory to the folder containing versioned .blend files.
STUDIO_TEMPLATES = {
'ADM': {
'label': "Autour de Minuit",
'description': "Scans <shot>/<task>/ for the latest versioned .blend (e.g. lighting3d, anim3d)",
'path_segments': ["{task}"],
'task_choices': [
('lighting3d', "Lighting 3D", ""),
('anim3d', "Animation 3D", ""),
('layout3d', "Layout 3D", ""),
('compositing', "Compositing", ""),
],
},
'LCPROD': {
'label': "La Cabane Productions - Lamb Stew",
'description': "Scans <shot>/03_ANIMATION/Publish/ for the latest versioned .blend",
'path_segments': ["03_ANIMATION", "Publish"],
},
}
STUDIO_TEMPLATE_ITEMS = [
(key, tpl['label'], tpl['description'])
for key, tpl in STUDIO_TEMPLATES.items()
]
# Collect all task choices across templates that have them.
TASK_CHOICE_ITEMS = []
_seen = set()
for _tpl in STUDIO_TEMPLATES.values():
for item in _tpl.get('task_choices', []):
if item[0] not in _seen:
TASK_CHOICE_ITEMS.append(item)
_seen.add(item[0])
def _find_subdir_ci(parent, name):
"""Find a child directory matching `name` case-insensitively (prefix match)."""
name_upper = name.upper()
for child in parent.iterdir():
if child.is_dir() and child.name.upper().startswith(name_upper):
return child
return None
def find_latest_publishes(root_dir, template_key, task=''):
"""Scan a sequence folder for the latest .blend in each shot using the given template."""
template = STUDIO_TEMPLATES.get(template_key)
if not template:
return []
results = []
root = Path(root_dir)
if not root.is_dir():
return results
path_segments = [seg.replace("{task}", task) if "{task}" in seg else seg
for seg in template['path_segments']]
for shot_dir in sorted(root.iterdir()):
if not shot_dir.is_dir():
continue
# Walk the template path segments from the shot directory
current = shot_dir
for segment in path_segments:
current = _find_subdir_ci(current, segment)
if current is None:
break
if current is None:
continue
# Find the .blend with the highest _vNNN version number
best_version = -1
best_file = None
for f in current.iterdir():
if f.suffix.lower() != '.blend':
continue
m = VERSION_RE.search(f.name)
if m:
ver = int(m.group(1))
if ver > best_version:
best_version = ver
best_file = f
if best_file:
results.append((shot_dir.name, best_file.absolute()))
return results
class BAT_OT_scan_sequence(Operator):
"""Scan the current file browser directory for latest published blend files"""
bl_idname = "bat.scan_sequence"
bl_label = "Scan Current Folder"
def execute(self, context):
wm = context.window_manager
# Read the directory the user is currently browsing in the file selector
try:
seq_dir = context.space_data.params.directory.decode('utf-8')
except (AttributeError, UnicodeDecodeError):
seq_dir = None
if not seq_dir:
self.report({"ERROR"}, "Could not read current file browser directory")
return {"CANCELLED"}
wm.bat_sequence_dir = seq_dir
wm.bat_sequence_files.clear()
publishes = find_latest_publishes(seq_dir, wm.bat_sequence_template, wm.bat_sequence_task)
if not publishes:
self.report({"WARNING"}, "No published blend files found in %s" % seq_dir)
return {"CANCELLED"}
for shot_name, filepath in publishes:
entry = wm.bat_sequence_files.add()
entry.shot_name = shot_name
entry.filepath = str(filepath)
entry.enabled = True
# Update the file browser filename based on the scanned folder name
folder_name = Path(seq_dir.rstrip("/\\")).name
try:
context.space_data.params.filename = folder_name + "_bat_pack.zip"
except (AttributeError, TypeError):
pass
self.report({"INFO"}, "Found %d published blend files" % len(publishes))
return {"FINISHED"}
class BAT_OT_sequence_pack(Operator, ExportHelper):
"""Pack a sequence of shots: scan a folder for latest published blend files and pack them into a ZIP"""
bl_idname = "bat.sequence_pack"
bl_label = "BAT - Pack Sequence"
filename_ext = ".zip"
filter_glob: bpy.props.StringProperty(default="*.zip", options={'HIDDEN'})
def invoke(self, context, event):
wm = context.window_manager
# Default sequence_dir to blend file's parent if not already set
if not wm.bat_sequence_dir and bpy.data.is_saved:
wm.bat_sequence_dir = str(Path(bpy.data.filepath).parent)
# Pre-fill filename from the last scanned directory
if wm.bat_sequence_dir:
folder_name = Path(wm.bat_sequence_dir.rstrip("/\\")).name
self.filepath = folder_name + "_bat_pack.zip"
return super().invoke(context, event)
def draw(self, context):
layout = self.layout
wm = context.window_manager
layout.label(text="Studio Template:")
layout.prop(wm, "bat_sequence_template", text="")
# Show task selector if the current template has task choices
template = STUDIO_TEMPLATES.get(wm.bat_sequence_template)
if template and template.get('task_choices'):
layout.prop(wm, "bat_sequence_task", text="Task")
layout.separator()
layout.label(text="Navigate to the sequence folder, then:")
layout.operator(BAT_OT_scan_sequence.bl_idname, icon='FILE_REFRESH')
if wm.bat_sequence_dir:
layout.label(text="Scanned: %s" % Path(wm.bat_sequence_dir).name)
if len(wm.bat_sequence_files) > 0:
layout.separator()
layout.label(text="Published blend files:")
box = layout.box()
for entry in wm.bat_sequence_files:
row = box.row()
row.prop(entry, "enabled", text="")
row.label(text=entry.shot_name)
row.label(text=Path(entry.filepath).name)
def execute(self, context):
from blender_asset_tracer.pack.zipped import ZipPacker
wm = context.window_manager
bpaths = [Path(entry.filepath) for entry in wm.bat_sequence_files if entry.enabled]
if not bpaths:
self.report({"ERROR"}, "No blend files selected. Scan a folder first.")
return {"CANCELLED"}
target = bpy.path.ensure_ext(self.filepath, ".zip")
project = Path(os.path.commonpath([p.parent for p in bpaths]))
self.report({"INFO"}, "Packing %d blend files..." % len(bpaths))
progress_cb = BlenderProgressCallback(wm)
try:
with ZipPacker(bpaths, project, target, keep_hierarchy=True) as packer:
packer.progress_cb = progress_cb
packer.strategise()
packer.execute()
except Exception as ex:
self.report({"ERROR"}, "Packing failed: %s" % str(ex))
return {"CANCELLED"}
with zipfile.ZipFile(target) as inzip:
inzip.testzip()
self.report({"INFO"}, "Written to %s" % target)
open_folder(Path(target).parent)
return {"FINISHED"}
def menu_func(self, context):
layout = self.layout
layout.separator()
layout.operator(ExportBatPack.bl_idname)
filepath = layout.operator(BAT_OT_export_zip.bl_idname)
layout.operator(BAT_OT_sequence_pack.bl_idname)
try:
prefs = bpy.context.preferences.addons["blender_asset_tracer"].preferences

View File

@ -96,7 +96,7 @@ class Packer:
def __init__(
self,
bfile: pathlib.Path,
bfile: typing.Union[pathlib.Path, typing.List[pathlib.Path]],
project: pathlib.Path,
target: str,
*,
@ -105,7 +105,11 @@ class Packer:
relative_only=False,
keep_hierarchy=False,
) -> None:
self.blendfile = bfile
if isinstance(bfile, list):
self.blendfiles = bfile
else:
self.blendfiles = [bfile]
self.blendfile = self.blendfiles[0] # backward compat
self.project = project
self.target = target
self._target_path = self._make_target_path(target)
@ -136,6 +140,7 @@ class Packer:
self.missing_files = set() # type: typing.Set[pathlib.Path]
self._new_location_paths = set() # type: typing.Set[pathlib.Path]
self._output_path = None # type: typing.Optional[pathlib.PurePath]
self._output_paths = [] # type: typing.List[pathlib.PurePath]
# Filled by execute()
self._file_transferer = None # type: typing.Optional[transfer.FileTransferer]
@ -172,6 +177,11 @@ class Packer:
assert self._output_path is not None
return self._output_path
@property
def output_paths(self) -> typing.List[pathlib.PurePath]:
"""The paths of all packed blend files in the target directory."""
return list(self._output_paths)
@property
def progress_cb(self) -> progress.Callback:
return self._progress_cb
@ -235,46 +245,47 @@ class Packer:
in the execute() function.
"""
# The blendfile that we pack is generally not its own dependency, so
# we have to explicitly add it to the _packed_paths.
bfile_path = bpathlib.make_absolute(self.blendfile)
# Both paths have to be resolved first, because this also translates
# network shares mapped to Windows drive letters back to their UNC
# notation. Only resolving one but not the other (which can happen
# with the abosolute() call above) can cause errors.
if self.keep_hierarchy:
bfile_pp = self._target_path / bpathlib.strip_root(bfile_path)
else:
bfile_pp = self._target_path / bfile_path.relative_to(
bpathlib.make_absolute(self.project)
)
self._output_path = bfile_pp
self._progress_cb.pack_start()
act = self._actions[bfile_path]
act.path_action = PathAction.KEEP_PATH
act.new_path = bfile_pp
self._check_aborted()
self._new_location_paths = set()
for usage in trace.deps(self.blendfile, self._progress_cb):
self._check_aborted()
asset_path = usage.abspath
if any(asset_path.match(glob) for glob in self._exclude_globs):
log.info("Excluding file: %s", asset_path)
continue
self._output_paths = []
if self.relative_only and not usage.asset_path.startswith(b"//"):
log.info("Skipping absolute path: %s", usage.asset_path)
continue
for bf in self.blendfiles:
bfile_path = bpathlib.make_absolute(bf)
if usage.is_sequence:
self._visit_sequence(asset_path, usage)
# Both paths have to be resolved first, because this also translates
# network shares mapped to Windows drive letters back to their UNC
# notation. Only resolving one but not the other (which can happen
# with the abosolute() call above) can cause errors.
if self.keep_hierarchy:
bfile_pp = self._target_path / bpathlib.strip_root(bfile_path)
else:
self._visit_asset(asset_path, usage)
bfile_pp = self._target_path / bfile_path.relative_to(
bpathlib.make_absolute(self.project)
)
self._output_paths.append(bfile_pp)
act = self._actions[bfile_path]
act.path_action = PathAction.KEEP_PATH
act.new_path = bfile_pp
self._check_aborted()
for usage in trace.deps(bf, self._progress_cb):
self._check_aborted()
asset_path = usage.abspath
if any(asset_path.match(glob) for glob in self._exclude_globs):
log.info("Excluding file: %s", asset_path)
continue
if self.relative_only and not usage.asset_path.startswith(b"//"):
log.info("Skipping absolute path: %s", usage.asset_path)
continue
if usage.is_sequence:
self._visit_sequence(asset_path, usage)
else:
self._visit_asset(asset_path, usage)
self._output_path = self._output_paths[0] # backward compat
self._find_new_paths()
self._group_rewrites()
@ -644,11 +655,19 @@ class Packer:
log.debug("Writing info to %s", infopath)
with infopath.open("wt", encoding="utf8") as infofile:
print("This is a Blender Asset Tracer pack.", file=infofile)
print("Start by opening the following blend file:", file=infofile)
print(
" %s" % self._output_path.relative_to(self._target_path).as_posix(),
file=infofile,
)
if len(self._output_paths) > 1:
print("This pack contains the following blend files:", file=infofile)
for op in self._output_paths:
print(
" %s" % op.relative_to(self._target_path).as_posix(),
file=infofile,
)
else:
print("Start by opening the following blend file:", file=infofile)
print(
" %s" % self._output_path.relative_to(self._target_path).as_posix(),
file=infofile,
)
self._file_transferer.queue_move(infopath, self._target_path / infoname)

View File

@ -58,6 +58,7 @@ class ZipTransferrer(transfer.FileTransferer):
zippath = self.zippath.absolute()
log.info("Writing ZIP file to %s", zippath)
with zipfile.ZipFile(str(zippath), "w") as outzip:
for src, dst, act in self.iter_queue():
assert src.is_absolute(), "expecting only absolute paths, not %r" % src
@ -89,3 +90,4 @@ class ZipTransferrer(transfer.FileTransferer):
# be reported there.
self.queue.put((src, dst, act))
return
log.info("Finished writing ZIP file: %s", zippath)