551 lines
17 KiB
Python
Executable File
551 lines
17 KiB
Python
Executable File
# ***** BEGIN GPL LICENSE BLOCK *****
|
|
#
|
|
# This program is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU General Public License
|
|
# as published by the Free Software Foundation; either version 2
|
|
# of the License, or (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software Foundation,
|
|
# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
|
#
|
|
# ***** END GPL LICENCE BLOCK *****
|
|
#
|
|
# (c) 2018, Blender Foundation - Sybren A. Stüvel
|
|
"""Create a BAT-pack for the given blend file."""
|
|
import logging
|
|
import os
|
|
import pathlib
|
|
import sys
|
|
import typing
|
|
|
|
import blender_asset_tracer.pack.transfer
|
|
from blender_asset_tracer import pack, bpathlib
|
|
from blender_asset_tracer.pack import folder as folder_scan
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def add_parser(subparsers):
|
|
"""Add argparser for this subcommand."""
|
|
|
|
parser = subparsers.add_parser("pack", help=__doc__)
|
|
parser.set_defaults(func=cli_pack)
|
|
parser.add_argument("blendfile", nargs='?', type=pathlib.Path, default=None,
|
|
help="The Blend file to pack (omit when using --sequence).")
|
|
parser.add_argument(
|
|
"target",
|
|
type=str,
|
|
help="The target can be a directory, a ZIP file (does not have to exist "
|
|
"yet, just use 'something.zip' as target), "
|
|
"or a URL of S3 storage (s3://endpoint/path) "
|
|
"or Shaman storage (shaman://endpoint/#checkoutID).",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"-p",
|
|
"--project",
|
|
type=pathlib.Path,
|
|
help="Root directory of your project. Paths to below this directory are "
|
|
"kept in the BAT Pack as well, whereas references to assets from "
|
|
"outside this directory will have to be rewitten. The blend file MUST "
|
|
"be inside the project directory. If this option is ommitted, the "
|
|
"directory containing the blend file is taken as the project "
|
|
"directoy.",
|
|
)
|
|
parser.add_argument(
|
|
"-n",
|
|
"--noop",
|
|
default=False,
|
|
action="store_true",
|
|
help="Don't copy files, just show what would be done.",
|
|
)
|
|
parser.add_argument(
|
|
"-e",
|
|
"--exclude",
|
|
nargs="*",
|
|
default="",
|
|
help="Space-separated list of glob patterns (like '*.abc *.vbo') to "
|
|
"exclude.",
|
|
)
|
|
parser.add_argument(
|
|
"-c",
|
|
"--compress",
|
|
default=False,
|
|
action="store_true",
|
|
help="Compress blend files while copying. This option is only valid when "
|
|
"packing into a directory (contrary to ZIP file or S3 upload). "
|
|
"Note that files will NOT be compressed when the destination file "
|
|
"already exists and has the same size as the original file.",
|
|
)
|
|
parser.add_argument(
|
|
"-r",
|
|
"--relative-only",
|
|
default=False,
|
|
action="store_true",
|
|
help="Only pack assets that are referred to with a relative path (e.g. "
|
|
"starting with `//`.",
|
|
)
|
|
parser.add_argument(
|
|
"--keep-hierarchy",
|
|
default=False,
|
|
action="store_true",
|
|
help="Preserve the full filesystem directory hierarchy in the pack. "
|
|
"All files (including the blend file) are placed at their absolute "
|
|
"path structure under the target directory. Paths in blend files are "
|
|
"rewritten to relative paths within this structure.",
|
|
)
|
|
parser.add_argument(
|
|
"--sequence",
|
|
nargs="+",
|
|
type=pathlib.Path,
|
|
metavar="BLENDFILE",
|
|
help="(Deprecated: use 'pack-sequence' subcommand instead.) "
|
|
"Pack multiple blend files together, deduplicating shared dependencies.",
|
|
)
|
|
|
|
|
|
def add_sequence_parser(subparsers):
|
|
"""Add argparser for the pack-sequence subcommand."""
|
|
|
|
parser = subparsers.add_parser(
|
|
"pack-sequence",
|
|
help="Pack multiple blend files together, deduplicating shared dependencies.",
|
|
)
|
|
parser.set_defaults(func=cli_pack_sequence)
|
|
parser.add_argument(
|
|
"blendfiles",
|
|
nargs="+",
|
|
type=pathlib.Path,
|
|
help="Blend files to pack.",
|
|
)
|
|
parser.add_argument(
|
|
"-t",
|
|
"--target",
|
|
type=str,
|
|
required=True,
|
|
help="Target directory or ZIP file.",
|
|
)
|
|
parser.add_argument(
|
|
"-p",
|
|
"--project",
|
|
type=pathlib.Path,
|
|
help="Root directory of your project.",
|
|
)
|
|
parser.add_argument(
|
|
"-n",
|
|
"--noop",
|
|
default=False,
|
|
action="store_true",
|
|
help="Don't copy files, just show what would be done.",
|
|
)
|
|
parser.add_argument(
|
|
"-e",
|
|
"--exclude",
|
|
nargs="*",
|
|
default="",
|
|
help="Space-separated list of glob patterns to exclude.",
|
|
)
|
|
parser.add_argument(
|
|
"-c",
|
|
"--compress",
|
|
default=False,
|
|
action="store_true",
|
|
help="Compress blend files while copying.",
|
|
)
|
|
parser.add_argument(
|
|
"-r",
|
|
"--relative-only",
|
|
default=False,
|
|
action="store_true",
|
|
help="Only pack assets referred to with a relative path.",
|
|
)
|
|
|
|
|
|
def add_folder_parser(subparsers):
|
|
"""Add argparser for the pack-folder subcommand."""
|
|
|
|
parser = subparsers.add_parser(
|
|
"pack-folder",
|
|
help="Pack every blend file in a directory tree, together with their "
|
|
"dependencies, deduplicating shared assets.",
|
|
)
|
|
parser.set_defaults(func=cli_pack_folder)
|
|
parser.add_argument(
|
|
"folder",
|
|
type=pathlib.Path,
|
|
help="Directory to scan recursively for blend files.",
|
|
)
|
|
parser.add_argument(
|
|
"-t",
|
|
"--target",
|
|
type=str,
|
|
required=True,
|
|
help="Target directory or ZIP file.",
|
|
)
|
|
parser.add_argument(
|
|
"-p",
|
|
"--project",
|
|
type=pathlib.Path,
|
|
help="Root directory of your project. Defaults to the scanned folder, "
|
|
"so that the pack mirrors the folder's own structure and assets from "
|
|
"outside it end up in '_outside_project'.",
|
|
)
|
|
parser.add_argument(
|
|
"-n",
|
|
"--noop",
|
|
default=False,
|
|
action="store_true",
|
|
help="Don't copy files, just show what would be done.",
|
|
)
|
|
parser.add_argument(
|
|
"-e",
|
|
"--exclude",
|
|
nargs="*",
|
|
default="",
|
|
help="Space-separated list of glob patterns to exclude.",
|
|
)
|
|
parser.add_argument(
|
|
"-c",
|
|
"--compress",
|
|
default=False,
|
|
action="store_true",
|
|
help="Compress blend files while copying.",
|
|
)
|
|
parser.add_argument(
|
|
"-r",
|
|
"--relative-only",
|
|
default=False,
|
|
action="store_true",
|
|
help="Only pack assets referred to with a relative path.",
|
|
)
|
|
parser.add_argument(
|
|
"--keep-hierarchy",
|
|
default=False,
|
|
action="store_true",
|
|
help="Preserve the full filesystem directory hierarchy in the pack, "
|
|
"instead of packing relative to the scanned folder.",
|
|
)
|
|
parser.add_argument(
|
|
"--latest-only",
|
|
default=False,
|
|
action="store_true",
|
|
help="Of every 'name_vNNN.blend' version series in a directory, only "
|
|
"pack the highest version. Variants like 'name_vNNN_light.blend' form "
|
|
"their own series; files without a version are always packed.",
|
|
)
|
|
parser.add_argument(
|
|
"--follow-symlinks",
|
|
default=False,
|
|
action="store_true",
|
|
help="Descend into symlinked directories while scanning.",
|
|
)
|
|
parser.add_argument(
|
|
"--ignore-dir",
|
|
nargs="*",
|
|
default=list(folder_scan.DEFAULT_IGNORE_DIRS),
|
|
metavar="GLOB",
|
|
help="Globs of directory names to skip while scanning (default: %s)."
|
|
% " ".join(folder_scan.DEFAULT_IGNORE_DIRS),
|
|
)
|
|
parser.add_argument(
|
|
"--no-validate",
|
|
dest="validate",
|
|
default=True,
|
|
action="store_false",
|
|
help="Don't check that each '.blend' file really is a blend file. "
|
|
"Faster, but a non-blend file will abort the pack.",
|
|
)
|
|
|
|
|
|
def derive_common_project(bpaths: typing.List[pathlib.Path]) -> pathlib.Path:
|
|
"""Derive common project directory from multiple blend file paths.
|
|
|
|
Raises ValueError if paths span multiple drives or if common root
|
|
is the filesystem root.
|
|
"""
|
|
if len(bpaths) == 1:
|
|
return bpaths[0].parent
|
|
|
|
try:
|
|
ppath = pathlib.Path(os.path.commonpath([p.parent for p in bpaths]))
|
|
except ValueError:
|
|
raise ValueError(
|
|
"Blend files span multiple drives or have no common path. "
|
|
"Specify the project directory explicitly with -p/--project."
|
|
)
|
|
|
|
if ppath == pathlib.Path(ppath.anchor):
|
|
raise ValueError(
|
|
"Computed project path is the filesystem root (%s). "
|
|
"Specify the project directory explicitly." % ppath
|
|
)
|
|
|
|
return ppath
|
|
|
|
|
|
def cli_pack_sequence(args):
|
|
"""CLI entry point for pack-sequence subcommand."""
|
|
# Synthesize args to reuse paths_from_cli and create_packer
|
|
args.blendfile = None
|
|
args.sequence = args.blendfiles
|
|
args.keep_hierarchy = True # always for sequence packing
|
|
if not hasattr(args, 'target') or args.target is None:
|
|
log.critical("No target specified. Use -t/--target.")
|
|
sys.exit(3)
|
|
|
|
run_packer(args, *paths_from_cli(args))
|
|
|
|
|
|
def cli_pack_folder(args):
|
|
"""CLI entry point for pack-folder subcommand."""
|
|
fpath = args.folder
|
|
if not fpath.exists():
|
|
log.critical("Folder %s does not exist", fpath)
|
|
sys.exit(3)
|
|
if not fpath.is_dir():
|
|
log.critical("%s is not a directory", fpath)
|
|
sys.exit(3)
|
|
fpath = bpathlib.make_absolute(fpath)
|
|
|
|
log.info("Scanning %s for blend files", fpath)
|
|
scan = folder_scan.find_blend_files(
|
|
fpath,
|
|
follow_symlinks=args.follow_symlinks,
|
|
ignore_dirs=args.ignore_dir,
|
|
latest_only=args.latest_only,
|
|
validate=args.validate,
|
|
)
|
|
|
|
for dirpath, reason in scan.unreadable_dirs:
|
|
log.error("Could not scan %s: %s", dirpath, reason)
|
|
if scan.skipped:
|
|
log.warning("Skipped %d file(s) while scanning", len(scan.skipped))
|
|
if not scan.blendfiles:
|
|
log.critical("No blend files found in %s", fpath)
|
|
sys.exit(3)
|
|
log.warning("Found %d blend files in %s", len(scan.blendfiles), fpath)
|
|
|
|
# Synthesize args to reuse paths_from_cli and create_packer.
|
|
args.blendfile = None
|
|
args.sequence = scan.blendfiles
|
|
if args.project is None:
|
|
args.project = fpath
|
|
|
|
run_packer(args, *paths_from_cli(args))
|
|
|
|
|
|
def cli_pack(args):
|
|
if args.sequence:
|
|
log.warning("--sequence on 'pack' is deprecated. Use 'bat pack-sequence -t TARGET FILE...' instead.")
|
|
|
|
run_packer(args, *paths_from_cli(args))
|
|
|
|
|
|
def run_packer(
|
|
args, bpaths: typing.List[pathlib.Path], ppath: pathlib.Path, tpath: str
|
|
) -> None:
|
|
"""Create the packer for these paths, and run it."""
|
|
|
|
with create_packer(args, bpaths, ppath, tpath) as packer:
|
|
packer.strategise()
|
|
try:
|
|
packer.execute()
|
|
except blender_asset_tracer.pack.transfer.FileTransferError as ex:
|
|
log.error(
|
|
"%d files couldn't be copied, starting with %s",
|
|
len(ex.files_remaining),
|
|
ex.files_remaining[0],
|
|
)
|
|
raise SystemExit(1)
|
|
|
|
|
|
def create_packer(
|
|
args, bpaths: typing.List[pathlib.Path], ppath: pathlib.Path, target: str
|
|
) -> pack.Packer:
|
|
if target.startswith("s3:/"):
|
|
if len(bpaths) > 1:
|
|
raise ValueError("S3 uploader does not support --sequence")
|
|
|
|
if args.noop:
|
|
raise ValueError("S3 uploader does not support no-op.")
|
|
|
|
if args.compress:
|
|
raise ValueError("S3 uploader does not support on-the-fly compression")
|
|
|
|
if args.relative_only:
|
|
raise ValueError("S3 uploader does not support the --relative-only option")
|
|
|
|
if args.keep_hierarchy:
|
|
raise ValueError("S3 uploader does not support the --keep-hierarchy option")
|
|
|
|
packer = create_s3packer(bpaths[0], ppath, pathlib.PurePosixPath(target))
|
|
|
|
elif (
|
|
target.startswith("shaman+http:/")
|
|
or target.startswith("shaman+https:/")
|
|
or target.startswith("shaman:/")
|
|
):
|
|
if len(bpaths) > 1:
|
|
raise ValueError("Shaman uploader does not support --sequence")
|
|
|
|
if args.noop:
|
|
raise ValueError("Shaman uploader does not support no-op.")
|
|
|
|
if args.compress:
|
|
raise ValueError("Shaman uploader does not support on-the-fly compression")
|
|
|
|
if args.relative_only:
|
|
raise ValueError(
|
|
"Shaman uploader does not support the --relative-only option"
|
|
)
|
|
|
|
if args.keep_hierarchy:
|
|
raise ValueError(
|
|
"Shaman uploader does not support the --keep-hierarchy option"
|
|
)
|
|
|
|
packer = create_shamanpacker(bpaths[0], ppath, target)
|
|
|
|
elif target.lower().endswith(".zip"):
|
|
from blender_asset_tracer.pack import zipped
|
|
|
|
if args.compress:
|
|
raise ValueError("ZIP packer does not support on-the-fly compression")
|
|
|
|
packer = zipped.ZipPacker(
|
|
bpaths, ppath, target, noop=args.noop, relative_only=args.relative_only,
|
|
keep_hierarchy=args.keep_hierarchy,
|
|
)
|
|
else:
|
|
packer = pack.Packer(
|
|
bpaths,
|
|
ppath,
|
|
target,
|
|
noop=args.noop,
|
|
compress=args.compress,
|
|
relative_only=args.relative_only,
|
|
keep_hierarchy=args.keep_hierarchy,
|
|
)
|
|
|
|
if args.exclude:
|
|
# args.exclude is a list, due to nargs='*', so we have to split and flatten.
|
|
globs = [glob for globs in args.exclude for glob in globs.split()]
|
|
log.info("Excluding: %s", ", ".join(repr(g) for g in globs))
|
|
packer.exclude(*globs)
|
|
return packer
|
|
|
|
|
|
def create_s3packer(bpath, ppath, tpath) -> pack.Packer:
|
|
from blender_asset_tracer.pack import s3
|
|
|
|
# Split the target path into 's3:/', hostname, and actual target path
|
|
parts = tpath.parts
|
|
endpoint = "https://%s/" % parts[1]
|
|
tpath = pathlib.Path(*tpath.parts[2:])
|
|
log.info("Uploading to S3-compatible storage %s at %s", endpoint, tpath)
|
|
|
|
return s3.S3Packer(bpath, ppath, tpath, endpoint=endpoint)
|
|
|
|
|
|
def create_shamanpacker(
|
|
bpath: pathlib.Path, ppath: pathlib.Path, tpath: str
|
|
) -> pack.Packer:
|
|
"""Creates a package for sending files to a Shaman server.
|
|
|
|
URLs should have the form:
|
|
shaman://hostname/base/url#jobID
|
|
This uses HTTPS to connect to the server. To connect using HTTP, use:
|
|
shaman+http://hostname/base-url#jobID
|
|
"""
|
|
from blender_asset_tracer.pack import shaman
|
|
|
|
endpoint, checkout_id = shaman.parse_endpoint(tpath)
|
|
if not checkout_id:
|
|
log.warning(
|
|
"No checkout ID given on the URL. Going to send BAT pack to Shaman, "
|
|
"but NOT creating a checkout"
|
|
)
|
|
|
|
log.info("Uploading to Shaman server %s with job %s", endpoint, checkout_id)
|
|
return shaman.ShamanPacker(
|
|
bpath, ppath, "/", endpoint=endpoint, checkout_id=checkout_id
|
|
)
|
|
|
|
|
|
def paths_from_cli(args) -> typing.Tuple[typing.List[pathlib.Path], pathlib.Path, str]:
|
|
"""Return paths to blendfile(s), project, and pack target.
|
|
|
|
Calls sys.exit() if anything is wrong.
|
|
"""
|
|
# Collect blend files from positional and --sequence arguments.
|
|
bpaths = [] # type: typing.List[pathlib.Path]
|
|
if args.blendfile is not None:
|
|
bpaths.append(args.blendfile)
|
|
if args.sequence:
|
|
bpaths.extend(args.sequence)
|
|
if not bpaths:
|
|
log.critical("No blend file specified. Provide a positional blendfile or use --sequence.")
|
|
sys.exit(3)
|
|
|
|
# Deduplicate preserving order, in case the same file appears both as
|
|
# positional arg and in --sequence.
|
|
bpaths = list(dict.fromkeys(bpaths))
|
|
|
|
# Validate each blend file and make absolute.
|
|
for i, bpath in enumerate(bpaths):
|
|
if not bpath.exists():
|
|
log.critical("File %s does not exist", bpath)
|
|
sys.exit(3)
|
|
if bpath.is_dir():
|
|
log.critical("%s is a directory, should be a blend file", bpath)
|
|
sys.exit(3)
|
|
bpaths[i] = bpathlib.make_absolute(bpath)
|
|
|
|
tpath = args.target
|
|
|
|
if args.project is None:
|
|
try:
|
|
ppath = derive_common_project(bpaths)
|
|
except ValueError as ex:
|
|
log.critical("%s", ex)
|
|
sys.exit(5)
|
|
log.warning("No project path given, using %s", ppath)
|
|
else:
|
|
ppath = bpathlib.make_absolute(args.project)
|
|
|
|
if not ppath.exists():
|
|
log.critical("Project directory %s does not exist", ppath)
|
|
sys.exit(5)
|
|
|
|
if not ppath.is_dir():
|
|
log.warning(
|
|
"Project path %s is not a directory; using the parent %s",
|
|
ppath,
|
|
ppath.parent,
|
|
)
|
|
ppath = ppath.parent
|
|
|
|
for bpath in bpaths:
|
|
try:
|
|
bpath.relative_to(ppath)
|
|
except ValueError:
|
|
log.critical(
|
|
"Project directory %s does not contain blend file %s",
|
|
ppath,
|
|
bpath,
|
|
)
|
|
sys.exit(5)
|
|
|
|
for bpath in bpaths:
|
|
log.info("Blend file to pack: %s", bpath)
|
|
log.info("Project path: %s", ppath)
|
|
log.info("Pack will be created in: %s", tpath)
|
|
|
|
return bpaths, ppath, tpath
|