autour_de_minuit/blender-asset-tracer
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Cleanup: reformat with Black

Browse Source
This commit is contained in:
Sybren A. Stüvel 2021-07-22 10:13:14 +02:00
parent 99389e8ece
commit 803c38dac1
52 changed files with 2587 additions and 1782 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
blender_asset_tracer/__init__.py
View File

@ -20,4 +20,4 @@
# <pep8 compliant> # <pep8 compliant>
__version__ = '1.3.1' __version__ = "1.3.1"

231
blender_asset_tracer/blendfile/__init__.py
View File

@ -38,34 +38,35 @@ from blender_asset_tracer import bpathlib
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
FILE_BUFFER_SIZE = 1024 * 1024 FILE_BUFFER_SIZE = 1024 * 1024
BLENDFILE_MAGIC = b'BLENDER' BLENDFILE_MAGIC = b"BLENDER"
GZIP_MAGIC = b'\x1f\x8b' GZIP_MAGIC = b"\x1f\x8b"
BFBList = typing.List['BlendFileBlock'] BFBList = typing.List["BlendFileBlock"]
_cached_bfiles = {} # type: typing.Dict[pathlib.Path, BlendFile] _cached_bfiles = {} # type: typing.Dict[pathlib.Path, BlendFile]
def open_cached(path: pathlib.Path, mode='rb', def open_cached(
assert_cached: typing.Optional[bool] = None) -> 'BlendFile': path: pathlib.Path, mode="rb", assert_cached: typing.Optional[bool] = None
) -> "BlendFile":
"""Open a blend file, ensuring it is only opened once.""" """Open a blend file, ensuring it is only opened once."""
my_log = log.getChild('open_cached') my_log = log.getChild("open_cached")
bfile_path = bpathlib.make_absolute(path) bfile_path = bpathlib.make_absolute(path)
if assert_cached is not None: if assert_cached is not None:
is_cached = bfile_path in _cached_bfiles is_cached = bfile_path in _cached_bfiles
if assert_cached and not is_cached: if assert_cached and not is_cached:
raise AssertionError('File %s was not cached' % bfile_path) raise AssertionError("File %s was not cached" % bfile_path)
elif not assert_cached and is_cached: elif not assert_cached and is_cached:
raise AssertionError('File %s was cached' % bfile_path) raise AssertionError("File %s was cached" % bfile_path)
try: try:
bfile = _cached_bfiles[bfile_path] bfile = _cached_bfiles[bfile_path]
except KeyError: except KeyError:
my_log.debug('Opening non-cached %s', path) my_log.debug("Opening non-cached %s", path)
bfile = BlendFile(path, mode=mode) bfile = BlendFile(path, mode=mode)
_cached_bfiles[bfile_path] = bfile _cached_bfiles[bfile_path] = bfile
else: else:
my_log.debug('Returning cached %s', path) my_log.debug("Returning cached %s", path)
return bfile return bfile
@ -76,13 +77,13 @@ def close_all_cached() -> None:
# Don't even log anything when there is nothing to close # Don't even log anything when there is nothing to close
return return
log.debug('Closing %d cached blend files', len(_cached_bfiles)) log.debug("Closing %d cached blend files", len(_cached_bfiles))
for bfile in list(_cached_bfiles.values()): for bfile in list(_cached_bfiles.values()):
bfile.close() bfile.close()
_cached_bfiles.clear() _cached_bfiles.clear()
def _cache(path: pathlib.Path, bfile: 'BlendFile'): def _cache(path: pathlib.Path, bfile: "BlendFile"):
"""Add a BlendFile to the cache.""" """Add a BlendFile to the cache."""
bfile_path = bpathlib.make_absolute(path) bfile_path = bpathlib.make_absolute(path)
_cached_bfiles[bfile_path] = bfile _cached_bfiles[bfile_path] = bfile
@ -102,9 +103,10 @@ class BlendFile:
uncompressed files, but a temporary file for compressed files. uncompressed files, but a temporary file for compressed files.
:ivar fileobj: the file object that's being accessed. :ivar fileobj: the file object that's being accessed.
""" """
log = log.getChild('BlendFile')
def __init__(self, path: pathlib.Path, mode='rb') -> None: log = log.getChild("BlendFile")
def __init__(self, path: pathlib.Path, mode="rb") -> None:
"""Create a BlendFile instance for the blend file at the path. """Create a BlendFile instance for the blend file at the path.
Opens the file for reading or writing pending on the access. Compressed Opens the file for reading or writing pending on the access. Compressed
@ -121,7 +123,9 @@ class BlendFile:
self.blocks = [] # type: BFBList self.blocks = [] # type: BFBList
"""BlendFileBlocks of this file, in disk order.""" """BlendFileBlocks of this file, in disk order."""
self.code_index = collections.defaultdict(list) # type: typing.Dict[bytes, BFBList] self.code_index = collections.defaultdict(
list
) # type: typing.Dict[bytes, BFBList]
self.structs = [] # type: typing.List[dna.Struct] self.structs = [] # type: typing.List[dna.Struct]
self.sdna_index_from_id = {} # type: typing.Dict[bytes, int] self.sdna_index_from_id = {} # type: typing.Dict[bytes, int]
self.block_from_addr = {} # type: typing.Dict[int, BlendFileBlock] self.block_from_addr = {} # type: typing.Dict[int, BlendFileBlock]
@ -141,8 +145,8 @@ class BlendFile:
correct magic bytes. correct magic bytes.
""" """
if 'b' not in mode: if "b" not in mode:
raise ValueError('Only binary modes are supported, not %r' % mode) raise ValueError("Only binary modes are supported, not %r" % mode)
self.filepath = path self.filepath = path
@ -165,7 +169,9 @@ class BlendFile:
with gzip.GzipFile(fileobj=fileobj, mode=mode) as gzfile: with gzip.GzipFile(fileobj=fileobj, mode=mode) as gzfile:
magic = gzfile.read(len(BLENDFILE_MAGIC)) magic = gzfile.read(len(BLENDFILE_MAGIC))
if magic != BLENDFILE_MAGIC: if magic != BLENDFILE_MAGIC:
raise exceptions.BlendFileError("Compressed file is not a blend file", path) raise exceptions.BlendFileError(
"Compressed file is not a blend file", path
)
data = magic data = magic
while data: while data:
@ -187,10 +193,10 @@ class BlendFile:
self.sdna_index_from_id.clear() self.sdna_index_from_id.clear()
while True: while True:
block = BlendFileBlock(self) block = BlendFileBlock(self)
if block.code == b'ENDB': if block.code == b"ENDB":
break break
if block.code == b'DNA1': if block.code == b"DNA1":
self.decode_structs(block) self.decode_structs(block)
else: else:
self.fileobj.seek(block.size, os.SEEK_CUR) self.fileobj.seek(block.size, os.SEEK_CUR)
@ -200,33 +206,34 @@ class BlendFile:
self.block_from_addr[block.addr_old] = block self.block_from_addr[block.addr_old] = block
if not self.structs: if not self.structs:
raise exceptions.NoDNA1Block("No DNA1 block in file, not a valid .blend file", raise exceptions.NoDNA1Block(
self.filepath) "No DNA1 block in file, not a valid .blend file", self.filepath
)
def __repr__(self) -> str: def __repr__(self) -> str:
clsname = self.__class__.__qualname__ clsname = self.__class__.__qualname__
if self.filepath == self.raw_filepath: if self.filepath == self.raw_filepath:
return '<%s %r>' % (clsname, self.filepath) return "<%s %r>" % (clsname, self.filepath)
return '<%s %r reading from %r>' % (clsname, self.filepath, self.raw_filepath) return "<%s %r reading from %r>" % (clsname, self.filepath, self.raw_filepath)
def __enter__(self) -> 'BlendFile': def __enter__(self) -> "BlendFile":
return self return self
def __exit__(self, exctype, excvalue, traceback) -> None: def __exit__(self, exctype, excvalue, traceback) -> None:
self.close() self.close()
def copy_and_rebind(self, path: pathlib.Path, mode='rb') -> None: def copy_and_rebind(self, path: pathlib.Path, mode="rb") -> None:
"""Change which file is bound to this BlendFile. """Change which file is bound to this BlendFile.
This allows cloning a previously opened file, and rebinding it to reuse This allows cloning a previously opened file, and rebinding it to reuse
the already-loaded DNA structs and data blocks. the already-loaded DNA structs and data blocks.
""" """
log.debug('Rebinding %r to %s', self, path) log.debug("Rebinding %r to %s", self, path)
self.close() self.close()
_uncache(self.filepath) _uncache(self.filepath)
self.log.debug('Copying %s to %s', self.filepath, path) self.log.debug("Copying %s to %s", self.filepath, path)
# TODO(Sybren): remove str() calls when targeting Python 3.6+ # TODO(Sybren): remove str() calls when targeting Python 3.6+
shutil.copy(str(self.filepath), str(path)) shutil.copy(str(self.filepath), str(path))
@ -239,10 +246,10 @@ class BlendFile:
def mark_modified(self) -> None: def mark_modified(self) -> None:
"""Recompess the file when it is closed.""" """Recompess the file when it is closed."""
self.log.debug('Marking %s as modified', self.raw_filepath) self.log.debug("Marking %s as modified", self.raw_filepath)
self._is_modified = True self._is_modified = True
def find_blocks_from_code(self, code: bytes) -> typing.List['BlendFileBlock']: def find_blocks_from_code(self, code: bytes) -> typing.List["BlendFileBlock"]:
assert isinstance(code, bytes) assert isinstance(code, bytes)
return self.code_index[code] return self.code_index[code]
@ -255,13 +262,13 @@ class BlendFile:
return return
if self._is_modified: if self._is_modified:
log.debug('closing blend file %s after it was modified', self.raw_filepath) log.debug("closing blend file %s after it was modified", self.raw_filepath)
if self._is_modified and self.is_compressed: if self._is_modified and self.is_compressed:
log.debug("recompressing modified blend file %s", self.raw_filepath) log.debug("recompressing modified blend file %s", self.raw_filepath)
self.fileobj.seek(os.SEEK_SET, 0) self.fileobj.seek(os.SEEK_SET, 0)
with gzip.open(str(self.filepath), 'wb') as gzfile: with gzip.open(str(self.filepath), "wb") as gzfile:
while True: while True:
data = self.fileobj.read(FILE_BUFFER_SIZE) data = self.fileobj.read(FILE_BUFFER_SIZE)
if not data: if not data:
@ -284,11 +291,15 @@ class BlendFile:
curr_struct = self.structs[sdna_index_curr] curr_struct = self.structs[sdna_index_curr]
next_struct = self.structs[sdna_index_next] next_struct = self.structs[sdna_index_next]
if curr_struct.size > next_struct.size: if curr_struct.size > next_struct.size:
raise RuntimeError("Can't refine to smaller type (%s -> %s)" % raise RuntimeError(
(curr_struct.dna_type_id.decode('utf-8'), "Can't refine to smaller type (%s -> %s)"
next_struct.dna_type_id.decode('utf-8'))) % (
curr_struct.dna_type_id.decode("utf-8"),
next_struct.dna_type_id.decode("utf-8"),
)
)
def decode_structs(self, block: 'BlendFileBlock'): def decode_structs(self, block: "BlendFileBlock"):
""" """
DNACatalog is a catalog of all information in the DNA1 file-block DNACatalog is a catalog of all information in the DNA1 file-block
""" """
@ -356,7 +367,9 @@ class BlendFile:
dna_offset = 0 dna_offset = 0
for field_index in range(fields_len): for field_index in range(fields_len):
field_type_index, field_name_index = shortstruct2.unpack_from(data, offset) field_type_index, field_name_index = shortstruct2.unpack_from(
data, offset
)
offset += 4 offset += 4
dna_type = types[field_type_index] dna_type = types[field_type_index]
@ -381,18 +394,22 @@ class BlendFile:
root = bpathlib.BlendPath(bfile_dir) root = bpathlib.BlendPath(bfile_dir)
abspath = relpath.absolute(root) abspath = relpath.absolute(root)
my_log = self.log.getChild('abspath') my_log = self.log.getChild("abspath")
my_log.debug('Resolved %s relative to %s to %s', relpath, self.filepath, abspath) my_log.debug(
"Resolved %s relative to %s to %s", relpath, self.filepath, abspath
)
return abspath return abspath
def dereference_pointer(self, address: int) -> 'BlendFileBlock': def dereference_pointer(self, address: int) -> "BlendFileBlock":
"""Return the pointed-to block, or raise SegmentationFault.""" """Return the pointed-to block, or raise SegmentationFault."""
try: try:
return self.block_from_addr[address] return self.block_from_addr[address]
except KeyError: except KeyError:
raise exceptions.SegmentationFault('address does not exist', address) from None raise exceptions.SegmentationFault(
"address does not exist", address
) from None
def struct(self, name: bytes) -> dna.Struct: def struct(self, name: bytes) -> dna.Struct:
index = self.sdna_index_from_id[name] index = self.sdna_index_from_id[name]
@ -410,19 +427,26 @@ class BlendFileBlock:
# dependency tracer significantly (p<0.001) faster. In my test case the # dependency tracer significantly (p<0.001) faster. In my test case the
# speed improvement was 16% for a 'bam list' command. # speed improvement was 16% for a 'bam list' command.
__slots__ = ( __slots__ = (
'bfile', 'code', 'size', 'addr_old', 'sdna_index', "bfile",
'count', 'file_offset', 'endian', '_id_name', "code",
"size",
"addr_old",
"sdna_index",
"count",
"file_offset",
"endian",
"_id_name",
) )
log = log.getChild('BlendFileBlock') log = log.getChild("BlendFileBlock")
old_structure = struct.Struct(b'4sI') old_structure = struct.Struct(b"4sI")
"""old blend files ENDB block structure""" """old blend files ENDB block structure"""
def __init__(self, bfile: BlendFile) -> None: def __init__(self, bfile: BlendFile) -> None:
self.bfile = bfile self.bfile = bfile
# Defaults; actual values are set by interpreting the block header. # Defaults; actual values are set by interpreting the block header.
self.code = b'' self.code = b""
self.size = 0 self.size = 0
self.addr_old = 0 self.addr_old = 0
self.sdna_index = 0 self.sdna_index = 0
@ -438,10 +462,14 @@ class BlendFileBlock:
header_struct = bfile.block_header_struct header_struct = bfile.block_header_struct
data = bfile.fileobj.read(header_struct.size) data = bfile.fileobj.read(header_struct.size)
if len(data) != header_struct.size: if len(data) != header_struct.size:
self.log.warning("Blend file %s seems to be truncated, " self.log.warning(
"expected %d bytes but could read only %d", "Blend file %s seems to be truncated, "
bfile.filepath, header_struct.size, len(data)) "expected %d bytes but could read only %d",
self.code = b'ENDB' bfile.filepath,
header_struct.size,
len(data),
)
self.code = b"ENDB"
return return
# header size can be 8, 20, or 24 bytes long # header size can be 8, 20, or 24 bytes long
@ -449,14 +477,14 @@ class BlendFileBlock:
# 20: normal headers 32 bit platform # 20: normal headers 32 bit platform
# 24: normal headers 64 bit platform # 24: normal headers 64 bit platform
if len(data) <= 15: if len(data) <= 15:
self.log.debug('interpreting block as old-style ENB block') self.log.debug("interpreting block as old-style ENB block")
blockheader = self.old_structure.unpack(data) blockheader = self.old_structure.unpack(data)
self.code = self.endian.read_data0(blockheader[0]) self.code = self.endian.read_data0(blockheader[0])
return return
blockheader = header_struct.unpack(data) blockheader = header_struct.unpack(data)
self.code = self.endian.read_data0(blockheader[0]) self.code = self.endian.read_data0(blockheader[0])
if self.code != b'ENDB': if self.code != b"ENDB":
self.size = blockheader[1] self.size = blockheader[1]
self.addr_old = blockheader[2] self.addr_old = blockheader[2]
self.sdna_index = blockheader[3] self.sdna_index = blockheader[3]
@ -478,11 +506,13 @@ class BlendFileBlock:
def __eq__(self, other: object) -> bool: def __eq__(self, other: object) -> bool:
if not isinstance(other, BlendFileBlock): if not isinstance(other, BlendFileBlock):
return False return False
return (self.code == other.code and return (
self.addr_old == other.addr_old and self.code == other.code
self.bfile.filepath == other.bfile.filepath) and self.addr_old == other.addr_old
and self.bfile.filepath == other.bfile.filepath
)
def __lt__(self, other: 'BlendFileBlock') -> bool: def __lt__(self, other: "BlendFileBlock") -> bool:
"""Order blocks by file path and offset within that file.""" """Order blocks by file path and offset within that file."""
if not isinstance(other, BlendFileBlock): if not isinstance(other, BlendFileBlock):
raise NotImplemented() raise NotImplemented()
@ -504,7 +534,7 @@ class BlendFileBlock:
@property @property
def dna_type_name(self) -> str: def dna_type_name(self) -> str:
return self.dna_type_id.decode('ascii') return self.dna_type_id.decode("ascii")
@property @property
def id_name(self) -> typing.Optional[bytes]: def id_name(self) -> typing.Optional[bytes]:
@ -515,7 +545,7 @@ class BlendFileBlock:
""" """
if self._id_name is ...: if self._id_name is ...:
try: try:
self._id_name = self[b'id', b'name'] self._id_name = self[b"id", b"name"]
except KeyError: except KeyError:
self._id_name = None self._id_name = None
@ -553,16 +583,19 @@ class BlendFileBlock:
:returns: tuple (offset in bytes, length of array in items) :returns: tuple (offset in bytes, length of array in items)
""" """
field, field_offset = self.dna_type.field_from_path(self.bfile.header.pointer_size, path) field, field_offset = self.dna_type.field_from_path(
self.bfile.header.pointer_size, path
)
return self.file_offset + field_offset, field.name.array_size return self.file_offset + field_offset, field.name.array_size
def get(self, def get(
path: dna.FieldPath, self,
default=..., path: dna.FieldPath,
null_terminated=True, default=...,
as_str=False, null_terminated=True,
return_field=False as_str=False,
) -> typing.Any: return_field=False,
) -> typing.Any:
"""Read a property and return the value. """Read a property and return the value.
:param path: name of the property (like `b'loc'`), tuple of names :param path: name of the property (like `b'loc'`), tuple of names
@ -583,21 +616,25 @@ class BlendFileBlock:
dna_struct = self.bfile.structs[self.sdna_index] dna_struct = self.bfile.structs[self.sdna_index]
field, value = dna_struct.field_get( field, value = dna_struct.field_get(
self.bfile.header, self.bfile.fileobj, path, self.bfile.header,
self.bfile.fileobj,
path,
default=default, default=default,
null_terminated=null_terminated, as_str=as_str, null_terminated=null_terminated,
as_str=as_str,
) )
if return_field: if return_field:
return value, field return value, field
return value return value
def get_recursive_iter(self, def get_recursive_iter(
path: dna.FieldPath, self,
path_root: dna.FieldPath = b'', path: dna.FieldPath,
default=..., path_root: dna.FieldPath = b"",
null_terminated=True, default=...,
as_str=True, null_terminated=True,
) -> typing.Iterator[typing.Tuple[dna.FieldPath, typing.Any]]: as_str=True,
) -> typing.Iterator[typing.Tuple[dna.FieldPath, typing.Any]]:
"""Generator, yields (path, property value) tuples. """Generator, yields (path, property value) tuples.
If a property cannot be decoded, a string representing its DNA type If a property cannot be decoded, a string representing its DNA type
@ -613,20 +650,24 @@ class BlendFileBlock:
try: try:
# Try accessing as simple property # Try accessing as simple property
yield (path_full, yield (path_full, self.get(path_full, default, null_terminated, as_str))
self.get(path_full, default, null_terminated, as_str))
except exceptions.NoReaderImplemented as ex: except exceptions.NoReaderImplemented as ex:
# This was not a simple property, so recurse into its DNA Struct. # This was not a simple property, so recurse into its DNA Struct.
dna_type = ex.dna_type dna_type = ex.dna_type
struct_index = self.bfile.sdna_index_from_id.get(dna_type.dna_type_id) struct_index = self.bfile.sdna_index_from_id.get(dna_type.dna_type_id)
if struct_index is None: if struct_index is None:
yield (path_full, "<%s>" % dna_type.dna_type_id.decode('ascii')) yield (path_full, "<%s>" % dna_type.dna_type_id.decode("ascii"))
return return
# Recurse through the fields. # Recurse through the fields.
for f in dna_type.fields: for f in dna_type.fields:
yield from self.get_recursive_iter(f.name.name_only, path_full, default=default, yield from self.get_recursive_iter(
null_terminated=null_terminated, as_str=as_str) f.name.name_only,
path_full,
default=default,
null_terminated=null_terminated,
as_str=as_str,
)
def hash(self) -> int: def hash(self) -> int:
"""Generate a pointer-independent hash for the block. """Generate a pointer-independent hash for the block.
@ -657,9 +698,10 @@ class BlendFileBlock:
return dna_struct.field_set(self.bfile.header, self.bfile.fileobj, path, value) return dna_struct.field_set(self.bfile.header, self.bfile.fileobj, path, value)
def get_pointer( def get_pointer(
self, path: dna.FieldPath, self,
default=..., path: dna.FieldPath,
) -> typing.Union[None, 'BlendFileBlock']: default=...,
) -> typing.Union[None, "BlendFileBlock"]:
"""Same as get() but dereferences a pointer. """Same as get() but dereferences a pointer.
:raises exceptions.SegmentationFault: when there is no datablock with :raises exceptions.SegmentationFault: when there is no datablock with
@ -681,8 +723,9 @@ class BlendFileBlock:
ex.field_path = path ex.field_path = path
raise raise
def iter_array_of_pointers(self, path: dna.FieldPath, array_size: int) \ def iter_array_of_pointers(
-> typing.Iterator['BlendFileBlock']: self, path: dna.FieldPath, array_size: int
) -> typing.Iterator["BlendFileBlock"]:
"""Dereference pointers from an array-of-pointers field. """Dereference pointers from an array-of-pointers field.
Use this function when you have a field like Mesh materials: Use this function when you have a field like Mesh materials:
@ -698,8 +741,9 @@ class BlendFileBlock:
array = self.get_pointer(path) array = self.get_pointer(path)
assert array is not None assert array is not None
assert array.code == b'DATA', \ assert array.code == b"DATA", (
'Array data block should have code DATA, is %r' % array.code.decode() "Array data block should have code DATA, is %r" % array.code.decode()
)
file_offset = array.file_offset file_offset = array.file_offset
endian = self.bfile.header.endian endian = self.bfile.header.endian
@ -713,8 +757,9 @@ class BlendFileBlock:
continue continue
yield self.bfile.dereference_pointer(address) yield self.bfile.dereference_pointer(address)
def iter_fixed_array_of_pointers(self, path: dna.FieldPath) \ def iter_fixed_array_of_pointers(
-> typing.Iterator['BlendFileBlock']: self, path: dna.FieldPath
) -> typing.Iterator["BlendFileBlock"]:
"""Yield blocks from a fixed-size array field. """Yield blocks from a fixed-size array field.
Use this function when you have a field like lamp textures: Use this function when you have a field like lamp textures:
@ -758,16 +803,18 @@ class BlendFileBlock:
try: try:
yield self[k] yield self[k]
except exceptions.NoReaderImplemented as ex: except exceptions.NoReaderImplemented as ex:
yield '<%s>' % ex.dna_type.dna_type_id.decode('ascii') yield "<%s>" % ex.dna_type.dna_type_id.decode("ascii")
def items(self) -> typing.Iterable[typing.Tuple[bytes, typing.Any]]: def items(self) -> typing.Iterable[typing.Tuple[bytes, typing.Any]]:
for k in self.keys(): for k in self.keys():
try: try:
yield (k, self[k]) yield (k, self[k])
except exceptions.NoReaderImplemented as ex: except exceptions.NoReaderImplemented as ex:
yield (k, '<%s>' % ex.dna_type.dna_type_id.decode('ascii')) yield (k, "<%s>" % ex.dna_type.dna_type_id.decode("ascii"))
def items_recursive(self) -> typing.Iterator[typing.Tuple[dna.FieldPath, typing.Any]]: def items_recursive(
self,
) -> typing.Iterator[typing.Tuple[dna.FieldPath, typing.Any]]:
"""Generator, yields (property path, property value) recursively for all properties.""" """Generator, yields (property path, property value) recursively for all properties."""
for k in self.keys(): for k in self.keys():
yield from self.get_recursive_iter(k, as_str=False) yield from self.get_recursive_iter(k, as_str=False)

137
blender_asset_tracer/blendfile/dna.py
View File

@ -42,38 +42,38 @@ class Name:
self.array_size = self.calc_array_size() self.array_size = self.calc_array_size()
def __repr__(self): def __repr__(self):
return '%s(%r)' % (type(self).__qualname__, self.name_full) return "%s(%r)" % (type(self).__qualname__, self.name_full)
def as_reference(self, parent) -> bytes: def as_reference(self, parent) -> bytes:
if not parent: if not parent:
return self.name_only return self.name_only
return parent + b'.' + self.name_only return parent + b"." + self.name_only
def calc_name_only(self) -> bytes: def calc_name_only(self) -> bytes:
result = self.name_full.strip(b'*()') result = self.name_full.strip(b"*()")
index = result.find(b'[') index = result.find(b"[")
if index == -1: if index == -1:
return result return result
return result[:index] return result[:index]
def calc_is_pointer(self) -> bool: def calc_is_pointer(self) -> bool:
return b'*' in self.name_full return b"*" in self.name_full
def calc_is_method_pointer(self): def calc_is_method_pointer(self):
return b'(*' in self.name_full return b"(*" in self.name_full
def calc_array_size(self): def calc_array_size(self):
result = 1 result = 1
partial_name = self.name_full partial_name = self.name_full
while True: while True:
idx_start = partial_name.find(b'[') idx_start = partial_name.find(b"[")
if idx_start < 0: if idx_start < 0:
break break
idx_stop = partial_name.find(b']') idx_stop = partial_name.find(b"]")
result *= int(partial_name[idx_start + 1:idx_stop]) result *= int(partial_name[idx_start + 1 : idx_stop])
partial_name = partial_name[idx_stop + 1:] partial_name = partial_name[idx_stop + 1 :]
return result return result
@ -89,24 +89,20 @@ class Field:
:ivar offset: cached offset of the field, in bytes. :ivar offset: cached offset of the field, in bytes.
""" """
def __init__(self, def __init__(self, dna_type: "Struct", name: Name, size: int, offset: int) -> None:
dna_type: 'Struct',
name: Name,
size: int,
offset: int) -> None:
self.dna_type = dna_type self.dna_type = dna_type
self.name = name self.name = name
self.size = size self.size = size
self.offset = offset self.offset = offset
def __repr__(self): def __repr__(self):
return '<%r %r (%s)>' % (type(self).__qualname__, self.name, self.dna_type) return "<%r %r (%s)>" % (type(self).__qualname__, self.name, self.dna_type)
class Struct: class Struct:
"""dna.Struct is a C-type structure stored in the DNA.""" """dna.Struct is a C-type structure stored in the DNA."""
log = log.getChild('Struct') log = log.getChild("Struct")
def __init__(self, dna_type_id: bytes, size: int = None) -> None: def __init__(self, dna_type_id: bytes, size: int = None) -> None:
""" """
@ -121,13 +117,13 @@ class Struct:
self._fields_by_name = {} # type: typing.Dict[bytes, Field] self._fields_by_name = {} # type: typing.Dict[bytes, Field]
def __repr__(self): def __repr__(self):
return '%s(%r)' % (type(self).__qualname__, self.dna_type_id) return "%s(%r)" % (type(self).__qualname__, self.dna_type_id)
@property @property
def size(self) -> int: def size(self) -> int:
if self._size is None: if self._size is None:
if not self._fields: if not self._fields:
raise ValueError('Unable to determine size of fieldless %r' % self) raise ValueError("Unable to determine size of fieldless %r" % self)
last_field = max(self._fields, key=lambda f: f.offset) last_field = max(self._fields, key=lambda f: f.offset)
self._size = last_field.offset + last_field.size self._size = last_field.offset + last_field.size
return self._size return self._size
@ -151,10 +147,9 @@ class Struct:
def has_field(self, field_name: bytes) -> bool: def has_field(self, field_name: bytes) -> bool:
return field_name in self._fields_by_name return field_name in self._fields_by_name
def field_from_path(self, def field_from_path(
pointer_size: int, self, pointer_size: int, path: FieldPath
path: FieldPath) \ ) -> typing.Tuple[Field, int]:
-> typing.Tuple[Field, int]:
""" """
Support lookups as bytes or a tuple of bytes and optional index. Support lookups as bytes or a tuple of bytes and optional index.
@ -181,12 +176,14 @@ class Struct:
index = 0 index = 0
if not isinstance(name, bytes): if not isinstance(name, bytes):
raise TypeError('name should be bytes, but is %r' % type(name)) raise TypeError("name should be bytes, but is %r" % type(name))
field = self._fields_by_name.get(name) field = self._fields_by_name.get(name)
if not field: if not field:
raise KeyError('%r has no field %r, only %r' % raise KeyError(
(self, name, sorted(self._fields_by_name.keys()))) "%r has no field %r, only %r"
% (self, name, sorted(self._fields_by_name.keys()))
)
offset = field.offset offset = field.offset
if index: if index:
@ -195,8 +192,10 @@ class Struct:
else: else:
index_offset = field.dna_type.size * index index_offset = field.dna_type.size * index
if index_offset >= field.size: if index_offset >= field.size:
raise OverflowError('path %r is out of bounds of its DNA type %s' % raise OverflowError(
(path, field.dna_type)) "path %r is out of bounds of its DNA type %s"
% (path, field.dna_type)
)
offset += index_offset offset += index_offset
if name_tail: if name_tail:
@ -205,14 +204,15 @@ class Struct:
return field, offset return field, offset
def field_get(self, def field_get(
file_header: header.BlendFileHeader, self,
fileobj: typing.IO[bytes], file_header: header.BlendFileHeader,
path: FieldPath, fileobj: typing.IO[bytes],
default=..., path: FieldPath,
null_terminated=True, default=...,
as_str=True, null_terminated=True,
) -> typing.Tuple[typing.Optional[Field], typing.Any]: as_str=True,
) -> typing.Tuple[typing.Optional[Field], typing.Any]:
"""Read the value of the field from the blend file. """Read the value of the field from the blend file.
Assumes the file pointer of `fileobj` is seek()ed to the start of the Assumes the file pointer of `fileobj` is seek()ed to the start of the
@ -248,22 +248,26 @@ class Struct:
# Some special cases (pointers, strings/bytes) # Some special cases (pointers, strings/bytes)
if dna_name.is_pointer: if dna_name.is_pointer:
return field, endian.read_pointer(fileobj, file_header.pointer_size) return field, endian.read_pointer(fileobj, file_header.pointer_size)
if dna_type.dna_type_id == b'char': if dna_type.dna_type_id == b"char":
return field, self._field_get_char(file_header, fileobj, field, null_terminated, as_str) return field, self._field_get_char(
file_header, fileobj, field, null_terminated, as_str
)
simple_readers = { simple_readers = {
b'int': endian.read_int, b"int": endian.read_int,
b'short': endian.read_short, b"short": endian.read_short,
b'uint64_t': endian.read_ulong, b"uint64_t": endian.read_ulong,
b'float': endian.read_float, b"float": endian.read_float,
} }
try: try:
simple_reader = simple_readers[dna_type.dna_type_id] simple_reader = simple_readers[dna_type.dna_type_id]
except KeyError: except KeyError:
raise exceptions.NoReaderImplemented( raise exceptions.NoReaderImplemented(
"%r exists but not simple type (%r), can't resolve field %r" % "%r exists but not simple type (%r), can't resolve field %r"
(path, dna_type.dna_type_id.decode(), dna_name.name_only), % (path, dna_type.dna_type_id.decode(), dna_name.name_only),
dna_name, dna_type) from None dna_name,
dna_type,
) from None
if isinstance(path, tuple) and len(path) > 1 and isinstance(path[-1], int): if isinstance(path, tuple) and len(path) > 1 and isinstance(path[-1], int):
# The caller wants to get a single item from an array. The offset we seeked to already # The caller wants to get a single item from an array. The offset we seeked to already
@ -275,12 +279,14 @@ class Struct:
return field, [simple_reader(fileobj) for _ in range(dna_name.array_size)] return field, [simple_reader(fileobj) for _ in range(dna_name.array_size)]
return field, simple_reader(fileobj) return field, simple_reader(fileobj)
def _field_get_char(self, def _field_get_char(
file_header: header.BlendFileHeader, self,
fileobj: typing.IO[bytes], file_header: header.BlendFileHeader,
field: 'Field', fileobj: typing.IO[bytes],
null_terminated: typing.Optional[bool], field: "Field",
as_str: bool) -> typing.Any: null_terminated: typing.Optional[bool],
as_str: bool,
) -> typing.Any:
dna_name = field.name dna_name = field.name
endian = file_header.endian endian = file_header.endian
@ -294,21 +300,23 @@ class Struct:
data = fileobj.read(dna_name.array_size) data = fileobj.read(dna_name.array_size)
if as_str: if as_str:
return data.decode('utf8') return data.decode("utf8")
return data return data
def field_set(self, def field_set(
file_header: header.BlendFileHeader, self,
fileobj: typing.IO[bytes], file_header: header.BlendFileHeader,
path: bytes, fileobj: typing.IO[bytes],
value: typing.Any): path: bytes,
value: typing.Any,
):
"""Write a value to the blend file. """Write a value to the blend file.
Assumes the file pointer of `fileobj` is seek()ed to the start of the Assumes the file pointer of `fileobj` is seek()ed to the start of the
struct on disk (e.g. the start of the BlendFileBlock containing the struct on disk (e.g. the start of the BlendFileBlock containing the
data). data).
""" """
assert isinstance(path, bytes), 'path should be bytes, but is %s' % type(path) assert isinstance(path, bytes), "path should be bytes, but is %s" % type(path)
field, offset = self.field_from_path(file_header.pointer_size, path) field, offset = self.field_from_path(file_header.pointer_size, path)
@ -316,17 +324,22 @@ class Struct:
dna_name = field.name dna_name = field.name
endian = file_header.endian endian = file_header.endian
if dna_type.dna_type_id != b'char': if dna_type.dna_type_id != b"char":
msg = "Setting type %r is not supported for %s.%s" % ( msg = "Setting type %r is not supported for %s.%s" % (
dna_type, self.dna_type_id.decode(), dna_name.name_full.decode()) dna_type,
self.dna_type_id.decode(),
dna_name.name_full.decode(),
)
raise exceptions.NoWriterImplemented(msg, dna_name, dna_type) raise exceptions.NoWriterImplemented(msg, dna_name, dna_type)
fileobj.seek(offset, os.SEEK_CUR) fileobj.seek(offset, os.SEEK_CUR)
if self.log.isEnabledFor(logging.DEBUG): if self.log.isEnabledFor(logging.DEBUG):
filepos = fileobj.tell() filepos = fileobj.tell()
thing = 'string' if isinstance(value, str) else 'bytes' thing = "string" if isinstance(value, str) else "bytes"
self.log.debug('writing %s %r at file offset %d / %x', thing, value, filepos, filepos) self.log.debug(
"writing %s %r at file offset %d / %x", thing, value, filepos, filepos
)
if isinstance(value, str): if isinstance(value, str):
return endian.write_string(fileobj, value, dna_name.array_size) return endian.write_string(fileobj, value, dna_name.array_size)

54
blender_asset_tracer/blendfile/dna_io.py
View File

@ -27,14 +27,14 @@ import typing
class EndianIO: class EndianIO:
# TODO(Sybren): note as UCHAR: struct.Struct = None and move actual structs to LittleEndianTypes # TODO(Sybren): note as UCHAR: struct.Struct = None and move actual structs to LittleEndianTypes
UCHAR = struct.Struct(b'<B') UCHAR = struct.Struct(b"<B")
USHORT = struct.Struct(b'<H') USHORT = struct.Struct(b"<H")
USHORT2 = struct.Struct(b'<HH') # two shorts in a row USHORT2 = struct.Struct(b"<HH") # two shorts in a row
SSHORT = struct.Struct(b'<h') SSHORT = struct.Struct(b"<h")
UINT = struct.Struct(b'<I') UINT = struct.Struct(b"<I")
SINT = struct.Struct(b'<i') SINT = struct.Struct(b"<i")
FLOAT = struct.Struct(b'<f') FLOAT = struct.Struct(b"<f")
ULONG = struct.Struct(b'<Q') ULONG = struct.Struct(b"<Q")
@classmethod @classmethod
def _read(cls, fileobj: typing.IO[bytes], typestruct: struct.Struct): def _read(cls, fileobj: typing.IO[bytes], typestruct: struct.Struct):
@ -42,7 +42,7 @@ class EndianIO:
try: try:
return typestruct.unpack(data)[0] return typestruct.unpack(data)[0]
except struct.error as ex: except struct.error as ex:
raise struct.error('%s (read %d bytes)' % (ex, len(data))) from None raise struct.error("%s (read %d bytes)" % (ex, len(data))) from None
@classmethod @classmethod
def read_char(cls, fileobj: typing.IO[bytes]): def read_char(cls, fileobj: typing.IO[bytes]):
@ -80,10 +80,12 @@ class EndianIO:
return cls.read_uint(fileobj) return cls.read_uint(fileobj)
if pointer_size == 8: if pointer_size == 8:
return cls.read_ulong(fileobj) return cls.read_ulong(fileobj)
raise ValueError('unsupported pointer size %d' % pointer_size) raise ValueError("unsupported pointer size %d" % pointer_size)
@classmethod @classmethod
def write_string(cls, fileobj: typing.IO[bytes], astring: str, fieldlen: int) -> int: def write_string(
cls, fileobj: typing.IO[bytes], astring: str, fieldlen: int
) -> int:
"""Write a (truncated) string as UTF-8. """Write a (truncated) string as UTF-8.
The string will always be written 0-terminated. The string will always be written 0-terminated.
@ -94,7 +96,7 @@ class EndianIO:
:returns: the number of bytes written. :returns: the number of bytes written.
""" """
assert isinstance(astring, str) assert isinstance(astring, str)
encoded = astring.encode('utf-8') encoded = astring.encode("utf-8")
# Take into account we also need space for a trailing 0-byte. # Take into account we also need space for a trailing 0-byte.
maxlen = fieldlen - 1 maxlen = fieldlen - 1
@ -106,13 +108,13 @@ class EndianIO:
# is valid UTF-8 again. # is valid UTF-8 again.
while True: while True:
try: try:
encoded.decode('utf8') encoded.decode("utf8")
except UnicodeDecodeError: except UnicodeDecodeError:
encoded = encoded[:-1] encoded = encoded[:-1]
else: else:
break break
return fileobj.write(encoded + b'\0') return fileobj.write(encoded + b"\0")
@classmethod @classmethod
def write_bytes(cls, fileobj: typing.IO[bytes], data: bytes, fieldlen: int) -> int: def write_bytes(cls, fileobj: typing.IO[bytes], data: bytes, fieldlen: int) -> int:
@ -126,7 +128,7 @@ class EndianIO:
if len(data) >= fieldlen: if len(data) >= fieldlen:
to_write = data[0:fieldlen] to_write = data[0:fieldlen]
else: else:
to_write = data + b'\0' to_write = data + b"\0"
return fileobj.write(to_write) return fileobj.write(to_write)
@ -137,12 +139,12 @@ class EndianIO:
@classmethod @classmethod
def read_data0_offset(cls, data, offset): def read_data0_offset(cls, data, offset):
add = data.find(b'\0', offset) - offset add = data.find(b"\0", offset) - offset
return data[offset:offset + add] return data[offset : offset + add]
@classmethod @classmethod
def read_data0(cls, data): def read_data0(cls, data):
add = data.find(b'\0') add = data.find(b"\0")
if add < 0: if add < 0:
return data return data
return data[:add] return data[:add]
@ -153,11 +155,11 @@ class LittleEndianTypes(EndianIO):
class BigEndianTypes(LittleEndianTypes): class BigEndianTypes(LittleEndianTypes):
UCHAR = struct.Struct(b'>B') UCHAR = struct.Struct(b">B")
USHORT = struct.Struct(b'>H') USHORT = struct.Struct(b">H")
USHORT2 = struct.Struct(b'>HH') # two shorts in a row USHORT2 = struct.Struct(b">HH") # two shorts in a row
SSHORT = struct.Struct(b'>h') SSHORT = struct.Struct(b">h")
UINT = struct.Struct(b'>I') UINT = struct.Struct(b">I")
SINT = struct.Struct(b'>i') SINT = struct.Struct(b">i")
FLOAT = struct.Struct(b'>f') FLOAT = struct.Struct(b">f")
ULONG = struct.Struct(b'>Q') ULONG = struct.Struct(b">Q")

2
blender_asset_tracer/blendfile/exceptions.py
View File

@ -32,7 +32,7 @@ class BlendFileError(Exception):
self.filepath = filepath self.filepath = filepath
def __str__(self): def __str__(self):
return '%s: %s' % (super().__str__(), self.filepath) return "%s: %s" % (super().__str__(), self.filepath)
class NoDNA1Block(BlendFileError): class NoDNA1Block(BlendFileError):

39
blender_asset_tracer/blendfile/header.py
View File

@ -37,7 +37,8 @@ class BlendFileHeader:
It contains information about the hardware architecture, which is relevant It contains information about the hardware architecture, which is relevant
to the structure of the rest of the file. to the structure of the rest of the file.
""" """
structure = struct.Struct(b'7s1s1s3s')
structure = struct.Struct(b"7s1s1s3s")
def __init__(self, fileobj: typing.IO[bytes], path: pathlib.Path) -> None: def __init__(self, fileobj: typing.IO[bytes], path: pathlib.Path) -> None:
log.debug("reading blend-file-header %s", path) log.debug("reading blend-file-header %s", path)
@ -48,31 +49,39 @@ class BlendFileHeader:
self.magic = values[0] self.magic = values[0]
pointer_size_id = values[1] pointer_size_id = values[1]
if pointer_size_id == b'-': if pointer_size_id == b"-":
self.pointer_size = 8 self.pointer_size = 8
elif pointer_size_id == b'_': elif pointer_size_id == b"_":
self.pointer_size = 4 self.pointer_size = 4
else: else:
raise exceptions.BlendFileError('invalid pointer size %r' % pointer_size_id, path) raise exceptions.BlendFileError(
"invalid pointer size %r" % pointer_size_id, path
)
endian_id = values[2] endian_id = values[2]
if endian_id == b'v': if endian_id == b"v":
self.endian = dna_io.LittleEndianTypes self.endian = dna_io.LittleEndianTypes
self.endian_str = b'<' # indication for struct.Struct() self.endian_str = b"<" # indication for struct.Struct()
elif endian_id == b'V': elif endian_id == b"V":
self.endian = dna_io.BigEndianTypes self.endian = dna_io.BigEndianTypes
self.endian_str = b'>' # indication for struct.Struct() self.endian_str = b">" # indication for struct.Struct()
else: else:
raise exceptions.BlendFileError('invalid endian indicator %r' % endian_id, path) raise exceptions.BlendFileError(
"invalid endian indicator %r" % endian_id, path
)
version_id = values[3] version_id = values[3]
self.version = int(version_id) self.version = int(version_id)
def create_block_header_struct(self) -> struct.Struct: def create_block_header_struct(self) -> struct.Struct:
"""Create a Struct instance for parsing data block headers.""" """Create a Struct instance for parsing data block headers."""
return struct.Struct(b''.join(( return struct.Struct(
self.endian_str, b"".join(
b'4sI', (
b'I' if self.pointer_size == 4 else b'Q', self.endian_str,
b'II', b"4sI",
))) b"I" if self.pointer_size == 4 else b"Q",
b"II",
)
)
)

22
blender_asset_tracer/blendfile/iterators.py
View File

@ -26,8 +26,9 @@ from . import BlendFileBlock
from .dna import FieldPath from .dna import FieldPath
def listbase(block: typing.Optional[BlendFileBlock], next_path: FieldPath = b'next') \ def listbase(
-> typing.Iterator[BlendFileBlock]: block: typing.Optional[BlendFileBlock], next_path: FieldPath = b"next"
) -> typing.Iterator[BlendFileBlock]:
"""Generator, yields all blocks in the ListBase linked list.""" """Generator, yields all blocks in the ListBase linked list."""
while block: while block:
yield block yield block
@ -37,8 +38,9 @@ def listbase(block: typing.Optional[BlendFileBlock], next_path: FieldPath = b'ne
block = block.bfile.dereference_pointer(next_ptr) block = block.bfile.dereference_pointer(next_ptr)
def sequencer_strips(sequence_editor: BlendFileBlock) \ def sequencer_strips(
-> typing.Iterator[typing.Tuple[BlendFileBlock, int]]: sequence_editor: BlendFileBlock,
) -> typing.Iterator[typing.Tuple[BlendFileBlock, int]]:
"""Generator, yield all sequencer strip blocks with their type number. """Generator, yield all sequencer strip blocks with their type number.
Recurses into meta strips, yielding both the meta strip itself and the Recurses into meta strips, yielding both the meta strip itself and the
@ -49,16 +51,16 @@ def sequencer_strips(sequence_editor: BlendFileBlock) \
def iter_seqbase(seqbase) -> typing.Iterator[typing.Tuple[BlendFileBlock, int]]: def iter_seqbase(seqbase) -> typing.Iterator[typing.Tuple[BlendFileBlock, int]]:
for seq in listbase(seqbase): for seq in listbase(seqbase):
seq.refine_type(b'Sequence') seq.refine_type(b"Sequence")
seq_type = seq[b'type'] seq_type = seq[b"type"]
yield seq, seq_type yield seq, seq_type
if seq_type == cdefs.SEQ_TYPE_META: if seq_type == cdefs.SEQ_TYPE_META:
# Recurse into this meta-sequence. # Recurse into this meta-sequence.
subseq = seq.get_pointer((b'seqbase', b'first')) subseq = seq.get_pointer((b"seqbase", b"first"))
yield from iter_seqbase(subseq) yield from iter_seqbase(subseq)
sbase = sequence_editor.get_pointer((b'seqbase', b'first')) sbase = sequence_editor.get_pointer((b"seqbase", b"first"))
yield from iter_seqbase(sbase) yield from iter_seqbase(sbase)
@ -66,5 +68,5 @@ def modifiers(object_block: BlendFileBlock) -> typing.Iterator[BlendFileBlock]:
"""Generator, yield the object's modifiers.""" """Generator, yield the object's modifiers."""
# 'ob->modifiers[...]' # 'ob->modifiers[...]'
mods = object_block.get_pointer((b'modifiers', b'first')) mods = object_block.get_pointer((b"modifiers", b"first"))
yield from listbase(mods, next_path=(b'modifier', b'next')) yield from listbase(mods, next_path=(b"modifier", b"next"))

65
blender_asset_tracer/bpathlib.py
View File

@ -35,14 +35,16 @@ class BlendPath(bytes):
def __new__(cls, path): def __new__(cls, path):
if isinstance(path, pathlib.PurePath): if isinstance(path, pathlib.PurePath):
path = str(path).encode('utf-8') path = str(path).encode("utf-8")
if not isinstance(path, bytes): if not isinstance(path, bytes):
raise TypeError('path must be bytes or pathlib.Path, but is %r' % path) raise TypeError("path must be bytes or pathlib.Path, but is %r" % path)
return super().__new__(cls, path.replace(b'\\', b'/')) return super().__new__(cls, path.replace(b"\\", b"/"))
@classmethod @classmethod
def mkrelative(cls, asset_path: pathlib.PurePath, bfile_path: pathlib.PurePath) -> 'BlendPath': def mkrelative(
cls, asset_path: pathlib.PurePath, bfile_path: pathlib.PurePath
) -> "BlendPath":
"""Construct a BlendPath to the asset relative to the blend file. """Construct a BlendPath to the asset relative to the blend file.
Assumes that bfile_path is absolute. Assumes that bfile_path is absolute.
@ -53,10 +55,14 @@ class BlendPath(bytes):
from collections import deque from collections import deque
# Only compare absolute paths. # Only compare absolute paths.
assert bfile_path.is_absolute(), \ assert bfile_path.is_absolute(), (
'BlendPath().mkrelative(bfile_path=%r) should get absolute bfile_path' % bfile_path "BlendPath().mkrelative(bfile_path=%r) should get absolute bfile_path"
assert asset_path.is_absolute(), \ % bfile_path
'BlendPath().mkrelative(asset_path=%r) should get absolute asset_path' % asset_path )
assert asset_path.is_absolute(), (
"BlendPath().mkrelative(asset_path=%r) should get absolute asset_path"
% asset_path
)
# There is no way to construct a relative path between drives. # There is no way to construct a relative path between drives.
if bfile_path.drive != asset_path.drive: if bfile_path.drive != asset_path.drive:
@ -77,8 +83,8 @@ class BlendPath(bytes):
rel_asset = pathlib.PurePath(*asset_parts) rel_asset = pathlib.PurePath(*asset_parts)
# TODO(Sybren): should we use sys.getfilesystemencoding() instead? # TODO(Sybren): should we use sys.getfilesystemencoding() instead?
rel_bytes = str(rel_asset).encode('utf-8') rel_bytes = str(rel_asset).encode("utf-8")
as_bytes = b'//' + len(bdir_parts) * b'../' + rel_bytes as_bytes = b"//" + len(bdir_parts) * b"../" + rel_bytes
return cls(as_bytes) return cls(as_bytes)
def __str__(self) -> str: def __str__(self) -> str:
@ -87,23 +93,23 @@ class BlendPath(bytes):
Undecodable bytes are ignored so this function can be safely used Undecodable bytes are ignored so this function can be safely used
for reporting. for reporting.
""" """
return self.decode('utf8', errors='replace') return self.decode("utf8", errors="replace")
def __repr__(self) -> str: def __repr__(self) -> str:
return 'BlendPath(%s)' % super().__repr__() return "BlendPath(%s)" % super().__repr__()
def __truediv__(self, subpath: bytes): def __truediv__(self, subpath: bytes):
"""Slash notation like pathlib.Path.""" """Slash notation like pathlib.Path."""
sub = BlendPath(subpath) sub = BlendPath(subpath)
if sub.is_absolute(): if sub.is_absolute():
raise ValueError("'a / b' only works when 'b' is a relative path") raise ValueError("'a / b' only works when 'b' is a relative path")
return BlendPath(self.rstrip(b'/') + b'/' + sub) return BlendPath(self.rstrip(b"/") + b"/" + sub)
def __rtruediv__(self, parentpath: bytes): def __rtruediv__(self, parentpath: bytes):
"""Slash notation like pathlib.Path.""" """Slash notation like pathlib.Path."""
if self.is_absolute(): if self.is_absolute():
raise ValueError("'a / b' only works when 'b' is a relative path") raise ValueError("'a / b' only works when 'b' is a relative path")
return BlendPath(parentpath.rstrip(b'/') + b'/' + self) return BlendPath(parentpath.rstrip(b"/") + b"/" + self)
def to_path(self) -> pathlib.PurePath: def to_path(self) -> pathlib.PurePath:
"""Convert this path to a pathlib.PurePath. """Convert this path to a pathlib.PurePath.
@ -118,32 +124,34 @@ class BlendPath(bytes):
""" """
# TODO(Sybren): once we target Python 3.6, implement __fspath__(). # TODO(Sybren): once we target Python 3.6, implement __fspath__().
try: try:
decoded = self.decode('utf8') decoded = self.decode("utf8")
except UnicodeDecodeError: except UnicodeDecodeError:
decoded = self.decode(sys.getfilesystemencoding()) decoded = self.decode(sys.getfilesystemencoding())
if self.is_blendfile_relative(): if self.is_blendfile_relative():
raise ValueError('to_path() cannot be used on blendfile-relative paths') raise ValueError("to_path() cannot be used on blendfile-relative paths")
return pathlib.PurePath(decoded) return pathlib.PurePath(decoded)
def is_blendfile_relative(self) -> bool: def is_blendfile_relative(self) -> bool:
return self[:2] == b'//' return self[:2] == b"//"
def is_absolute(self) -> bool: def is_absolute(self) -> bool:
if self.is_blendfile_relative(): if self.is_blendfile_relative():
return False return False
if self[0:1] == b'/': if self[0:1] == b"/":
return True return True
# Windows style path starting with drive letter. # Windows style path starting with drive letter.
if (len(self) >= 3 and if (
(self.decode('utf8'))[0] in string.ascii_letters and len(self) >= 3
self[1:2] == b':' and and (self.decode("utf8"))[0] in string.ascii_letters
self[2:3] in {b'\\', b'/'}): and self[1:2] == b":"
and self[2:3] in {b"\\", b"/"}
):
return True return True
return False return False
def absolute(self, root: bytes = b'') -> 'BlendPath': def absolute(self, root: bytes = b"") -> "BlendPath":
"""Determine absolute path. """Determine absolute path.
:param root: root directory to compute paths relative to. :param root: root directory to compute paths relative to.
@ -175,9 +183,9 @@ def make_absolute(path: pathlib.PurePath) -> pathlib.Path:
The type of the returned path is determined by the current platform. The type of the returned path is determined by the current platform.
""" """
str_path = path.as_posix() str_path = path.as_posix()
if len(str_path) >= 2 and str_path[0].isalpha() and str_path[1] == ':': if len(str_path) >= 2 and str_path[0].isalpha() and str_path[1] == ":":
# This is an absolute Windows path. It must be handled with care on non-Windows platforms. # This is an absolute Windows path. It must be handled with care on non-Windows platforms.
if platform.system() != 'Windows': if platform.system() != "Windows":
# Normalize the POSIX-like part of the path, but leave out the drive letter. # Normalize the POSIX-like part of the path, but leave out the drive letter.
non_drive_path = str_path[2:] non_drive_path = str_path[2:]
normalized = os.path.normpath(non_drive_path) normalized = os.path.normpath(non_drive_path)
@ -203,7 +211,12 @@ def strip_root(path: pathlib.PurePath) -> pathlib.PurePosixPath:
# This happens when running on POSIX but still handling paths # This happens when running on POSIX but still handling paths
# originating from a Windows machine. # originating from a Windows machine.
parts = path.parts parts = path.parts
if parts and len(parts[0]) == 2 and parts[0][0].isalpha() and parts[0][1] == ':': if (
parts
and len(parts[0]) == 2
and parts[0][0].isalpha()
and parts[0][1] == ":"
):
# The first part is a drive letter. # The first part is a drive letter.
return pathlib.PurePosixPath(parts[0][0], *path.parts[1:]) return pathlib.PurePosixPath(parts[0][0], *path.parts[1:])

6
blender_asset_tracer/cdefs.py
View File

@ -64,9 +64,9 @@ PTCACHE_EXTERNAL = 512
# BKE_pointcache.h # BKE_pointcache.h
PTCACHE_FILE_PTCACHE = 0 PTCACHE_FILE_PTCACHE = 0
PTCACHE_FILE_OPENVDB = 1 PTCACHE_FILE_OPENVDB = 1
PTCACHE_EXT = b'.bphys' PTCACHE_EXT = b".bphys"
PTCACHE_EXT_VDB = b'.vdb' PTCACHE_EXT_VDB = b".vdb"
PTCACHE_PATH = b'blendcache_' PTCACHE_PATH = b"blendcache_"
# BKE_node.h # BKE_node.h
SH_NODE_TEX_IMAGE = 143 SH_NODE_TEX_IMAGE = 143

81
blender_asset_tracer/cli/__init__.py
View File

@ -29,27 +29,45 @@ from . import blocks, common, pack, list_deps
def cli_main(): def cli_main():
from blender_asset_tracer import __version__ from blender_asset_tracer import __version__
parser = argparse.ArgumentParser(description='BAT: Blender Asset Tracer v%s' % __version__)
common.add_flag(parser, 'profile', help='Run the profiler, write to bam.prof') parser = argparse.ArgumentParser(
description="BAT: Blender Asset Tracer v%s" % __version__
)
common.add_flag(parser, "profile", help="Run the profiler, write to bam.prof")
# func is set by subparsers to indicate which function to run. # func is set by subparsers to indicate which function to run.
parser.set_defaults(func=None, parser.set_defaults(func=None, loglevel=logging.WARNING)
loglevel=logging.WARNING)
loggroup = parser.add_mutually_exclusive_group() loggroup = parser.add_mutually_exclusive_group()
loggroup.add_argument('-v', '--verbose', dest='loglevel', loggroup.add_argument(
action='store_const', const=logging.INFO, "-v",
help='Log INFO level and higher') "--verbose",
loggroup.add_argument('-d', '--debug', dest='loglevel', dest="loglevel",
action='store_const', const=logging.DEBUG, action="store_const",
help='Log everything') const=logging.INFO,
loggroup.add_argument('-q', '--quiet', dest='loglevel', help="Log INFO level and higher",
action='store_const', const=logging.ERROR, )
help='Log at ERROR level and higher') loggroup.add_argument(
"-d",
"--debug",
dest="loglevel",
action="store_const",
const=logging.DEBUG,
help="Log everything",
)
loggroup.add_argument(
"-q",
"--quiet",
dest="loglevel",
action="store_const",
const=logging.ERROR,
help="Log at ERROR level and higher",
)
subparsers = parser.add_subparsers( subparsers = parser.add_subparsers(
help='Choose a subcommand to actually make BAT do something. ' help="Choose a subcommand to actually make BAT do something. "
'Global options go before the subcommand, ' "Global options go before the subcommand, "
'whereas subcommand-specific options go after it. ' "whereas subcommand-specific options go after it. "
'Use --help after the subcommand to get more info.') "Use --help after the subcommand to get more info."
)
blocks.add_parser(subparsers) blocks.add_parser(subparsers)
pack.add_parser(subparsers) pack.add_parser(subparsers)
@ -59,32 +77,35 @@ def cli_main():
config_logging(args) config_logging(args)
from blender_asset_tracer import __version__ from blender_asset_tracer import __version__
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
# Make sure the things we log in our local logger are visible # Make sure the things we log in our local logger are visible
if args.profile and args.loglevel > logging.INFO: if args.profile and args.loglevel > logging.INFO:
log.setLevel(logging.INFO) log.setLevel(logging.INFO)
log.debug('Running BAT version %s', __version__) log.debug("Running BAT version %s", __version__)
if not args.func: if not args.func:
parser.error('No subcommand was given') parser.error("No subcommand was given")
start_time = time.time() start_time = time.time()
if args.profile: if args.profile:
import cProfile import cProfile
prof_fname = 'bam.prof' prof_fname = "bam.prof"
log.info('Running profiler') log.info("Running profiler")
cProfile.runctx('args.func(args)', cProfile.runctx(
globals=globals(), "args.func(args)", globals=globals(), locals=locals(), filename=prof_fname
locals=locals(), )
filename=prof_fname) log.info("Profiler exported data to %s", prof_fname)
log.info('Profiler exported data to %s', prof_fname) log.info(
log.info('Run "pyprof2calltree -i %r -k" to convert and open in KCacheGrind', prof_fname) 'Run "pyprof2calltree -i %r -k" to convert and open in KCacheGrind',
prof_fname,
)
else: else:
retval = args.func(args) retval = args.func(args)
duration = datetime.timedelta(seconds=time.time() - start_time) duration = datetime.timedelta(seconds=time.time() - start_time)
log.info('Command took %s to complete', duration) log.info("Command took %s to complete", duration)
def config_logging(args): def config_logging(args):
@ -92,8 +113,8 @@ def config_logging(args):
logging.basicConfig( logging.basicConfig(
level=logging.WARNING, level=logging.WARNING,
format='%(asctime)-15s %(levelname)8s %(name)-40s %(message)s', format="%(asctime)-15s %(levelname)8s %(name)-40s %(message)s",
) )
# Only set the log level on our own logger. Otherwise # Only set the log level on our own logger. Otherwise
# debug logging will be completely swamped. # debug logging will be completely swamped.
logging.getLogger('blender_asset_tracer').setLevel(args.loglevel) logging.getLogger("blender_asset_tracer").setLevel(args.loglevel)

93
blender_asset_tracer/cli/blocks.py
View File

@ -34,19 +34,29 @@ class BlockTypeInfo:
self.num_blocks = 0 self.num_blocks = 0
self.sizes = [] self.sizes = []
self.blocks = [] self.blocks = []
self.name = 'unset' self.name = "unset"
def add_parser(subparsers): def add_parser(subparsers):
"""Add argparser for this subcommand.""" """Add argparser for this subcommand."""
parser = subparsers.add_parser('blocks', help=__doc__) parser = subparsers.add_parser("blocks", help=__doc__)
parser.set_defaults(func=cli_blocks) parser.set_defaults(func=cli_blocks)
parser.add_argument('blendfile', type=pathlib.Path) parser.add_argument("blendfile", type=pathlib.Path)
parser.add_argument('-d', '--dump', default=False, action='store_true', parser.add_argument(
help='Hex-dump the biggest block') "-d",
parser.add_argument('-l', '--limit', default=10, type=int, "--dump",
help='Limit the number of DNA types shown, default is 10') default=False,
action="store_true",
help="Hex-dump the biggest block",
)
parser.add_argument(
"-l",
"--limit",
default=10,
type=int,
help="Limit the number of DNA types shown, default is 10",
)
def by_total_bytes(info: BlockTypeInfo) -> int: def by_total_bytes(info: BlockTypeInfo) -> int:
@ -54,23 +64,23 @@ def by_total_bytes(info: BlockTypeInfo) -> int:
def block_key(block: blendfile.BlendFileBlock) -> str: def block_key(block: blendfile.BlendFileBlock) -> str:
return '%s-%s' % (block.dna_type_name, block.code.decode()) return "%s-%s" % (block.dna_type_name, block.code.decode())
def cli_blocks(args): def cli_blocks(args):
bpath = args.blendfile bpath = args.blendfile
if not bpath.exists(): if not bpath.exists():
log.fatal('File %s does not exist', args.blendfile) log.fatal("File %s does not exist", args.blendfile)
return 3 return 3
per_blocktype = collections.defaultdict(BlockTypeInfo) per_blocktype = collections.defaultdict(BlockTypeInfo)
print('Opening %s' % bpath) print("Opening %s" % bpath)
bfile = blendfile.BlendFile(bpath) bfile = blendfile.BlendFile(bpath)
print('Inspecting %s' % bpath) print("Inspecting %s" % bpath)
for block in bfile.blocks: for block in bfile.blocks:
if block.code == b'DNA1': if block.code == b"DNA1":
continue continue
index_as = block_key(block) index_as = block_key(block)
@ -81,49 +91,58 @@ def cli_blocks(args):
info.sizes.append(block.size) info.sizes.append(block.size)
info.blocks.append(block) info.blocks.append(block)
fmt = '%-35s %10s %10s %10s %10s' fmt = "%-35s %10s %10s %10s %10s"
print(fmt % ('Block type', 'Total Size', 'Num blocks', 'Avg Size', 'Median')) print(fmt % ("Block type", "Total Size", "Num blocks", "Avg Size", "Median"))
print(fmt % (35 * '-', 10 * '-', 10 * '-', 10 * '-', 10 * '-')) print(fmt % (35 * "-", 10 * "-", 10 * "-", 10 * "-", 10 * "-"))
infos = sorted(per_blocktype.values(), key=by_total_bytes, reverse=True) infos = sorted(per_blocktype.values(), key=by_total_bytes, reverse=True)
for info in infos[:args.limit]: for info in infos[: args.limit]:
median_size = sorted(info.sizes)[len(info.sizes) // 2] median_size = sorted(info.sizes)[len(info.sizes) // 2]
print(fmt % (info.name, print(
common.humanize_bytes(info.total_bytes), fmt
info.num_blocks, % (
common.humanize_bytes(info.total_bytes // info.num_blocks), info.name,
common.humanize_bytes(median_size) common.humanize_bytes(info.total_bytes),
)) info.num_blocks,
common.humanize_bytes(info.total_bytes // info.num_blocks),
common.humanize_bytes(median_size),
)
)
print(70 * '-') print(70 * "-")
# From the blocks of the most space-using category, the biggest block. # From the blocks of the most space-using category, the biggest block.
biggest_block = sorted(infos[0].blocks, biggest_block = sorted(infos[0].blocks, key=lambda blck: blck.size, reverse=True)[0]
key=lambda blck: blck.size, print(
reverse=True)[0] "Biggest %s block is %s at address %s"
print('Biggest %s block is %s at address %s' % ( % (
block_key(biggest_block), block_key(biggest_block),
common.humanize_bytes(biggest_block.size), common.humanize_bytes(biggest_block.size),
biggest_block.addr_old, biggest_block.addr_old,
)) )
)
print('Finding what points there') print("Finding what points there")
addr_to_find = biggest_block.addr_old addr_to_find = biggest_block.addr_old
found_pointer = False found_pointer = False
for block in bfile.blocks: for block in bfile.blocks:
for prop_path, prop_value in block.items_recursive(): for prop_path, prop_value in block.items_recursive():
if not isinstance(prop_value, int) or prop_value != addr_to_find: if not isinstance(prop_value, int) or prop_value != addr_to_find:
continue continue
print(' ', block, prop_path) print(" ", block, prop_path)
found_pointer = True found_pointer = True
if not found_pointer: if not found_pointer:
print('Nothing points there') print("Nothing points there")
if args.dump: if args.dump:
print('Hexdump:') print("Hexdump:")
bfile.fileobj.seek(biggest_block.file_offset) bfile.fileobj.seek(biggest_block.file_offset)
data = bfile.fileobj.read(biggest_block.size) data = bfile.fileobj.read(biggest_block.size)
line_len_bytes = 32 line_len_bytes = 32
import codecs import codecs
for offset in range(0, len(data), line_len_bytes): for offset in range(0, len(data), line_len_bytes):
line = codecs.encode(data[offset:offset + line_len_bytes], 'hex').decode() line = codecs.encode(data[offset : offset + line_len_bytes], "hex").decode()
print('%6d -' % offset, ' '.join(line[i:i + 2] for i in range(0, len(line), 2))) print(
"%6d -" % offset,
" ".join(line[i : i + 2] for i in range(0, len(line), 2)),
)

33
blender_asset_tracer/cli/common.py
View File

@ -29,11 +29,13 @@ def add_flag(argparser, flag_name: str, **kwargs):
The flag defaults to False, and when present on the CLI stores True. The flag defaults to False, and when present on the CLI stores True.
""" """
argparser.add_argument('-%s' % flag_name[0], argparser.add_argument(
'--%s' % flag_name, "-%s" % flag_name[0],
default=False, "--%s" % flag_name,
action='store_true', default=False,
**kwargs) action="store_true",
**kwargs
)
def shorten(cwd: pathlib.Path, somepath: pathlib.Path) -> pathlib.Path: def shorten(cwd: pathlib.Path, somepath: pathlib.Path) -> pathlib.Path:
@ -44,7 +46,7 @@ def shorten(cwd: pathlib.Path, somepath: pathlib.Path) -> pathlib.Path:
return somepath return somepath
def humanize_bytes(size_in_bytes: int, precision: typing.Optional[int]=None): def humanize_bytes(size_in_bytes: int, precision: typing.Optional[int] = None):
"""Return a humanized string representation of a number of bytes. """Return a humanized string representation of a number of bytes.
Source: http://code.activestate.com/recipes/577081-humanized-representation-of-a-number-of-bytes Source: http://code.activestate.com/recipes/577081-humanized-representation-of-a-number-of-bytes
@ -78,22 +80,23 @@ def humanize_bytes(size_in_bytes: int, precision: typing.Optional[int]=None):
precision = size_in_bytes >= 1024 precision = size_in_bytes >= 1024
abbrevs = ( abbrevs = (
(1 << 50, 'PB'), (1 << 50, "PB"),
(1 << 40, 'TB'), (1 << 40, "TB"),
(1 << 30, 'GB'), (1 << 30, "GB"),
(1 << 20, 'MB'), (1 << 20, "MB"),
(1 << 10, 'kB'), (1 << 10, "kB"),
(1, 'B') (1, "B"),
) )
for factor, suffix in abbrevs: for factor, suffix in abbrevs:
if size_in_bytes >= factor: if size_in_bytes >= factor:
break break
else: else:
factor = 1 factor = 1
suffix = 'B' suffix = "B"
return '%.*f %s' % (precision, size_in_bytes / factor, suffix) return "%.*f %s" % (precision, size_in_bytes / factor, suffix)
if __name__ == '__main__': if __name__ == "__main__":
import doctest import doctest
doctest.testmod() doctest.testmod()

43
blender_asset_tracer/cli/list_deps.py
View File

@ -36,27 +36,32 @@ log = logging.getLogger(__name__)
def add_parser(subparsers): def add_parser(subparsers):
"""Add argparser for this subcommand.""" """Add argparser for this subcommand."""
parser = subparsers.add_parser('list', help=__doc__) parser = subparsers.add_parser("list", help=__doc__)
parser.set_defaults(func=cli_list) parser.set_defaults(func=cli_list)
parser.add_argument('blendfile', type=pathlib.Path) parser.add_argument("blendfile", type=pathlib.Path)
common.add_flag(parser, 'json', help='Output as JSON instead of human-readable text') common.add_flag(
common.add_flag(parser, 'sha256', parser, "json", help="Output as JSON instead of human-readable text"
help='Include SHA256sums in the output. Note that those may differ from the ' )
'SHA256sums in a BAT-pack when paths are rewritten.') common.add_flag(
common.add_flag(parser, 'timing', help='Include timing information in the output') parser,
"sha256",
help="Include SHA256sums in the output. Note that those may differ from the "
"SHA256sums in a BAT-pack when paths are rewritten.",
)
common.add_flag(parser, "timing", help="Include timing information in the output")
def cli_list(args): def cli_list(args):
bpath = args.blendfile bpath = args.blendfile
if not bpath.exists(): if not bpath.exists():
log.fatal('File %s does not exist', args.blendfile) log.fatal("File %s does not exist", args.blendfile)
return 3 return 3
if args.json: if args.json:
if args.sha256: if args.sha256:
log.fatal('--sha256 can currently not be used in combination with --json') log.fatal("--sha256 can currently not be used in combination with --json")
if args.timing: if args.timing:
log.fatal('--timing can currently not be used in combination with --json') log.fatal("--timing can currently not be used in combination with --json")
report_json(bpath) report_json(bpath)
else: else:
report_text(bpath, include_sha256=args.sha256, show_timing=args.timing) report_text(bpath, include_sha256=args.sha256, show_timing=args.timing)
@ -66,13 +71,13 @@ def calc_sha_sum(filepath: pathlib.Path) -> typing.Tuple[str, float]:
start = time.time() start = time.time()
if filepath.is_dir(): if filepath.is_dir():
for subfile in filepath.rglob('*'): for subfile in filepath.rglob("*"):
calc_sha_sum(subfile) calc_sha_sum(subfile)
duration = time.time() - start duration = time.time() - start
return '-multiple-', duration return "-multiple-", duration
summer = hashlib.sha256() summer = hashlib.sha256()
with filepath.open('rb') as infile: with filepath.open("rb") as infile:
while True: while True:
block = infile.read(32 * 1024) block = infile.read(32 * 1024)
if not block: if not block:
@ -108,24 +113,24 @@ def report_text(bpath, *, include_sha256: bool, show_timing: bool):
for assetpath in usage.files(): for assetpath in usage.files():
assetpath = bpathlib.make_absolute(assetpath) assetpath = bpathlib.make_absolute(assetpath)
if assetpath in reported_assets: if assetpath in reported_assets:
log.debug('Already reported %s', assetpath) log.debug("Already reported %s", assetpath)
continue continue
if include_sha256: if include_sha256:
shasum, time_spent = calc_sha_sum(assetpath) shasum, time_spent = calc_sha_sum(assetpath)
time_spent_on_shasums += time_spent time_spent_on_shasums += time_spent
print(' ', shorten(assetpath), shasum) print(" ", shorten(assetpath), shasum)
else: else:
print(' ', shorten(assetpath)) print(" ", shorten(assetpath))
reported_assets.add(assetpath) reported_assets.add(assetpath)
if show_timing: if show_timing:
duration = time.time() - start_time duration = time.time() - start_time
print('Spent %.2f seconds on producing this listing' % duration) print("Spent %.2f seconds on producing this listing" % duration)
if include_sha256: if include_sha256:
print('Spent %.2f seconds on calculating SHA sums' % time_spent_on_shasums) print("Spent %.2f seconds on calculating SHA sums" % time_spent_on_shasums)
percentage = time_spent_on_shasums / duration * 100 percentage = time_spent_on_shasums / duration * 100
print(' (that is %d%% of the total time' % percentage) print(" (that is %d%% of the total time" % percentage)
class JSONSerialiser(json.JSONEncoder): class JSONSerialiser(json.JSONEncoder):

188
blender_asset_tracer/cli/pack.py
View File

@ -32,36 +32,62 @@ log = logging.getLogger(__name__)
def add_parser(subparsers): def add_parser(subparsers):
"""Add argparser for this subcommand.""" """Add argparser for this subcommand."""
parser = subparsers.add_parser('pack', help=__doc__) parser = subparsers.add_parser("pack", help=__doc__)
parser.set_defaults(func=cli_pack) parser.set_defaults(func=cli_pack)
parser.add_argument('blendfile', type=pathlib.Path, parser.add_argument("blendfile", type=pathlib.Path, help="The Blend file to pack.")
help='The Blend file to pack.') parser.add_argument(
parser.add_argument('target', type=str, "target",
help="The target can be a directory, a ZIP file (does not have to exist " type=str,
"yet, just use 'something.zip' as target), " help="The target can be a directory, a ZIP file (does not have to exist "
"or a URL of S3 storage (s3://endpoint/path) " "yet, just use 'something.zip' as target), "
"or Shaman storage (shaman://endpoint/#checkoutID).") "or a URL of S3 storage (s3://endpoint/path) "
"or Shaman storage (shaman://endpoint/#checkoutID).",
)
parser.add_argument('-p', '--project', type=pathlib.Path, parser.add_argument(
help='Root directory of your project. Paths to below this directory are ' "-p",
'kept in the BAT Pack as well, whereas references to assets from ' "--project",
'outside this directory will have to be rewitten. The blend file MUST ' type=pathlib.Path,
'be inside the project directory. If this option is ommitted, the ' help="Root directory of your project. Paths to below this directory are "
'directory containing the blend file is taken as the project ' "kept in the BAT Pack as well, whereas references to assets from "
'directoy.') "outside this directory will have to be rewitten. The blend file MUST "
parser.add_argument('-n', '--noop', default=False, action='store_true', "be inside the project directory. If this option is ommitted, the "
help="Don't copy files, just show what would be done.") "directory containing the blend file is taken as the project "
parser.add_argument('-e', '--exclude', nargs='*', default='', "directoy.",
help="Space-separated list of glob patterns (like '*.abc *.vbo') to " )
"exclude.") parser.add_argument(
parser.add_argument('-c', '--compress', default=False, action='store_true', "-n",
help='Compress blend files while copying. This option is only valid when ' "--noop",
'packing into a directory (contrary to ZIP file or S3 upload). ' default=False,
'Note that files will NOT be compressed when the destination file ' action="store_true",
'already exists and has the same size as the original file.') help="Don't copy files, just show what would be done.",
parser.add_argument('-r', '--relative-only', default=False, action='store_true', )
help='Only pack assets that are referred to with a relative path (e.g. ' parser.add_argument(
'starting with `//`.') "-e",
"--exclude",
nargs="*",
default="",
help="Space-separated list of glob patterns (like '*.abc *.vbo') to "
"exclude.",
)
parser.add_argument(
"-c",
"--compress",
default=False,
action="store_true",
help="Compress blend files while copying. This option is only valid when "
"packing into a directory (contrary to ZIP file or S3 upload). "
"Note that files will NOT be compressed when the destination file "
"already exists and has the same size as the original file.",
)
parser.add_argument(
"-r",
"--relative-only",
default=False,
action="store_true",
help="Only pack assets that are referred to with a relative path (e.g. "
"starting with `//`.",
)
def cli_pack(args): def cli_pack(args):
@ -72,55 +98,70 @@ def cli_pack(args):
try: try:
packer.execute() packer.execute()
except blender_asset_tracer.pack.transfer.FileTransferError as ex: except blender_asset_tracer.pack.transfer.FileTransferError as ex:
log.error("%d files couldn't be copied, starting with %s", log.error(
len(ex.files_remaining), ex.files_remaining[0]) "%d files couldn't be copied, starting with %s",
len(ex.files_remaining),
ex.files_remaining[0],
)
raise SystemExit(1) raise SystemExit(1)
def create_packer(args, bpath: pathlib.Path, ppath: pathlib.Path, target: str) -> pack.Packer: def create_packer(
if target.startswith('s3:/'): args, bpath: pathlib.Path, ppath: pathlib.Path, target: str
) -> pack.Packer:
if target.startswith("s3:/"):
if args.noop: if args.noop:
raise ValueError('S3 uploader does not support no-op.') raise ValueError("S3 uploader does not support no-op.")
if args.compress: if args.compress:
raise ValueError('S3 uploader does not support on-the-fly compression') raise ValueError("S3 uploader does not support on-the-fly compression")
if args.relative_only: if args.relative_only:
raise ValueError('S3 uploader does not support the --relative-only option') raise ValueError("S3 uploader does not support the --relative-only option")
packer = create_s3packer(bpath, ppath, pathlib.PurePosixPath(target)) packer = create_s3packer(bpath, ppath, pathlib.PurePosixPath(target))
elif target.startswith('shaman+http:/') or target.startswith('shaman+https:/') \ elif (
or target.startswith('shaman:/'): target.startswith("shaman+http:/")
or target.startswith("shaman+https:/")
or target.startswith("shaman:/")
):
if args.noop: if args.noop:
raise ValueError('Shaman uploader does not support no-op.') raise ValueError("Shaman uploader does not support no-op.")
if args.compress: if args.compress:
raise ValueError('Shaman uploader does not support on-the-fly compression') raise ValueError("Shaman uploader does not support on-the-fly compression")
if args.relative_only: if args.relative_only:
raise ValueError('Shaman uploader does not support the --relative-only option') raise ValueError(
"Shaman uploader does not support the --relative-only option"
)
packer = create_shamanpacker(bpath, ppath, target) packer = create_shamanpacker(bpath, ppath, target)
elif target.lower().endswith('.zip'): elif target.lower().endswith(".zip"):
from blender_asset_tracer.pack import zipped from blender_asset_tracer.pack import zipped
if args.compress: if args.compress:
raise ValueError('ZIP packer does not support on-the-fly compression') raise ValueError("ZIP packer does not support on-the-fly compression")
packer = zipped.ZipPacker(bpath, ppath, target, noop=args.noop, packer = zipped.ZipPacker(
relative_only=args.relative_only) bpath, ppath, target, noop=args.noop, relative_only=args.relative_only
)
else: else:
packer = pack.Packer(bpath, ppath, target, noop=args.noop, packer = pack.Packer(
compress=args.compress, relative_only=args.relative_only) bpath,
ppath,
target,
noop=args.noop,
compress=args.compress,
relative_only=args.relative_only,
)
if args.exclude: if args.exclude:
# args.exclude is a list, due to nargs='*', so we have to split and flatten. # args.exclude is a list, due to nargs='*', so we have to split and flatten.
globs = [glob globs = [glob for globs in args.exclude for glob in globs.split()]
for globs in args.exclude log.info("Excluding: %s", ", ".join(repr(g) for g in globs))
for glob in globs.split()]
log.info('Excluding: %s', ', '.join(repr(g) for g in globs))
packer.exclude(*globs) packer.exclude(*globs)
return packer return packer
@ -130,14 +171,16 @@ def create_s3packer(bpath, ppath, tpath) -> pack.Packer:
# Split the target path into 's3:/', hostname, and actual target path # Split the target path into 's3:/', hostname, and actual target path
parts = tpath.parts parts = tpath.parts
endpoint = 'https://%s/' % parts[1] endpoint = "https://%s/" % parts[1]
tpath = pathlib.Path(*tpath.parts[2:]) tpath = pathlib.Path(*tpath.parts[2:])
log.info('Uploading to S3-compatible storage %s at %s', endpoint, tpath) log.info("Uploading to S3-compatible storage %s at %s", endpoint, tpath)
return s3.S3Packer(bpath, ppath, tpath, endpoint=endpoint) return s3.S3Packer(bpath, ppath, tpath, endpoint=endpoint)
def create_shamanpacker(bpath: pathlib.Path, ppath: pathlib.Path, tpath: str) -> pack.Packer: def create_shamanpacker(
bpath: pathlib.Path, ppath: pathlib.Path, tpath: str
) -> pack.Packer:
"""Creates a package for sending files to a Shaman server. """Creates a package for sending files to a Shaman server.
URLs should have the form: URLs should have the form:
@ -149,11 +192,15 @@ def create_shamanpacker(bpath: pathlib.Path, ppath: pathlib.Path, tpath: str) ->
endpoint, checkout_id = shaman.parse_endpoint(tpath) endpoint, checkout_id = shaman.parse_endpoint(tpath)
if not checkout_id: if not checkout_id:
log.warning('No checkout ID given on the URL. Going to send BAT pack to Shaman, ' log.warning(
'but NOT creating a checkout') "No checkout ID given on the URL. Going to send BAT pack to Shaman, "
"but NOT creating a checkout"
)
log.info('Uploading to Shaman server %s with job %s', endpoint, checkout_id) log.info("Uploading to Shaman server %s with job %s", endpoint, checkout_id)
return shaman.ShamanPacker(bpath, ppath, '/', endpoint=endpoint, checkout_id=checkout_id) return shaman.ShamanPacker(
bpath, ppath, "/", endpoint=endpoint, checkout_id=checkout_id
)
def paths_from_cli(args) -> typing.Tuple[pathlib.Path, pathlib.Path, str]: def paths_from_cli(args) -> typing.Tuple[pathlib.Path, pathlib.Path, str]:
@ -163,10 +210,10 @@ def paths_from_cli(args) -> typing.Tuple[pathlib.Path, pathlib.Path, str]:
""" """
bpath = args.blendfile bpath = args.blendfile
if not bpath.exists(): if not bpath.exists():
log.critical('File %s does not exist', bpath) log.critical("File %s does not exist", bpath)
sys.exit(3) sys.exit(3)
if bpath.is_dir(): if bpath.is_dir():
log.critical('%s is a directory, should be a blend file') log.critical("%s is a directory, should be a blend file")
sys.exit(3) sys.exit(3)
bpath = bpathlib.make_absolute(bpath) bpath = bpathlib.make_absolute(bpath)
@ -174,27 +221,34 @@ def paths_from_cli(args) -> typing.Tuple[pathlib.Path, pathlib.Path, str]:
if args.project is None: if args.project is None:
ppath = bpathlib.make_absolute(bpath).parent ppath = bpathlib.make_absolute(bpath).parent
log.warning('No project path given, using %s', ppath) log.warning("No project path given, using %s", ppath)
else: else:
ppath = bpathlib.make_absolute(args.project) ppath = bpathlib.make_absolute(args.project)
if not ppath.exists(): if not ppath.exists():
log.critical('Project directory %s does not exist', ppath) log.critical("Project directory %s does not exist", ppath)
sys.exit(5) sys.exit(5)
if not ppath.is_dir(): if not ppath.is_dir():
log.warning('Project path %s is not a directory; using the parent %s', ppath, ppath.parent) log.warning(
"Project path %s is not a directory; using the parent %s",
ppath,
ppath.parent,
)
ppath = ppath.parent ppath = ppath.parent
try: try:
bpath.relative_to(ppath) bpath.relative_to(ppath)
except ValueError: except ValueError:
log.critical('Project directory %s does not contain blend file %s', log.critical(
args.project, bpath.absolute()) "Project directory %s does not contain blend file %s",
args.project,
bpath.absolute(),
)
sys.exit(5) sys.exit(5)
log.info('Blend file to pack: %s', bpath) log.info("Blend file to pack: %s", bpath)
log.info('Project path: %s', ppath) log.info("Project path: %s", ppath)
log.info('Pack will be created in: %s', tpath) log.info("Pack will be created in: %s", tpath)
return bpath, ppath, tpath return bpath, ppath, tpath

35
blender_asset_tracer/compressor.py
View File

@ -16,10 +16,10 @@ def move(src: pathlib.Path, dest: pathlib.Path):
Only compresses files ending in .blend; others are moved as-is. Only compresses files ending in .blend; others are moved as-is.
""" """
my_log = log.getChild('move') my_log = log.getChild("move")
my_log.debug('Moving %s to %s', src, dest) my_log.debug("Moving %s to %s", src, dest)
if src.suffix.lower() == '.blend': if src.suffix.lower() == ".blend":
_move_or_copy(src, dest, my_log, source_must_remain=False) _move_or_copy(src, dest, my_log, source_must_remain=False)
else: else:
shutil.move(str(src), str(dest)) shutil.move(str(src), str(dest))
@ -30,19 +30,22 @@ def copy(src: pathlib.Path, dest: pathlib.Path):
Only compresses files ending in .blend; others are copied as-is. Only compresses files ending in .blend; others are copied as-is.
""" """
my_log = log.getChild('copy') my_log = log.getChild("copy")
my_log.debug('Copying %s to %s', src, dest) my_log.debug("Copying %s to %s", src, dest)
if src.suffix.lower() == '.blend': if src.suffix.lower() == ".blend":
_move_or_copy(src, dest, my_log, source_must_remain=True) _move_or_copy(src, dest, my_log, source_must_remain=True)
else: else:
shutil.copy2(str(src), str(dest)) shutil.copy2(str(src), str(dest))
def _move_or_copy(src: pathlib.Path, dest: pathlib.Path, def _move_or_copy(
my_log: logging.Logger, src: pathlib.Path,
*, dest: pathlib.Path,
source_must_remain: bool): my_log: logging.Logger,
*,
source_must_remain: bool
):
"""Either move or copy a file, gzip-compressing if not compressed yet. """Either move or copy a file, gzip-compressing if not compressed yet.
:param src: File to copy/move. :param src: File to copy/move.
@ -50,27 +53,27 @@ def _move_or_copy(src: pathlib.Path, dest: pathlib.Path,
:source_must_remain: True to copy, False to move. :source_must_remain: True to copy, False to move.
:my_log: Logger to use for logging. :my_log: Logger to use for logging.
""" """
srcfile = src.open('rb') srcfile = src.open("rb")
try: try:
first_bytes = srcfile.read(2) first_bytes = srcfile.read(2)
if first_bytes == b'\x1f\x8b': if first_bytes == b"\x1f\x8b":
# Already a gzipped file. # Already a gzipped file.
srcfile.close() srcfile.close()
my_log.debug('Source file %s is GZipped already', src) my_log.debug("Source file %s is GZipped already", src)
if source_must_remain: if source_must_remain:
shutil.copy2(str(src), str(dest)) shutil.copy2(str(src), str(dest))
else: else:
shutil.move(str(src), str(dest)) shutil.move(str(src), str(dest))
return return
my_log.debug('Compressing %s on the fly while copying to %s', src, dest) my_log.debug("Compressing %s on the fly while copying to %s", src, dest)
with gzip.open(str(dest), mode='wb') as destfile: with gzip.open(str(dest), mode="wb") as destfile:
destfile.write(first_bytes) destfile.write(first_bytes)
shutil.copyfileobj(srcfile, destfile, BLOCK_SIZE) shutil.copyfileobj(srcfile, destfile, BLOCK_SIZE)
srcfile.close() srcfile.close()
if not source_must_remain: if not source_must_remain:
my_log.debug('Deleting source file %s', src) my_log.debug("Deleting source file %s", src)
src.unlink() src.unlink()
finally: finally:
if not srcfile.closed: if not srcfile.closed:

167
blender_asset_tracer/pack/__init__.py
View File

@ -93,14 +93,16 @@ class Packer:
instance. instance.
""" """
def __init__(self, def __init__(
bfile: pathlib.Path, self,
project: pathlib.Path, bfile: pathlib.Path,
target: str, project: pathlib.Path,
*, target: str,
noop=False, *,
compress=False, noop=False,
relative_only=False) -> None: compress=False,
relative_only=False
) -> None:
self.blendfile = bfile self.blendfile = bfile
self.project = project self.project = project
self.target = target self.target = target
@ -110,7 +112,7 @@ class Packer:
self.relative_only = relative_only self.relative_only = relative_only
self._aborted = threading.Event() self._aborted = threading.Event()
self._abort_lock = threading.RLock() self._abort_lock = threading.RLock()
self._abort_reason = '' self._abort_reason = ""
# Set this to a custom Callback() subclass instance before calling # Set this to a custom Callback() subclass instance before calling
# strategise() to receive progress reports. # strategise() to receive progress reports.
@ -120,14 +122,16 @@ class Packer:
self._exclude_globs = set() # type: typing.Set[str] self._exclude_globs = set() # type: typing.Set[str]
from blender_asset_tracer.cli import common from blender_asset_tracer.cli import common
self._shorten = functools.partial(common.shorten, self.project) self._shorten = functools.partial(common.shorten, self.project)
if noop: if noop:
log.warning('Running in no-op mode, only showing what will be done.') log.warning("Running in no-op mode, only showing what will be done.")
# Filled by strategise() # Filled by strategise()
self._actions = collections.defaultdict(AssetAction) \ self._actions = collections.defaultdict(
# type: typing.DefaultDict[pathlib.Path, AssetAction] AssetAction
) # type: typing.DefaultDict[pathlib.Path, AssetAction]
self.missing_files = set() # type: typing.Set[pathlib.Path] self.missing_files = set() # type: typing.Set[pathlib.Path]
self._new_location_paths = set() # type: typing.Set[pathlib.Path] self._new_location_paths = set() # type: typing.Set[pathlib.Path]
self._output_path = None # type: typing.Optional[pathlib.PurePath] self._output_path = None # type: typing.Optional[pathlib.PurePath]
@ -138,7 +142,7 @@ class Packer:
# Number of files we would copy, if not for --noop # Number of files we would copy, if not for --noop
self._file_count = 0 self._file_count = 0
self._tmpdir = tempfile.TemporaryDirectory(prefix='bat-', suffix='-batpack') self._tmpdir = tempfile.TemporaryDirectory(prefix="bat-", suffix="-batpack")
self._rewrite_in = pathlib.Path(self._tmpdir.name) self._rewrite_in = pathlib.Path(self._tmpdir.name)
def _make_target_path(self, target: str) -> pathlib.PurePath: def _make_target_path(self, target: str) -> pathlib.PurePath:
@ -155,7 +159,7 @@ class Packer:
self._tscb.flush() self._tscb.flush()
self._tmpdir.cleanup() self._tmpdir.cleanup()
def __enter__(self) -> 'Packer': def __enter__(self) -> "Packer":
return self return self
def __exit__(self, exc_type, exc_val, exc_tb) -> None: def __exit__(self, exc_type, exc_val, exc_tb) -> None:
@ -177,7 +181,7 @@ class Packer:
self._progress_cb = new_progress_cb self._progress_cb = new_progress_cb
self._tscb = progress.ThreadSafeCallback(self._progress_cb) self._tscb = progress.ThreadSafeCallback(self._progress_cb)
def abort(self, reason='') -> None: def abort(self, reason="") -> None:
"""Aborts the current packing process. """Aborts the current packing process.
Can be called from any thread. Aborts as soon as the running strategise Can be called from any thread. Aborts as soon as the running strategise
@ -196,12 +200,12 @@ class Packer:
with self._abort_lock: with self._abort_lock:
reason = self._abort_reason reason = self._abort_reason
if self._file_transferer is not None and self._file_transferer.has_error: if self._file_transferer is not None and self._file_transferer.has_error:
log.error('A transfer error occurred') log.error("A transfer error occurred")
reason = self._file_transferer.error_message() reason = self._file_transferer.error_message()
elif not self._aborted.is_set(): elif not self._aborted.is_set():
return return
log.warning('Aborting') log.warning("Aborting")
self._tscb.flush() self._tscb.flush()
self._progress_cb.pack_aborted(reason) self._progress_cb.pack_aborted(reason)
raise Aborted(reason) raise Aborted(reason)
@ -212,8 +216,10 @@ class Packer:
Must be called before calling strategise(). Must be called before calling strategise().
""" """
if self._actions: if self._actions:
raise RuntimeError('%s.exclude() must be called before strategise()' % raise RuntimeError(
self.__class__.__qualname__) "%s.exclude() must be called before strategise()"
% self.__class__.__qualname__
)
self._exclude_globs.update(globs) self._exclude_globs.update(globs)
def strategise(self) -> None: def strategise(self) -> None:
@ -236,7 +242,9 @@ class Packer:
# network shares mapped to Windows drive letters back to their UNC # network shares mapped to Windows drive letters back to their UNC
# notation. Only resolving one but not the other (which can happen # notation. Only resolving one but not the other (which can happen
# with the abosolute() call above) can cause errors. # with the abosolute() call above) can cause errors.
bfile_pp = self._target_path / bfile_path.relative_to(bpathlib.make_absolute(self.project)) bfile_pp = self._target_path / bfile_path.relative_to(
bpathlib.make_absolute(self.project)
)
self._output_path = bfile_pp self._output_path = bfile_pp
self._progress_cb.pack_start() self._progress_cb.pack_start()
@ -251,11 +259,11 @@ class Packer:
self._check_aborted() self._check_aborted()
asset_path = usage.abspath asset_path = usage.abspath
if any(asset_path.match(glob) for glob in self._exclude_globs): if any(asset_path.match(glob) for glob in self._exclude_globs):
log.info('Excluding file: %s', asset_path) log.info("Excluding file: %s", asset_path)
continue continue
if self.relative_only and not usage.asset_path.startswith(b'//'): if self.relative_only and not usage.asset_path.startswith(b"//"):
log.info('Skipping absolute path: %s', usage.asset_path) log.info("Skipping absolute path: %s", usage.asset_path)
continue continue
if usage.is_sequence: if usage.is_sequence:
@ -269,14 +277,22 @@ class Packer:
def _visit_sequence(self, asset_path: pathlib.Path, usage: result.BlockUsage): def _visit_sequence(self, asset_path: pathlib.Path, usage: result.BlockUsage):
assert usage.is_sequence assert usage.is_sequence
for first_path in file_sequence.expand_sequence(asset_path): def handle_missing_file():
if first_path.exists(): log.warning("Missing file: %s", asset_path)
break
else:
# At least the first file of a sequence must exist.
log.warning('Missing file: %s', asset_path)
self.missing_files.add(asset_path) self.missing_files.add(asset_path)
self._progress_cb.missing_file(asset_path) self._progress_cb.missing_file(asset_path)
try:
for file_path in file_sequence.expand_sequence(asset_path):
if file_path.exists():
break
else:
# At least some file of a sequence must exist.
handle_missing_file()
return
except file_sequence.DoesNotExist:
# The asset path should point to something existing.
handle_missing_file()
return return
# Handle this sequence as an asset. # Handle this sequence as an asset.
@ -291,7 +307,7 @@ class Packer:
# Sequences are allowed to not exist at this point. # Sequences are allowed to not exist at this point.
if not usage.is_sequence and not asset_path.exists(): if not usage.is_sequence and not asset_path.exists():
log.warning('Missing file: %s', asset_path) log.warning("Missing file: %s", asset_path)
self.missing_files.add(asset_path) self.missing_files.add(asset_path)
self._progress_cb.missing_file(asset_path) self._progress_cb.missing_file(asset_path)
return return
@ -315,11 +331,11 @@ class Packer:
act.usages.append(usage) act.usages.append(usage)
if needs_rewriting: if needs_rewriting:
log.info('%s needs rewritten path to %s', bfile_path, usage.asset_path) log.info("%s needs rewritten path to %s", bfile_path, usage.asset_path)
act.path_action = PathAction.FIND_NEW_LOCATION act.path_action = PathAction.FIND_NEW_LOCATION
self._new_location_paths.add(asset_path) self._new_location_paths.add(asset_path)
else: else:
log.debug('%s can keep using %s', bfile_path, usage.asset_path) log.debug("%s can keep using %s", bfile_path, usage.asset_path)
asset_pp = self._target_path / asset_path.relative_to(self.project) asset_pp = self._target_path / asset_path.relative_to(self.project)
act.new_path = asset_pp act.new_path = asset_pp
@ -331,7 +347,7 @@ class Packer:
assert isinstance(act, AssetAction) assert isinstance(act, AssetAction)
relpath = bpathlib.strip_root(path) relpath = bpathlib.strip_root(path)
act.new_path = pathlib.Path(self._target_path, '_outside_project', relpath) act.new_path = pathlib.Path(self._target_path, "_outside_project", relpath)
def _group_rewrites(self) -> None: def _group_rewrites(self) -> None:
"""For each blend file, collect which fields need rewriting. """For each blend file, collect which fields need rewriting.
@ -370,7 +386,7 @@ class Packer:
def execute(self) -> None: def execute(self) -> None:
"""Execute the strategy.""" """Execute the strategy."""
assert self._actions, 'Run strategise() first' assert self._actions, "Run strategise() first"
if not self.noop: if not self.noop:
self._rewrite_paths() self._rewrite_paths()
@ -408,7 +424,7 @@ class Packer:
This creates the BAT Pack but does not yet do any path rewriting. This creates the BAT Pack but does not yet do any path rewriting.
""" """
log.debug('Executing %d copy actions', len(self._actions)) log.debug("Executing %d copy actions", len(self._actions))
assert self._file_transferer is not None assert self._file_transferer is not None
@ -418,12 +434,12 @@ class Packer:
self._copy_asset_and_deps(asset_path, action) self._copy_asset_and_deps(asset_path, action)
if self.noop: if self.noop:
log.info('Would copy %d files to %s', self._file_count, self.target) log.info("Would copy %d files to %s", self._file_count, self.target)
return return
self._file_transferer.done_and_join() self._file_transferer.done_and_join()
self._on_file_transfer_finished(file_transfer_completed=True) self._on_file_transfer_finished(file_transfer_completed=True)
except KeyboardInterrupt: except KeyboardInterrupt:
log.info('File transfer interrupted with Ctrl+C, aborting.') log.info("File transfer interrupted with Ctrl+C, aborting.")
self._file_transferer.abort_and_join() self._file_transferer.abort_and_join()
self._on_file_transfer_finished(file_transfer_completed=False) self._on_file_transfer_finished(file_transfer_completed=False)
raise raise
@ -465,18 +481,20 @@ class Packer:
# Use tempfile to create a unique name in our temporary directoy. # Use tempfile to create a unique name in our temporary directoy.
# The file should be deleted when self.close() is called, and not # The file should be deleted when self.close() is called, and not
# when the bfile_tp object is GC'd. # when the bfile_tp object is GC'd.
bfile_tmp = tempfile.NamedTemporaryFile(dir=str(self._rewrite_in), bfile_tmp = tempfile.NamedTemporaryFile(
prefix='bat-', dir=str(self._rewrite_in),
suffix='-' + bfile_path.name, prefix="bat-",
delete=False) suffix="-" + bfile_path.name,
delete=False,
)
bfile_tp = pathlib.Path(bfile_tmp.name) bfile_tp = pathlib.Path(bfile_tmp.name)
action.read_from = bfile_tp action.read_from = bfile_tp
log.info('Rewriting %s to %s', bfile_path, bfile_tp) log.info("Rewriting %s to %s", bfile_path, bfile_tp)
# The original blend file will have been cached, so we can use it # The original blend file will have been cached, so we can use it
# to avoid re-parsing all data blocks in the to-be-rewritten file. # to avoid re-parsing all data blocks in the to-be-rewritten file.
bfile = blendfile.open_cached(bfile_path, assert_cached=True) bfile = blendfile.open_cached(bfile_path, assert_cached=True)
bfile.copy_and_rebind(bfile_tp, mode='rb+') bfile.copy_and_rebind(bfile_tp, mode="rb+")
for usage in action.rewrites: for usage in action.rewrites:
self._check_aborted() self._check_aborted()
@ -484,25 +502,27 @@ class Packer:
asset_pp = self._actions[usage.abspath].new_path asset_pp = self._actions[usage.abspath].new_path
assert isinstance(asset_pp, pathlib.Path) assert isinstance(asset_pp, pathlib.Path)
log.debug(' - %s is packed at %s', usage.asset_path, asset_pp) log.debug(" - %s is packed at %s", usage.asset_path, asset_pp)
relpath = bpathlib.BlendPath.mkrelative(asset_pp, bfile_pp) relpath = bpathlib.BlendPath.mkrelative(asset_pp, bfile_pp)
if relpath == usage.asset_path: if relpath == usage.asset_path:
log.info(' - %s remained at %s', usage.asset_path, relpath) log.info(" - %s remained at %s", usage.asset_path, relpath)
continue continue
log.info(' - %s moved to %s', usage.asset_path, relpath) log.info(" - %s moved to %s", usage.asset_path, relpath)
# Find the same block in the newly copied file. # Find the same block in the newly copied file.
block = bfile.dereference_pointer(usage.block.addr_old) block = bfile.dereference_pointer(usage.block.addr_old)
if usage.path_full_field is None: if usage.path_full_field is None:
dir_field = usage.path_dir_field dir_field = usage.path_dir_field
assert dir_field is not None assert dir_field is not None
log.debug(' - updating field %s of block %s', log.debug(
dir_field.name.name_only, " - updating field %s of block %s",
block) dir_field.name.name_only,
block,
)
reldir = bpathlib.BlendPath.mkrelative(asset_pp.parent, bfile_pp) reldir = bpathlib.BlendPath.mkrelative(asset_pp.parent, bfile_pp)
written = block.set(dir_field.name.name_only, reldir) written = block.set(dir_field.name.name_only, reldir)
log.debug(' - written %d bytes', written) log.debug(" - written %d bytes", written)
# BIG FAT ASSUMPTION that the filename (e.g. basename # BIG FAT ASSUMPTION that the filename (e.g. basename
# without path) does not change. This makes things much # without path) does not change. This makes things much
@ -510,10 +530,13 @@ class Packer:
# filename fields are in different blocks. See the # filename fields are in different blocks. See the
# blocks2assets.scene() function for the implementation. # blocks2assets.scene() function for the implementation.
else: else:
log.debug(' - updating field %s of block %s', log.debug(
usage.path_full_field.name.name_only, block) " - updating field %s of block %s",
usage.path_full_field.name.name_only,
block,
)
written = block.set(usage.path_full_field.name.name_only, relpath) written = block.set(usage.path_full_field.name.name_only, relpath)
log.debug(' - written %d bytes', written) log.debug(" - written %d bytes", written)
# Make sure we close the file, otherwise changes may not be # Make sure we close the file, otherwise changes may not be
# flushed before it gets copied. # flushed before it gets copied.
@ -524,12 +547,13 @@ class Packer:
def _copy_asset_and_deps(self, asset_path: pathlib.Path, action: AssetAction): def _copy_asset_and_deps(self, asset_path: pathlib.Path, action: AssetAction):
# Copy the asset itself, but only if it's not a sequence (sequences are # Copy the asset itself, but only if it's not a sequence (sequences are
# handled below in the for-loop). # handled below in the for-loop).
if '*' not in str(asset_path): if "*" not in str(asset_path):
packed_path = action.new_path packed_path = action.new_path
assert packed_path is not None assert packed_path is not None
read_path = action.read_from or asset_path read_path = action.read_from or asset_path
self._send_to_target(read_path, packed_path, self._send_to_target(
may_move=action.read_from is not None) read_path, packed_path, may_move=action.read_from is not None
)
# Copy its sequence dependencies. # Copy its sequence dependencies.
for usage in action.usages: for usage in action.usages:
@ -541,7 +565,7 @@ class Packer:
# In case of globbing, we only support globbing by filename, # In case of globbing, we only support globbing by filename,
# and not by directory. # and not by directory.
assert '*' not in str(first_pp) or '*' in first_pp.name assert "*" not in str(first_pp) or "*" in first_pp.name
packed_base_dir = first_pp.parent packed_base_dir = first_pp.parent
for file_path in usage.files(): for file_path in usage.files():
@ -552,17 +576,16 @@ class Packer:
# Assumption: all data blocks using this asset use it the same way. # Assumption: all data blocks using this asset use it the same way.
break break
def _send_to_target(self, def _send_to_target(
asset_path: pathlib.Path, self, asset_path: pathlib.Path, target: pathlib.PurePath, may_move=False
target: pathlib.PurePath, ):
may_move=False):
if self.noop: if self.noop:
print('%s -> %s' % (asset_path, target)) print("%s -> %s" % (asset_path, target))
self._file_count += 1 self._file_count += 1
return return
verb = 'move' if may_move else 'copy' verb = "move" if may_move else "copy"
log.debug('Queueing %s of %s', verb, asset_path) log.debug("Queueing %s of %s", verb, asset_path)
self._tscb.flush() self._tscb.flush()
@ -575,13 +598,15 @@ class Packer:
def _write_info_file(self): def _write_info_file(self):
"""Write a little text file with info at the top of the pack.""" """Write a little text file with info at the top of the pack."""
infoname = 'pack-info.txt' infoname = "pack-info.txt"
infopath = self._rewrite_in / infoname infopath = self._rewrite_in / infoname
log.debug('Writing info to %s', infopath) log.debug("Writing info to %s", infopath)
with infopath.open('wt', encoding='utf8') as infofile: with infopath.open("wt", encoding="utf8") as infofile:
print('This is a Blender Asset Tracer pack.', file=infofile) print("This is a Blender Asset Tracer pack.", file=infofile)
print('Start by opening the following blend file:', file=infofile) print("Start by opening the following blend file:", file=infofile)
print(' %s' % self._output_path.relative_to(self._target_path).as_posix(), print(
file=infofile) " %s" % self._output_path.relative_to(self._target_path).as_posix(),
file=infofile,
)
self._file_transferer.queue_move(infopath, self._target_path / infoname) self._file_transferer.queue_move(infopath, self._target_path / infoname)

51
blender_asset_tracer/pack/filesystem.py
View File

@ -81,9 +81,9 @@ class FileCopier(transfer.FileTransferer):
# We have to catch exceptions in a broad way, as this is running in # We have to catch exceptions in a broad way, as this is running in
# a separate thread, and exceptions won't otherwise be seen. # a separate thread, and exceptions won't otherwise be seen.
if self._abort.is_set(): if self._abort.is_set():
log.debug('Error transferring %s to %s: %s', src, dst, ex) log.debug("Error transferring %s to %s: %s", src, dst, ex)
else: else:
msg = 'Error transferring %s to %s' % (src, dst) msg = "Error transferring %s to %s" % (src, dst)
log.exception(msg) log.exception(msg)
self.error_set(msg) self.error_set(msg)
# Put the files to copy back into the queue, and abort. This allows # Put the files to copy back into the queue, and abort. This allows
@ -93,16 +93,16 @@ class FileCopier(transfer.FileTransferer):
self.queue.put((src, dst, act), timeout=1.0) self.queue.put((src, dst, act), timeout=1.0)
break break
log.debug('All transfer threads queued') log.debug("All transfer threads queued")
pool.close() pool.close()
log.debug('Waiting for transfer threads to finish') log.debug("Waiting for transfer threads to finish")
pool.join() pool.join()
log.debug('All transfer threads finished') log.debug("All transfer threads finished")
if self.files_transferred: if self.files_transferred:
log.info('Transferred %d files', self.files_transferred) log.info("Transferred %d files", self.files_transferred)
if self.files_skipped: if self.files_skipped:
log.info('Skipped %d files', self.files_skipped) log.info("Skipped %d files", self.files_skipped)
def _thread(self, src: pathlib.Path, dst: pathlib.Path, act: transfer.Action): def _thread(self, src: pathlib.Path, dst: pathlib.Path, act: transfer.Action):
try: try:
@ -111,7 +111,7 @@ class FileCopier(transfer.FileTransferer):
if self.has_error or self._abort.is_set(): if self.has_error or self._abort.is_set():
raise AbortTransfer() raise AbortTransfer()
log.info('%s %s -> %s', act.name, src, dst) log.info("%s %s -> %s", act.name, src, dst)
tfunc(src, dst) tfunc(src, dst)
except AbortTransfer: except AbortTransfer:
# either self._error or self._abort is already set. We just have to # either self._error or self._abort is already set. We just have to
@ -121,9 +121,9 @@ class FileCopier(transfer.FileTransferer):
# We have to catch exceptions in a broad way, as this is running in # We have to catch exceptions in a broad way, as this is running in
# a separate thread, and exceptions won't otherwise be seen. # a separate thread, and exceptions won't otherwise be seen.
if self._abort.is_set(): if self._abort.is_set():
log.debug('Error transferring %s to %s: %s', src, dst, ex) log.debug("Error transferring %s to %s: %s", src, dst, ex)
else: else:
msg = 'Error transferring %s to %s' % (src, dst) msg = "Error transferring %s to %s" % (src, dst)
log.exception(msg) log.exception(msg)
self.error_set(msg) self.error_set(msg)
# Put the files to copy back into the queue, and abort. This allows # Put the files to copy back into the queue, and abort. This allows
@ -132,7 +132,9 @@ class FileCopier(transfer.FileTransferer):
# be reported there. # be reported there.
self.queue.put((src, dst, act), timeout=1.0) self.queue.put((src, dst, act), timeout=1.0)
def _skip_file(self, src: pathlib.Path, dst: pathlib.Path, act: transfer.Action) -> bool: def _skip_file(
self, src: pathlib.Path, dst: pathlib.Path, act: transfer.Action
) -> bool:
"""Skip this file (return True) or not (return False).""" """Skip this file (return True) or not (return False)."""
st_src = src.stat() # must exist, or it wouldn't be queued. st_src = src.stat() # must exist, or it wouldn't be queued.
if not dst.exists(): if not dst.exists():
@ -142,9 +144,9 @@ class FileCopier(transfer.FileTransferer):
if st_dst.st_size != st_src.st_size or st_dst.st_mtime < st_src.st_mtime: if st_dst.st_size != st_src.st_size or st_dst.st_mtime < st_src.st_mtime:
return False return False
log.info('SKIP %s; already exists', src) log.info("SKIP %s; already exists", src)
if act == transfer.Action.MOVE: if act == transfer.Action.MOVE:
log.debug('Deleting %s', src) log.debug("Deleting %s", src)
src.unlink() src.unlink()
self.files_skipped += 1 self.files_skipped += 1
return True return True
@ -171,19 +173,19 @@ class FileCopier(transfer.FileTransferer):
return return
if (srcpath, dstpath) in self.already_copied: if (srcpath, dstpath) in self.already_copied:
log.debug('SKIP %s; already copied', srcpath) log.debug("SKIP %s; already copied", srcpath)
return return
s_stat = srcpath.stat() # must exist, or it wouldn't be queued. s_stat = srcpath.stat() # must exist, or it wouldn't be queued.
if dstpath.exists(): if dstpath.exists():
d_stat = dstpath.stat() d_stat = dstpath.stat()
if d_stat.st_size == s_stat.st_size and d_stat.st_mtime >= s_stat.st_mtime: if d_stat.st_size == s_stat.st_size and d_stat.st_mtime >= s_stat.st_mtime:
log.info('SKIP %s; already exists', srcpath) log.info("SKIP %s; already exists", srcpath)
self.progress_cb.transfer_file_skipped(srcpath, dstpath) self.progress_cb.transfer_file_skipped(srcpath, dstpath)
self.files_skipped += 1 self.files_skipped += 1
return return
log.debug('Copying %s -> %s', srcpath, dstpath) log.debug("Copying %s -> %s", srcpath, dstpath)
self._copy(srcpath, dstpath) self._copy(srcpath, dstpath)
self.already_copied.add((srcpath, dstpath)) self.already_copied.add((srcpath, dstpath))
@ -191,8 +193,13 @@ class FileCopier(transfer.FileTransferer):
self.report_transferred(s_stat.st_size) self.report_transferred(s_stat.st_size)
def copytree(self, src: pathlib.Path, dst: pathlib.Path, def copytree(
symlinks=False, ignore_dangling_symlinks=False): self,
src: pathlib.Path,
dst: pathlib.Path,
symlinks=False,
ignore_dangling_symlinks=False,
):
"""Recursively copy a directory tree. """Recursively copy a directory tree.
Copy of shutil.copytree() with some changes: Copy of shutil.copytree() with some changes:
@ -204,7 +211,7 @@ class FileCopier(transfer.FileTransferer):
""" """
if (src, dst) in self.already_copied: if (src, dst) in self.already_copied:
log.debug('SKIP %s; already copied', src) log.debug("SKIP %s; already copied", src)
return return
if self.has_error or self._abort.is_set(): if self.has_error or self._abort.is_set():
@ -225,7 +232,9 @@ class FileCopier(transfer.FileTransferer):
# code with a custom `copy_function` may rely on copytree # code with a custom `copy_function` may rely on copytree
# doing the right thing. # doing the right thing.
linkto.symlink_to(dstpath) linkto.symlink_to(dstpath)
shutil.copystat(str(srcpath), str(dstpath), follow_symlinks=not symlinks) shutil.copystat(
str(srcpath), str(dstpath), follow_symlinks=not symlinks
)
else: else:
# ignore dangling symlink if the flag is on # ignore dangling symlink if the flag is on
if not linkto.exists() and ignore_dangling_symlinks: if not linkto.exists() and ignore_dangling_symlinks:
@ -250,7 +259,7 @@ class FileCopier(transfer.FileTransferer):
shutil.copystat(str(src), str(dst)) shutil.copystat(str(src), str(dst))
except OSError as why: except OSError as why:
# Copying file access times may fail on Windows # Copying file access times may fail on Windows
if getattr(why, 'winerror', None) is None: if getattr(why, "winerror", None) is None:
errors.append((src, dst, str(why))) errors.append((src, dst, str(why)))
if errors: if errors:
raise shutil.Error(errors) raise shutil.Error(errors)

25
blender_asset_tracer/pack/progress.py
View File

@ -37,9 +37,11 @@ class Callback(blender_asset_tracer.trace.progress.Callback):
def pack_start(self) -> None: def pack_start(self) -> None:
"""Called when packing starts.""" """Called when packing starts."""
def pack_done(self, def pack_done(
output_blendfile: pathlib.PurePath, self,
missing_files: typing.Set[pathlib.Path]) -> None: output_blendfile: pathlib.PurePath,
missing_files: typing.Set[pathlib.Path],
) -> None:
"""Called when packing is done.""" """Called when packing is done."""
def pack_aborted(self, reason: str): def pack_aborted(self, reason: str):
@ -86,7 +88,7 @@ class ThreadSafeCallback(Callback):
""" """
def __init__(self, wrapped: Callback) -> None: def __init__(self, wrapped: Callback) -> None:
self.log = log.getChild('ThreadSafeCallback') self.log = log.getChild("ThreadSafeCallback")
self.wrapped = wrapped self.wrapped = wrapped
# Thread-safe queue for passing progress reports on the main thread. # Thread-safe queue for passing progress reports on the main thread.
@ -104,9 +106,11 @@ class ThreadSafeCallback(Callback):
def pack_start(self) -> None: def pack_start(self) -> None:
self._queue(self.wrapped.pack_start) self._queue(self.wrapped.pack_start)
def pack_done(self, def pack_done(
output_blendfile: pathlib.PurePath, self,
missing_files: typing.Set[pathlib.Path]) -> None: output_blendfile: pathlib.PurePath,
missing_files: typing.Set[pathlib.Path],
) -> None:
self._queue(self.wrapped.pack_done, output_blendfile, missing_files) self._queue(self.wrapped.pack_done, output_blendfile, missing_files)
def pack_aborted(self, reason: str): def pack_aborted(self, reason: str):
@ -135,8 +139,9 @@ class ThreadSafeCallback(Callback):
while True: while True:
try: try:
call = self._reporting_queue.get(block=timeout is not None, call = self._reporting_queue.get(
timeout=timeout) block=timeout is not None, timeout=timeout
)
except queue.Empty: except queue.Empty:
return return
@ -145,4 +150,4 @@ class ThreadSafeCallback(Callback):
except Exception: except Exception:
# Don't let the handling of one callback call # Don't let the handling of one callback call
# block the entire flush process. # block the entire flush process.
self.log.exception('Error calling %s', call) self.log.exception("Error calling %s", call)

73
blender_asset_tracer/pack/s3.py
View File

@ -32,17 +32,18 @@ log = logging.getLogger(__name__)
# TODO(Sybren): compute MD5 sums of queued files in a separate thread, so that # TODO(Sybren): compute MD5 sums of queued files in a separate thread, so that
# we can upload a file to S3 and compute an MD5 of another file simultaneously. # we can upload a file to S3 and compute an MD5 of another file simultaneously.
def compute_md5(filepath: pathlib.Path) -> str: def compute_md5(filepath: pathlib.Path) -> str:
log.debug('Computing MD5sum of %s', filepath) log.debug("Computing MD5sum of %s", filepath)
hasher = hashlib.md5() hasher = hashlib.md5()
with filepath.open('rb') as infile: with filepath.open("rb") as infile:
while True: while True:
block = infile.read(102400) block = infile.read(102400)
if not block: if not block:
break break
hasher.update(block) hasher.update(block)
md5 = hasher.hexdigest() md5 = hasher.hexdigest()
log.debug('MD5sum of %s is %s', filepath, md5) log.debug("MD5sum of %s is %s", filepath, md5)
return md5 return md5
@ -63,20 +64,21 @@ class S3Packer(Packer):
components = urllib.parse.urlparse(endpoint) components = urllib.parse.urlparse(endpoint)
profile_name = components.netloc profile_name = components.netloc
endpoint = urllib.parse.urlunparse(components) endpoint = urllib.parse.urlunparse(components)
log.debug('Using Boto3 profile name %r for url %r', profile_name, endpoint) log.debug("Using Boto3 profile name %r for url %r", profile_name, endpoint)
self.session = boto3.Session(profile_name=profile_name) self.session = boto3.Session(profile_name=profile_name)
self.client = self.session.client('s3', endpoint_url=endpoint) self.client = self.session.client("s3", endpoint_url=endpoint)
def set_credentials(self, def set_credentials(
endpoint: str, self, endpoint: str, access_key_id: str, secret_access_key: str
access_key_id: str, ):
secret_access_key: str):
"""Set S3 credentials.""" """Set S3 credentials."""
self.client = self.session.client('s3', self.client = self.session.client(
endpoint_url=endpoint, "s3",
aws_access_key_id=access_key_id, endpoint_url=endpoint,
aws_secret_access_key=secret_access_key) aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key,
)
def _create_file_transferer(self) -> transfer.FileTransferer: def _create_file_transferer(self) -> transfer.FileTransferer:
return S3Transferrer(self.client) return S3Transferrer(self.client)
@ -107,7 +109,7 @@ class S3Transferrer(transfer.FileTransferer):
except Exception: except Exception:
# We have to catch exceptions in a broad way, as this is running in # We have to catch exceptions in a broad way, as this is running in
# a separate thread, and exceptions won't otherwise be seen. # a separate thread, and exceptions won't otherwise be seen.
log.exception('Error transferring %s to %s', src, dst) log.exception("Error transferring %s to %s", src, dst)
# Put the files to copy back into the queue, and abort. This allows # Put the files to copy back into the queue, and abort. This allows
# the main thread to inspect the queue and see which files were not # the main thread to inspect the queue and see which files were not
# copied. The one we just failed (due to this exception) should also # copied. The one we just failed (due to this exception) should also
@ -116,9 +118,9 @@ class S3Transferrer(transfer.FileTransferer):
return return
if files_transferred: if files_transferred:
log.info('Transferred %d files', files_transferred) log.info("Transferred %d files", files_transferred)
if files_skipped: if files_skipped:
log.info('Skipped %d files', files_skipped) log.info("Skipped %d files", files_skipped)
def upload_file(self, src: pathlib.Path, dst: pathlib.PurePath) -> bool: def upload_file(self, src: pathlib.Path, dst: pathlib.PurePath) -> bool:
"""Upload a file to an S3 bucket. """Upload a file to an S3 bucket.
@ -135,25 +137,30 @@ class S3Transferrer(transfer.FileTransferer):
existing_md5, existing_size = self.get_metadata(bucket, key) existing_md5, existing_size = self.get_metadata(bucket, key)
if md5 == existing_md5 and src.stat().st_size == existing_size: if md5 == existing_md5 and src.stat().st_size == existing_size:
log.debug('skipping %s, it already exists on the server with MD5 %s', log.debug(
src, existing_md5) "skipping %s, it already exists on the server with MD5 %s",
src,
existing_md5,
)
return False return False
log.info('Uploading %s', src) log.info("Uploading %s", src)
try: try:
self.client.upload_file(str(src), self.client.upload_file(
Bucket=bucket, str(src),
Key=key, Bucket=bucket,
Callback=self.report_transferred, Key=key,
ExtraArgs={'Metadata': {'md5': md5}}) Callback=self.report_transferred,
ExtraArgs={"Metadata": {"md5": md5}},
)
except self.AbortUpload: except self.AbortUpload:
return False return False
return True return True
def report_transferred(self, bytes_transferred: int): def report_transferred(self, bytes_transferred: int):
if self._abort.is_set(): if self._abort.is_set():
log.warning('Interrupting ongoing upload') log.warning("Interrupting ongoing upload")
raise self.AbortUpload('interrupting ongoing upload') raise self.AbortUpload("interrupting ongoing upload")
super().report_transferred(bytes_transferred) super().report_transferred(bytes_transferred)
def get_metadata(self, bucket: str, key: str) -> typing.Tuple[str, int]: def get_metadata(self, bucket: str, key: str) -> typing.Tuple[str, int]:
@ -165,18 +172,18 @@ class S3Transferrer(transfer.FileTransferer):
""" """
import botocore.exceptions import botocore.exceptions
log.debug('Getting metadata of %s/%s', bucket, key) log.debug("Getting metadata of %s/%s", bucket, key)
try: try:
info = self.client.head_object(Bucket=bucket, Key=key) info = self.client.head_object(Bucket=bucket, Key=key)
except botocore.exceptions.ClientError as ex: except botocore.exceptions.ClientError as ex:
error_code = ex.response.get('Error').get('Code', 'Unknown') error_code = ex.response.get("Error").get("Code", "Unknown")
# error_code already is a string, but this makes the code forward # error_code already is a string, but this makes the code forward
# compatible with a time where they use integer codes. # compatible with a time where they use integer codes.
if str(error_code) == '404': if str(error_code) == "404":
return '', -1 return "", -1
raise ValueError('error response:' % ex.response) from None raise ValueError("error response:" % ex.response) from None
try: try:
return info['Metadata']['md5'], info['ContentLength'] return info["Metadata"]["md5"], info["ContentLength"]
except KeyError: except KeyError:
return '', -1 return "", -1

58
blender_asset_tracer/pack/shaman/__init__.py
View File

@ -38,13 +38,15 @@ log = logging.getLogger(__name__)
class ShamanPacker(bat_pack.Packer): class ShamanPacker(bat_pack.Packer):
"""Creates BAT Packs on a Shaman server.""" """Creates BAT Packs on a Shaman server."""
def __init__(self, def __init__(
bfile: pathlib.Path, self,
project: pathlib.Path, bfile: pathlib.Path,
target: str, project: pathlib.Path,
endpoint: str, target: str,
checkout_id: str, endpoint: str,
**kwargs) -> None: checkout_id: str,
**kwargs
) -> None:
"""Constructor """Constructor
:param target: mock target '/' to construct project-relative paths. :param target: mock target '/' to construct project-relative paths.
@ -53,18 +55,20 @@ class ShamanPacker(bat_pack.Packer):
super().__init__(bfile, project, target, **kwargs) super().__init__(bfile, project, target, **kwargs)
self.checkout_id = checkout_id self.checkout_id = checkout_id
self.shaman_endpoint = endpoint self.shaman_endpoint = endpoint
self._checkout_location = '' self._checkout_location = ""
def _get_auth_token(self) -> str: def _get_auth_token(self) -> str:
# TODO: get a token from the Flamenco Server. # TODO: get a token from the Flamenco Server.
token_from_env = os.environ.get('SHAMAN_JWT_TOKEN') token_from_env = os.environ.get("SHAMAN_JWT_TOKEN")
if token_from_env: if token_from_env:
return token_from_env return token_from_env
log.warning('Using temporary hack to get auth token from Shaman, ' log.warning(
'set SHAMAN_JTW_TOKEN to prevent') "Using temporary hack to get auth token from Shaman, "
unauth_shaman = ShamanClient('', self.shaman_endpoint) "set SHAMAN_JTW_TOKEN to prevent"
resp = unauth_shaman.get('get-token', timeout=10) )
unauth_shaman = ShamanClient("", self.shaman_endpoint)
resp = unauth_shaman.get("get-token", timeout=10)
resp.raise_for_status() resp.raise_for_status()
return resp.text return resp.text
@ -72,13 +76,17 @@ class ShamanPacker(bat_pack.Packer):
# TODO: pass self._get_auth_token itself, so that the Transferer will be able to # TODO: pass self._get_auth_token itself, so that the Transferer will be able to
# decide when to get this token (and how many times). # decide when to get this token (and how many times).
auth_token = self._get_auth_token() auth_token = self._get_auth_token()
return ShamanTransferrer(auth_token, self.project, self.shaman_endpoint, self.checkout_id) return ShamanTransferrer(
auth_token, self.project, self.shaman_endpoint, self.checkout_id
)
def _make_target_path(self, target: str) -> pathlib.PurePath: def _make_target_path(self, target: str) -> pathlib.PurePath:
return pathlib.PurePosixPath('/') return pathlib.PurePosixPath("/")
def _on_file_transfer_finished(self, *, file_transfer_completed: bool): def _on_file_transfer_finished(self, *, file_transfer_completed: bool):
super()._on_file_transfer_finished(file_transfer_completed=file_transfer_completed) super()._on_file_transfer_finished(
file_transfer_completed=file_transfer_completed
)
assert isinstance(self._file_transferer, ShamanTransferrer) assert isinstance(self._file_transferer, ShamanTransferrer)
self._checkout_location = self._file_transferer.checkout_location self._checkout_location = self._file_transferer.checkout_location
@ -104,7 +112,7 @@ class ShamanPacker(bat_pack.Packer):
try: try:
super().execute() super().execute()
except requests.exceptions.ConnectionError as ex: except requests.exceptions.ConnectionError as ex:
log.exception('Error communicating with Shaman') log.exception("Error communicating with Shaman")
self.abort(str(ex)) self.abort(str(ex))
self._check_aborted() self._check_aborted()
@ -114,17 +122,19 @@ def parse_endpoint(shaman_url: str) -> typing.Tuple[str, str]:
urlparts = urllib.parse.urlparse(str(shaman_url)) urlparts = urllib.parse.urlparse(str(shaman_url))
if urlparts.scheme in {'shaman', 'shaman+https'}: if urlparts.scheme in {"shaman", "shaman+https"}:
scheme = 'https' scheme = "https"
elif urlparts.scheme == 'shaman+http': elif urlparts.scheme == "shaman+http":
scheme = 'http' scheme = "http"
else: else:
raise ValueError('Invalid scheme %r, choose shaman:// or shaman+http://', urlparts.scheme) raise ValueError(
"Invalid scheme %r, choose shaman:// or shaman+http://", urlparts.scheme
)
checkout_id = urllib.parse.unquote(urlparts.fragment) checkout_id = urllib.parse.unquote(urlparts.fragment)
path = urlparts.path or '/' path = urlparts.path or "/"
new_urlparts = (scheme, urlparts.netloc, path, *urlparts[3:-1], '') new_urlparts = (scheme, urlparts.netloc, path, *urlparts[3:-1], "")
endpoint = urllib.parse.urlunparse(new_urlparts) endpoint = urllib.parse.urlunparse(new_urlparts)
return endpoint, checkout_id return endpoint, checkout_id

52
blender_asset_tracer/pack/shaman/cache.py
View File

@ -30,7 +30,7 @@ from pathlib import Path
from . import time_tracker from . import time_tracker
CACHE_ROOT = Path().home() / '.cache/shaman-client/shasums' CACHE_ROOT = Path().home() / ".cache/shaman-client/shasums"
MAX_CACHE_FILES_AGE_SECS = 3600 * 24 * 60 # 60 days MAX_CACHE_FILES_AGE_SECS = 3600 * 24 * 60 # 60 days
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -53,7 +53,7 @@ def find_files(root: Path) -> typing.Iterable[Path]:
# Ignore hidden files/dirs; these can be things like '.svn' or '.git', # Ignore hidden files/dirs; these can be things like '.svn' or '.git',
# which shouldn't be sent to Shaman. # which shouldn't be sent to Shaman.
if path.name.startswith('.'): if path.name.startswith("."):
continue continue
if path.is_dir(): if path.is_dir():
@ -76,10 +76,10 @@ def compute_checksum(filepath: Path) -> str:
"""Compute the SHA256 checksum for the given file.""" """Compute the SHA256 checksum for the given file."""
blocksize = 32 * 1024 blocksize = 32 * 1024
log.debug('Computing checksum of %s', filepath) log.debug("Computing checksum of %s", filepath)
with time_tracker.track_time(TimeInfo, 'computing_checksums'): with time_tracker.track_time(TimeInfo, "computing_checksums"):
hasher = hashlib.sha256() hasher = hashlib.sha256()
with filepath.open('rb') as infile: with filepath.open("rb") as infile:
while True: while True:
block = infile.read(blocksize) block = infile.read(blocksize)
if not block: if not block:
@ -98,7 +98,9 @@ def _cache_path(filepath: Path) -> Path:
# Reverse the directory, because most variation is in the last bytes. # Reverse the directory, because most variation is in the last bytes.
rev_dir = str(filepath.parent)[::-1] rev_dir = str(filepath.parent)[::-1]
encoded_path = filepath.stem + rev_dir + filepath.suffix encoded_path = filepath.stem + rev_dir + filepath.suffix
cache_key = base64.urlsafe_b64encode(encoded_path.encode(fs_encoding)).decode().rstrip('=') cache_key = (
base64.urlsafe_b64encode(encoded_path.encode(fs_encoding)).decode().rstrip("=")
)
cache_path = CACHE_ROOT / cache_key[:10] / cache_key[10:] cache_path = CACHE_ROOT / cache_key[:10] / cache_key[10:]
return cache_path return cache_path
@ -111,42 +113,44 @@ def compute_cached_checksum(filepath: Path) -> str:
skip the actual SHA256 computation. skip the actual SHA256 computation.
""" """
with time_tracker.track_time(TimeInfo, 'checksum_cache_handling'): with time_tracker.track_time(TimeInfo, "checksum_cache_handling"):
current_stat = filepath.stat() current_stat = filepath.stat()
cache_path = _cache_path(filepath) cache_path = _cache_path(filepath)
try: try:
with cache_path.open('r') as cache_file: with cache_path.open("r") as cache_file:
payload = json.load(cache_file) payload = json.load(cache_file)
except (OSError, ValueError): except (OSError, ValueError):
# File may not exist, or have invalid contents. # File may not exist, or have invalid contents.
pass pass
else: else:
checksum = payload.get('checksum', '') checksum = payload.get("checksum", "")
cached_mtime = payload.get('file_mtime', 0.0) cached_mtime = payload.get("file_mtime", 0.0)
cached_size = payload.get('file_size', -1) cached_size = payload.get("file_size", -1)
if (checksum if (
and current_stat.st_size == cached_size checksum
and abs(cached_mtime - current_stat.st_mtime) < 0.01): and current_stat.st_size == cached_size
and abs(cached_mtime - current_stat.st_mtime) < 0.01
):
cache_path.touch() cache_path.touch()
return checksum return checksum
checksum = compute_checksum(filepath) checksum = compute_checksum(filepath)
with time_tracker.track_time(TimeInfo, 'checksum_cache_handling'): with time_tracker.track_time(TimeInfo, "checksum_cache_handling"):
payload = { payload = {
'checksum': checksum, "checksum": checksum,
'file_mtime': current_stat.st_mtime, "file_mtime": current_stat.st_mtime,
'file_size': current_stat.st_size, "file_size": current_stat.st_size,
} }
try: try:
cache_path.parent.mkdir(parents=True, exist_ok=True) cache_path.parent.mkdir(parents=True, exist_ok=True)
with cache_path.open('w') as cache_file: with cache_path.open("w") as cache_file:
json.dump(payload, cache_file) json.dump(payload, cache_file)
except IOError as ex: except IOError as ex:
log.warning('Unable to write checksum cache file %s: %s', cache_path, ex) log.warning("Unable to write checksum cache file %s: %s", cache_path, ex)
return checksum return checksum
@ -157,7 +161,7 @@ def cleanup_cache() -> None:
if not CACHE_ROOT.exists(): if not CACHE_ROOT.exists():
return return
with time_tracker.track_time(TimeInfo, 'checksum_cache_handling'): with time_tracker.track_time(TimeInfo, "checksum_cache_handling"):
queue = deque([CACHE_ROOT]) queue = deque([CACHE_ROOT])
rmdir_queue = [] rmdir_queue = []
@ -194,4 +198,8 @@ def cleanup_cache() -> None:
pass pass
if num_removed_dirs or num_removed_files: if num_removed_dirs or num_removed_files:
log.info('Cache Cleanup: removed %d dirs and %d files', num_removed_dirs, num_removed_files) log.info(
"Cache Cleanup: removed %d dirs and %d files",
num_removed_dirs,
num_removed_files,
)

28
blender_asset_tracer/pack/shaman/client.py
View File

@ -37,14 +37,14 @@ class ShamanClient:
) )
http_adapter = requests.adapters.HTTPAdapter(max_retries=retries) http_adapter = requests.adapters.HTTPAdapter(max_retries=retries)
self._session = requests.session() self._session = requests.session()
self._session.mount('https://', http_adapter) self._session.mount("https://", http_adapter)
self._session.mount('http://', http_adapter) self._session.mount("http://", http_adapter)
if auth_token: if auth_token:
self._session.headers['Authorization'] = 'Bearer ' + auth_token self._session.headers["Authorization"] = "Bearer " + auth_token
def request(self, method: str, url: str, **kwargs) -> requests.Response: def request(self, method: str, url: str, **kwargs) -> requests.Response:
kwargs.setdefault('timeout', 300) kwargs.setdefault("timeout", 300)
full_url = urllib.parse.urljoin(self._base_url, url) full_url = urllib.parse.urljoin(self._base_url, url)
return self._session.request(method, full_url, **kwargs) return self._session.request(method, full_url, **kwargs)
@ -56,8 +56,8 @@ class ShamanClient:
:rtype: requests.Response :rtype: requests.Response
""" """
kwargs.setdefault('allow_redirects', True) kwargs.setdefault("allow_redirects", True)
return self.request('GET', url, **kwargs) return self.request("GET", url, **kwargs)
def options(self, url, **kwargs): def options(self, url, **kwargs):
r"""Sends a OPTIONS request. Returns :class:`Response` object. r"""Sends a OPTIONS request. Returns :class:`Response` object.
@ -67,8 +67,8 @@ class ShamanClient:
:rtype: requests.Response :rtype: requests.Response
""" """
kwargs.setdefault('allow_redirects', True) kwargs.setdefault("allow_redirects", True)
return self.request('OPTIONS', url, **kwargs) return self.request("OPTIONS", url, **kwargs)
def head(self, url, **kwargs): def head(self, url, **kwargs):
r"""Sends a HEAD request. Returns :class:`Response` object. r"""Sends a HEAD request. Returns :class:`Response` object.
@ -78,8 +78,8 @@ class ShamanClient:
:rtype: requests.Response :rtype: requests.Response
""" """
kwargs.setdefault('allow_redirects', False) kwargs.setdefault("allow_redirects", False)
return self.request('HEAD', url, **kwargs) return self.request("HEAD", url, **kwargs)
def post(self, url, data=None, json=None, **kwargs): def post(self, url, data=None, json=None, **kwargs):
r"""Sends a POST request. Returns :class:`Response` object. r"""Sends a POST request. Returns :class:`Response` object.
@ -92,7 +92,7 @@ class ShamanClient:
:rtype: requests.Response :rtype: requests.Response
""" """
return self.request('POST', url, data=data, json=json, **kwargs) return self.request("POST", url, data=data, json=json, **kwargs)
def put(self, url, data=None, **kwargs): def put(self, url, data=None, **kwargs):
r"""Sends a PUT request. Returns :class:`Response` object. r"""Sends a PUT request. Returns :class:`Response` object.
@ -104,7 +104,7 @@ class ShamanClient:
:rtype: requests.Response :rtype: requests.Response
""" """
return self.request('PUT', url, data=data, **kwargs) return self.request("PUT", url, data=data, **kwargs)
def patch(self, url, data=None, **kwargs): def patch(self, url, data=None, **kwargs):
r"""Sends a PATCH request. Returns :class:`Response` object. r"""Sends a PATCH request. Returns :class:`Response` object.
@ -116,7 +116,7 @@ class ShamanClient:
:rtype: requests.Response :rtype: requests.Response
""" """
return self.request('PATCH', url, data=data, **kwargs) return self.request("PATCH", url, data=data, **kwargs)
def delete(self, url, **kwargs): def delete(self, url, **kwargs):
r"""Sends a DELETE request. Returns :class:`Response` object. r"""Sends a DELETE request. Returns :class:`Response` object.
@ -126,4 +126,4 @@ class ShamanClient:
:rtype: requests.Response :rtype: requests.Response
""" """
return self.request('DELETE', url, **kwargs) return self.request("DELETE", url, **kwargs)

173
blender_asset_tracer/pack/shaman/transfer.py
View File

@ -48,9 +48,15 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
class AbortUpload(Exception): class AbortUpload(Exception):
"""Raised from the upload callback to abort an upload.""" """Raised from the upload callback to abort an upload."""
def __init__(self, auth_token: str, project_root: pathlib.Path, def __init__(
shaman_endpoint: str, checkout_id: str) -> None: self,
auth_token: str,
project_root: pathlib.Path,
shaman_endpoint: str,
checkout_id: str,
) -> None:
from . import client from . import client
super().__init__() super().__init__()
self.client = client.ShamanClient(auth_token, shaman_endpoint) self.client = client.ShamanClient(auth_token, shaman_endpoint)
self.project_root = project_root self.project_root = project_root
@ -63,7 +69,7 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
# checkout. This can then be combined with the project-relative path # checkout. This can then be combined with the project-relative path
# of the to-be-rendered blend file (e.g. the one 'bat pack' was pointed # of the to-be-rendered blend file (e.g. the one 'bat pack' was pointed
# at). # at).
self._checkout_location = '' self._checkout_location = ""
self.uploaded_files = 0 self.uploaded_files = 0
self.uploaded_bytes = 0 self.uploaded_bytes = 0
@ -76,24 +82,32 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
# Construct the Shaman Checkout Definition file. # Construct the Shaman Checkout Definition file.
# This blocks until we know the entire list of files to transfer. # This blocks until we know the entire list of files to transfer.
definition_file, allowed_relpaths, delete_when_done = self._create_checkout_definition() (
definition_file,
allowed_relpaths,
delete_when_done,
) = self._create_checkout_definition()
if not definition_file: if not definition_file:
# An error has already been logged. # An error has already been logged.
return return
self.log.info('Created checkout definition file of %d KiB', self.log.info(
len(definition_file) // 1024) "Created checkout definition file of %d KiB",
self.log.info('Feeding %d files to the Shaman', len(self._file_info)) len(definition_file) // 1024,
)
self.log.info("Feeding %d files to the Shaman", len(self._file_info))
if self.log.isEnabledFor(logging.INFO): if self.log.isEnabledFor(logging.INFO):
for path in self._file_info: for path in self._file_info:
self.log.info(' - %s', path) self.log.info(" - %s", path)
# Try to upload all the files. # Try to upload all the files.
failed_paths = set() # type: typing.Set[str] failed_paths = set() # type: typing.Set[str]
max_tries = 50 max_tries = 50
for try_index in range(max_tries): for try_index in range(max_tries):
# Send the file to the Shaman and see what we still need to send there. # Send the file to the Shaman and see what we still need to send there.
to_upload = self._send_checkout_def_to_shaman(definition_file, allowed_relpaths) to_upload = self._send_checkout_def_to_shaman(
definition_file, allowed_relpaths
)
if to_upload is None: if to_upload is None:
# An error has already been logged. # An error has already been logged.
return return
@ -102,7 +116,7 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
break break
# Send the files that still need to be sent. # Send the files that still need to be sent.
self.log.info('Upload attempt %d', try_index + 1) self.log.info("Upload attempt %d", try_index + 1)
failed_paths = self._upload_files(to_upload) failed_paths = self._upload_files(to_upload)
if not failed_paths: if not failed_paths:
break break
@ -113,11 +127,13 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
# file to the Shaman and obtain a new list of files to upload. # file to the Shaman and obtain a new list of files to upload.
if failed_paths: if failed_paths:
self.log.error('Aborting upload due to too many failures') self.log.error("Aborting upload due to too many failures")
self.error_set('Giving up after %d attempts to upload the files' % max_tries) self.error_set(
"Giving up after %d attempts to upload the files" % max_tries
)
return return
self.log.info('All files uploaded succesfully') self.log.info("All files uploaded succesfully")
self._request_checkout(definition_file) self._request_checkout(definition_file)
# Delete the files that were supposed to be moved. # Delete the files that were supposed to be moved.
@ -127,12 +143,13 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
except Exception as ex: except Exception as ex:
# We have to catch exceptions in a broad way, as this is running in # We have to catch exceptions in a broad way, as this is running in
# a separate thread, and exceptions won't otherwise be seen. # a separate thread, and exceptions won't otherwise be seen.
self.log.exception('Error transferring files to Shaman') self.log.exception("Error transferring files to Shaman")
self.error_set('Unexpected exception transferring files to Shaman: %s' % ex) self.error_set("Unexpected exception transferring files to Shaman: %s" % ex)
# noinspection PyBroadException # noinspection PyBroadException
def _create_checkout_definition(self) \ def _create_checkout_definition(
-> typing.Tuple[bytes, typing.Set[str], typing.List[pathlib.Path]]: self,
) -> typing.Tuple[bytes, typing.Set[str], typing.List[pathlib.Path]]:
"""Create the checkout definition file for this BAT pack. """Create the checkout definition file for this BAT pack.
:returns: the checkout definition (as bytes), a set of paths in that file, :returns: the checkout definition (as bytes), a set of paths in that file,
@ -162,8 +179,8 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
filesize=filesize, filesize=filesize,
abspath=src, abspath=src,
) )
line = '%s %s %s' % (checksum, filesize, relpath) line = "%s %s %s" % (checksum, filesize, relpath)
definition_lines.append(line.encode('utf8')) definition_lines.append(line.encode("utf8"))
relpaths.add(relpath) relpaths.add(relpath)
if act == bat_transfer.Action.MOVE: if act == bat_transfer.Action.MOVE:
@ -171,7 +188,7 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
except Exception: except Exception:
# We have to catch exceptions in a broad way, as this is running in # We have to catch exceptions in a broad way, as this is running in
# a separate thread, and exceptions won't otherwise be seen. # a separate thread, and exceptions won't otherwise be seen.
msg = 'Error transferring %s to %s' % (src, dst) msg = "Error transferring %s to %s" % (src, dst)
self.log.exception(msg) self.log.exception(msg)
# Put the files to copy back into the queue, and abort. This allows # Put the files to copy back into the queue, and abort. This allows
# the main thread to inspect the queue and see which files were not # the main thread to inspect the queue and see which files were not
@ -179,35 +196,39 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
# be reported there. # be reported there.
self.queue.put((src, dst, act)) self.queue.put((src, dst, act))
self.error_set(msg) self.error_set(msg)
return b'', set(), delete_when_done return b"", set(), delete_when_done
cache.cleanup_cache() cache.cleanup_cache()
return b'\n'.join(definition_lines), relpaths, delete_when_done return b"\n".join(definition_lines), relpaths, delete_when_done
def _send_checkout_def_to_shaman(self, definition_file: bytes, def _send_checkout_def_to_shaman(
allowed_relpaths: typing.Set[str]) \ self, definition_file: bytes, allowed_relpaths: typing.Set[str]
-> typing.Optional[collections.deque]: ) -> typing.Optional[collections.deque]:
"""Send the checkout definition file to the Shaman. """Send the checkout definition file to the Shaman.
:return: An iterable of paths (relative to the project root) that still :return: An iterable of paths (relative to the project root) that still
need to be uploaded, or None if there was an error. need to be uploaded, or None if there was an error.
""" """
resp = self.client.post('checkout/requirements', data=definition_file, stream=True, resp = self.client.post(
headers={'Content-Type': 'text/plain'}, "checkout/requirements",
timeout=15) data=definition_file,
stream=True,
headers={"Content-Type": "text/plain"},
timeout=15,
)
if resp.status_code >= 300: if resp.status_code >= 300:
msg = 'Error from Shaman, code %d: %s' % (resp.status_code, resp.text) msg = "Error from Shaman, code %d: %s" % (resp.status_code, resp.text)
self.log.error(msg) self.log.error(msg)
self.error_set(msg) self.error_set(msg)
return None return None
to_upload = collections.deque() # type: collections.deque to_upload = collections.deque() # type: collections.deque
for line in resp.iter_lines(): for line in resp.iter_lines():
response, path = line.decode().split(' ', 1) response, path = line.decode().split(" ", 1)
self.log.debug(' %s: %s', response, path) self.log.debug(" %s: %s", response, path)
if path not in allowed_relpaths: if path not in allowed_relpaths:
msg = 'Shaman requested path we did not intend to upload: %r' % path msg = "Shaman requested path we did not intend to upload: %r" % path
self.log.error(msg) self.log.error(msg)
self.error_set(msg) self.error_set(msg)
return None return None
@ -216,13 +237,13 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
to_upload.appendleft(path) to_upload.appendleft(path)
elif response == response_already_uploading: elif response == response_already_uploading:
to_upload.append(path) to_upload.append(path)
elif response == 'ERROR': elif response == "ERROR":
msg = 'Error from Shaman: %s' % path msg = "Error from Shaman: %s" % path
self.log.error(msg) self.log.error(msg)
self.error_set(msg) self.error_set(msg)
return None return None
else: else:
msg = 'Unknown response from Shaman for path %r: %r' % (path, response) msg = "Unknown response from Shaman for path %r: %r" % (path, response)
self.log.error(msg) self.log.error(msg)
self.error_set(msg) self.error_set(msg)
return None return None
@ -240,7 +261,9 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
def defer(some_path: str): def defer(some_path: str):
nonlocal to_upload nonlocal to_upload
self.log.info(' %s deferred (already being uploaded by someone else)', some_path) self.log.info(
" %s deferred (already being uploaded by someone else)", some_path
)
deferred_paths.add(some_path) deferred_paths.add(some_path)
# Instead of deferring this one file, randomize the files to upload. # Instead of deferring this one file, randomize the files to upload.
@ -251,35 +274,41 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
to_upload = collections.deque(all_files) to_upload = collections.deque(all_files)
if not to_upload: if not to_upload:
self.log.info('All %d files are at the Shaman already', len(self._file_info)) self.log.info(
"All %d files are at the Shaman already", len(self._file_info)
)
self.report_transferred(0) self.report_transferred(0)
return failed_paths return failed_paths
self.log.info('Going to upload %d of %d files', len(to_upload), len(self._file_info)) self.log.info(
"Going to upload %d of %d files", len(to_upload), len(self._file_info)
)
while to_upload: while to_upload:
# After too many failures, just retry to get a fresh set of files to upload. # After too many failures, just retry to get a fresh set of files to upload.
if len(failed_paths) > MAX_FAILED_PATHS: if len(failed_paths) > MAX_FAILED_PATHS:
self.log.info('Too many failures, going to abort this iteration') self.log.info("Too many failures, going to abort this iteration")
failed_paths.update(to_upload) failed_paths.update(to_upload)
return failed_paths return failed_paths
path = to_upload.popleft() path = to_upload.popleft()
fileinfo = self._file_info[path] fileinfo = self._file_info[path]
self.log.info(' %s', path) self.log.info(" %s", path)
headers = { headers = {
'X-Shaman-Original-Filename': path, "X-Shaman-Original-Filename": path,
} }
# Let the Shaman know whether we can defer uploading this file or not. # Let the Shaman know whether we can defer uploading this file or not.
can_defer = (len(deferred_paths) < MAX_DEFERRED_PATHS can_defer = (
and path not in deferred_paths len(deferred_paths) < MAX_DEFERRED_PATHS
and len(to_upload)) and path not in deferred_paths
and len(to_upload)
)
if can_defer: if can_defer:
headers['X-Shaman-Can-Defer-Upload'] = 'true' headers["X-Shaman-Can-Defer-Upload"] = "true"
url = 'files/%s/%d' % (fileinfo.checksum, fileinfo.filesize) url = "files/%s/%d" % (fileinfo.checksum, fileinfo.filesize)
try: try:
with fileinfo.abspath.open('rb') as infile: with fileinfo.abspath.open("rb") as infile:
resp = self.client.post(url, data=infile, headers=headers) resp = self.client.post(url, data=infile, headers=headers)
except requests.ConnectionError as ex: except requests.ConnectionError as ex:
@ -290,7 +319,9 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
# connection. # connection.
defer(path) defer(path)
else: else:
self.log.info(' %s could not be uploaded, might retry later: %s', path, ex) self.log.info(
" %s could not be uploaded, might retry later: %s", path, ex
)
failed_paths.add(path) failed_paths.add(path)
continue continue
@ -301,12 +332,15 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
if can_defer: if can_defer:
defer(path) defer(path)
else: else:
self.log.info(' %s skipped (already existed on the server)', path) self.log.info(" %s skipped (already existed on the server)", path)
continue continue
if resp.status_code >= 300: if resp.status_code >= 300:
msg = 'Error from Shaman uploading %s, code %d: %s' % ( msg = "Error from Shaman uploading %s, code %d: %s" % (
fileinfo.abspath, resp.status_code, resp.text) fileinfo.abspath,
resp.status_code,
resp.text,
)
self.log.error(msg) self.log.error(msg)
self.error_set(msg) self.error_set(msg)
return failed_paths return failed_paths
@ -318,42 +352,53 @@ class ShamanTransferrer(bat_transfer.FileTransferer):
self.report_transferred(file_size) self.report_transferred(file_size)
if not failed_paths: if not failed_paths:
self.log.info('Done uploading %d bytes in %d files', self.log.info(
self.uploaded_bytes, self.uploaded_files) "Done uploading %d bytes in %d files",
self.uploaded_bytes,
self.uploaded_files,
)
else: else:
self.log.info('Uploaded %d bytes in %d files so far', self.log.info(
self.uploaded_bytes, self.uploaded_files) "Uploaded %d bytes in %d files so far",
self.uploaded_bytes,
self.uploaded_files,
)
return failed_paths return failed_paths
def report_transferred(self, bytes_transferred: int): def report_transferred(self, bytes_transferred: int):
if self._abort.is_set(): if self._abort.is_set():
self.log.warning('Interrupting ongoing upload') self.log.warning("Interrupting ongoing upload")
raise self.AbortUpload('interrupting ongoing upload') raise self.AbortUpload("interrupting ongoing upload")
super().report_transferred(bytes_transferred) super().report_transferred(bytes_transferred)
def _request_checkout(self, definition_file: bytes): def _request_checkout(self, definition_file: bytes):
"""Ask the Shaman to create a checkout of this BAT pack.""" """Ask the Shaman to create a checkout of this BAT pack."""
if not self.checkout_id: if not self.checkout_id:
self.log.warning('NOT requesting checkout at Shaman') self.log.warning("NOT requesting checkout at Shaman")
return return
self.log.info('Requesting checkout at Shaman for checkout_id=%r', self.checkout_id) self.log.info(
resp = self.client.post('checkout/create/%s' % self.checkout_id, data=definition_file, "Requesting checkout at Shaman for checkout_id=%r", self.checkout_id
headers={'Content-Type': 'text/plain'}) )
resp = self.client.post(
"checkout/create/%s" % self.checkout_id,
data=definition_file,
headers={"Content-Type": "text/plain"},
)
if resp.status_code >= 300: if resp.status_code >= 300:
msg = 'Error from Shaman, code %d: %s' % (resp.status_code, resp.text) msg = "Error from Shaman, code %d: %s" % (resp.status_code, resp.text)
self.log.error(msg) self.log.error(msg)
self.error_set(msg) self.error_set(msg)
return return
self._checkout_location = resp.text.strip() self._checkout_location = resp.text.strip()
self.log.info('Response from Shaman, code %d: %s', resp.status_code, resp.text) self.log.info("Response from Shaman, code %d: %s", resp.status_code, resp.text)
@property @property
def checkout_location(self) -> str: def checkout_location(self) -> str:
"""Returns the checkout location, or '' if no checkout was made.""" """Returns the checkout location, or '' if no checkout was made."""
if not self._checkout_location: if not self._checkout_location:
return '' return ""
return self._checkout_location return self._checkout_location

50
blender_asset_tracer/pack/transfer.py
View File

@ -56,7 +56,7 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
def __init__(self) -> None: def __init__(self) -> None:
super().__init__() super().__init__()
self.log = log.getChild('FileTransferer') self.log = log.getChild("FileTransferer")
# For copying in a different process. By using a priority queue the files # For copying in a different process. By using a priority queue the files
# are automatically sorted alphabetically, which means we go through all files # are automatically sorted alphabetically, which means we go through all files
@ -67,13 +67,15 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
# maxsize=100 is just a guess as to a reasonable upper limit. When this limit # maxsize=100 is just a guess as to a reasonable upper limit. When this limit
# is reached, the main thread will simply block while waiting for this thread # is reached, the main thread will simply block while waiting for this thread
# to finish copying a file. # to finish copying a file.
self.queue = queue.PriorityQueue(maxsize=100) # type: queue.PriorityQueue[QueueItem] self.queue = queue.PriorityQueue(
maxsize=100
) # type: queue.PriorityQueue[QueueItem]
self.done = threading.Event() self.done = threading.Event()
self._abort = threading.Event() # Indicates user-requested abort self._abort = threading.Event() # Indicates user-requested abort
self.__error_mutex = threading.Lock() self.__error_mutex = threading.Lock()
self.__error = threading.Event() # Indicates abort due to some error self.__error = threading.Event() # Indicates abort due to some error
self.__error_message = '' self.__error_message = ""
# Instantiate a dummy progress callback so that we can call it # Instantiate a dummy progress callback so that we can call it
# without checking for None all the time. # without checking for None all the time.
@ -87,8 +89,12 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
def queue_copy(self, src: pathlib.Path, dst: pathlib.PurePath): def queue_copy(self, src: pathlib.Path, dst: pathlib.PurePath):
"""Queue a copy action from 'src' to 'dst'.""" """Queue a copy action from 'src' to 'dst'."""
assert not self.done.is_set(), 'Queueing not allowed after done_and_join() was called' assert (
assert not self._abort.is_set(), 'Queueing not allowed after abort_and_join() was called' not self.done.is_set()
), "Queueing not allowed after done_and_join() was called"
assert (
not self._abort.is_set()
), "Queueing not allowed after abort_and_join() was called"
if self.__error.is_set(): if self.__error.is_set():
return return
self.queue.put((src, dst, Action.COPY)) self.queue.put((src, dst, Action.COPY))
@ -96,8 +102,12 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
def queue_move(self, src: pathlib.Path, dst: pathlib.PurePath): def queue_move(self, src: pathlib.Path, dst: pathlib.PurePath):
"""Queue a move action from 'src' to 'dst'.""" """Queue a move action from 'src' to 'dst'."""
assert not self.done.is_set(), 'Queueing not allowed after done_and_join() was called' assert (
assert not self._abort.is_set(), 'Queueing not allowed after abort_and_join() was called' not self.done.is_set()
), "Queueing not allowed after done_and_join() was called"
assert (
not self._abort.is_set()
), "Queueing not allowed after abort_and_join() was called"
if self.__error.is_set(): if self.__error.is_set():
return return
self.queue.put((src, dst, Action.MOVE)) self.queue.put((src, dst, Action.MOVE))
@ -107,7 +117,9 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
"""Report transfer of `block_size` bytes.""" """Report transfer of `block_size` bytes."""
self.total_transferred_bytes += bytes_transferred self.total_transferred_bytes += bytes_transferred
self.progress_cb.transfer_progress(self.total_queued_bytes, self.total_transferred_bytes) self.progress_cb.transfer_progress(
self.total_queued_bytes, self.total_transferred_bytes
)
def done_and_join(self) -> None: def done_and_join(self) -> None:
"""Indicate all files have been queued, and wait until done. """Indicate all files have been queued, and wait until done.
@ -128,7 +140,8 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
assert files_remaining assert files_remaining
raise FileTransferError( raise FileTransferError(
"%d files couldn't be transferred" % len(files_remaining), "%d files couldn't be transferred" % len(files_remaining),
files_remaining) files_remaining,
)
def _files_remaining(self) -> typing.List[pathlib.Path]: def _files_remaining(self) -> typing.List[pathlib.Path]:
"""Source files that were queued but not transferred.""" """Source files that were queued but not transferred."""
@ -140,7 +153,7 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
def abort(self) -> None: def abort(self) -> None:
"""Abort the file transfer, immediately returns.""" """Abort the file transfer, immediately returns."""
log.info('Aborting') log.info("Aborting")
self._abort.set() self._abort.set()
def abort_and_join(self) -> None: def abort_and_join(self) -> None:
@ -152,8 +165,11 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
files_remaining = self._files_remaining() files_remaining = self._files_remaining()
if not files_remaining: if not files_remaining:
return return
log.warning("%d files couldn't be transferred, starting with %s", log.warning(
len(files_remaining), files_remaining[0]) "%d files couldn't be transferred, starting with %s",
len(files_remaining),
files_remaining[0],
)
def iter_queue(self) -> typing.Iterable[QueueItem]: def iter_queue(self) -> typing.Iterable[QueueItem]:
"""Generator, yield queued items until the work is done.""" """Generator, yield queued items until the work is done."""
@ -176,13 +192,13 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
if timeout: if timeout:
run_until = time.time() + timeout run_until = time.time() + timeout
else: else:
run_until = float('inf') run_until = float("inf")
# We can't simply block the thread, we have to keep watching the # We can't simply block the thread, we have to keep watching the
# progress queue. # progress queue.
while self.is_alive(): while self.is_alive():
if time.time() > run_until: if time.time() > run_until:
self.log.warning('Timeout while waiting for transfer to finish') self.log.warning("Timeout while waiting for transfer to finish")
return return
self.progress_cb.flush(timeout=0.5) self.progress_cb.flush(timeout=0.5)
@ -192,11 +208,11 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
def delete_file(self, path: pathlib.Path): def delete_file(self, path: pathlib.Path):
"""Deletes a file, only logging a warning if deletion fails.""" """Deletes a file, only logging a warning if deletion fails."""
log.debug('Deleting %s, file has been transferred', path) log.debug("Deleting %s, file has been transferred", path)
try: try:
path.unlink() path.unlink()
except IOError as ex: except IOError as ex:
log.warning('Unable to delete %s: %s', path, ex) log.warning("Unable to delete %s: %s", path, ex)
@property @property
def has_error(self) -> bool: def has_error(self) -> bool:
@ -217,5 +233,5 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
"""Retrieve the error messsage, or an empty string if no error occurred.""" """Retrieve the error messsage, or an empty string if no error occurred."""
with self.__error_mutex: with self.__error_mutex:
if not self.__error.is_set(): if not self.__error.is_set():
return '' return ""
return self.__error_message return self.__error_message

16
blender_asset_tracer/pack/zipped.py
View File

@ -30,7 +30,7 @@ from . import Packer, transfer
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
# Suffixes to store uncompressed in the zip. # Suffixes to store uncompressed in the zip.
STORE_ONLY = {'.jpg', '.jpeg', '.exr'} STORE_ONLY = {".jpg", ".jpeg", ".exr"}
class ZipPacker(Packer): class ZipPacker(Packer):
@ -58,9 +58,9 @@ class ZipTransferrer(transfer.FileTransferer):
zippath = self.zippath.absolute() zippath = self.zippath.absolute()
with zipfile.ZipFile(str(zippath), 'w') as outzip: with zipfile.ZipFile(str(zippath), "w") as outzip:
for src, dst, act in self.iter_queue(): for src, dst, act in self.iter_queue():
assert src.is_absolute(), 'expecting only absolute paths, not %r' % src assert src.is_absolute(), "expecting only absolute paths, not %r" % src
dst = pathlib.Path(dst).absolute() dst = pathlib.Path(dst).absolute()
try: try:
@ -69,18 +69,20 @@ class ZipTransferrer(transfer.FileTransferer):
# Don't bother trying to compress already-compressed files. # Don't bother trying to compress already-compressed files.
if src.suffix.lower() in STORE_ONLY: if src.suffix.lower() in STORE_ONLY:
compression = zipfile.ZIP_STORED compression = zipfile.ZIP_STORED
log.debug('ZIP %s -> %s (uncompressed)', src, relpath) log.debug("ZIP %s -> %s (uncompressed)", src, relpath)
else: else:
compression = zipfile.ZIP_DEFLATED compression = zipfile.ZIP_DEFLATED
log.debug('ZIP %s -> %s', src, relpath) log.debug("ZIP %s -> %s", src, relpath)
outzip.write(str(src), arcname=str(relpath), compress_type=compression) outzip.write(
str(src), arcname=str(relpath), compress_type=compression
)
if act == transfer.Action.MOVE: if act == transfer.Action.MOVE:
self.delete_file(src) self.delete_file(src)
except Exception: except Exception:
# We have to catch exceptions in a broad way, as this is running in # We have to catch exceptions in a broad way, as this is running in
# a separate thread, and exceptions won't otherwise be seen. # a separate thread, and exceptions won't otherwise be seen.
log.exception('Error transferring %s to %s', src, dst) log.exception("Error transferring %s to %s", src, dst)
# Put the files to copy back into the queue, and abort. This allows # Put the files to copy back into the queue, and abort. This allows
# the main thread to inspect the queue and see which files were not # the main thread to inspect the queue and see which files were not
# copied. The one we just failed (due to this exception) should also # copied. The one we just failed (due to this exception) should also

22
blender_asset_tracer/trace/__init__.py
View File

@ -28,23 +28,28 @@ log = logging.getLogger(__name__)
codes_to_skip = { codes_to_skip = {
# These blocks never have external assets: # These blocks never have external assets:
b'ID', b'WM', b'SN', b"ID",
b"WM",
b"SN",
# These blocks are skipped for now, until we have proof they point to # These blocks are skipped for now, until we have proof they point to
# assets otherwise missed: # assets otherwise missed:
b'GR', b'WO', b'BR', b'LS', b"GR",
b"WO",
b"BR",
b"LS",
} }
def deps(bfilepath: pathlib.Path, progress_cb: typing.Optional[progress.Callback] = None) \ def deps(
-> typing.Iterator[result.BlockUsage]: bfilepath: pathlib.Path, progress_cb: typing.Optional[progress.Callback] = None
) -> typing.Iterator[result.BlockUsage]:
"""Open the blend file and report its dependencies. """Open the blend file and report its dependencies.
:param bfilepath: File to open. :param bfilepath: File to open.
:param progress_cb: Progress callback object. :param progress_cb: Progress callback object.
""" """
log.info('opening: %s', bfilepath) log.info("opening: %s", bfilepath)
bfile = blendfile.open_cached(bfilepath) bfile = blendfile.open_cached(bfilepath)
bi = file2blocks.BlockIterator() bi = file2blocks.BlockIterator()
@ -64,8 +69,9 @@ def deps(bfilepath: pathlib.Path, progress_cb: typing.Optional[progress.Callback
yield block_usage yield block_usage
def asset_holding_blocks(blocks: typing.Iterable[blendfile.BlendFileBlock]) \ def asset_holding_blocks(
-> typing.Iterator[blendfile.BlendFileBlock]: blocks: typing.Iterable[blendfile.BlendFileBlock],
) -> typing.Iterator[blendfile.BlendFileBlock]:
"""Generator, yield data blocks that could reference external assets.""" """Generator, yield data blocks that could reference external assets."""
for block in blocks: for block in blocks:
assert isinstance(block, blendfile.BlendFileBlock) assert isinstance(block, blendfile.BlendFileBlock)

89
blender_asset_tracer/trace/blocks2assets.py
View File

@ -39,17 +39,17 @@ _funcs_for_code = {} # type: typing.Dict[bytes, typing.Callable]
def iter_assets(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]: def iter_assets(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
"""Generator, yield the assets used by this data block.""" """Generator, yield the assets used by this data block."""
assert block.code != b'DATA' assert block.code != b"DATA"
try: try:
block_reader = _funcs_for_code[block.code] block_reader = _funcs_for_code[block.code]
except KeyError: except KeyError:
if block.code not in _warned_about_types: if block.code not in _warned_about_types:
log.debug('No reader implemented for block type %r', block.code.decode()) log.debug("No reader implemented for block type %r", block.code.decode())
_warned_about_types.add(block.code) _warned_about_types.add(block.code)
return return
log.debug('Tracing block %r', block) log.debug("Tracing block %r", block)
yield from block_reader(block) yield from block_reader(block)
@ -70,8 +70,8 @@ def skip_packed(wrapped):
@functools.wraps(wrapped) @functools.wraps(wrapped)
def wrapper(block: blendfile.BlendFileBlock, *args, **kwargs): def wrapper(block: blendfile.BlendFileBlock, *args, **kwargs):
if block.get(b'packedfile', default=False): if block.get(b"packedfile", default=False):
log.debug('Datablock %r is packed; skipping', block.id_name) log.debug("Datablock %r is packed; skipping", block.id_name)
return return
yield from wrapped(block, *args, **kwargs) yield from wrapped(block, *args, **kwargs)
@ -79,32 +79,36 @@ def skip_packed(wrapped):
return wrapper return wrapper
@dna_code('CF') @dna_code("CF")
def cache_file(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]: def cache_file(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
"""Cache file data blocks.""" """Cache file data blocks."""
path, field = block.get(b'filepath', return_field=True) path, field = block.get(b"filepath", return_field=True)
yield result.BlockUsage(block, path, path_full_field=field) yield result.BlockUsage(block, path, path_full_field=field)
@dna_code('IM') @dna_code("IM")
@skip_packed @skip_packed
def image(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]: def image(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
"""Image data blocks.""" """Image data blocks."""
# old files miss this # old files miss this
image_source = block.get(b'source', default=cdefs.IMA_SRC_FILE) image_source = block.get(b"source", default=cdefs.IMA_SRC_FILE)
if image_source not in {cdefs.IMA_SRC_FILE, cdefs.IMA_SRC_SEQUENCE, cdefs.IMA_SRC_MOVIE}: if image_source not in {
cdefs.IMA_SRC_FILE,
cdefs.IMA_SRC_SEQUENCE,
cdefs.IMA_SRC_MOVIE,
}:
return return
pathname, field = block.get(b'name', return_field=True) pathname, field = block.get(b"name", return_field=True)
is_sequence = image_source == cdefs.IMA_SRC_SEQUENCE is_sequence = image_source == cdefs.IMA_SRC_SEQUENCE
yield result.BlockUsage(block, pathname, is_sequence, path_full_field=field) yield result.BlockUsage(block, pathname, is_sequence, path_full_field=field)
@dna_code('LI') @dna_code("LI")
def library(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]: def library(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
"""Library data blocks.""" """Library data blocks."""
path, field = block.get(b'name', return_field=True) path, field = block.get(b"name", return_field=True)
yield result.BlockUsage(block, path, path_full_field=field) yield result.BlockUsage(block, path, path_full_field=field)
# The 'filepath' also points to the blend file. However, this is set to the # The 'filepath' also points to the blend file. However, this is set to the
@ -112,37 +116,37 @@ def library(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsag
# is thus not a property we have to report or rewrite. # is thus not a property we have to report or rewrite.
@dna_code('ME') @dna_code("ME")
def mesh(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]: def mesh(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
"""Mesh data blocks.""" """Mesh data blocks."""
block_external = block.get_pointer((b'ldata', b'external'), None) block_external = block.get_pointer((b"ldata", b"external"), None)
if block_external is None: if block_external is None:
block_external = block.get_pointer((b'fdata', b'external'), None) block_external = block.get_pointer((b"fdata", b"external"), None)
if block_external is None: if block_external is None:
return return
path, field = block_external.get(b'filename', return_field=True) path, field = block_external.get(b"filename", return_field=True)
yield result.BlockUsage(block, path, path_full_field=field) yield result.BlockUsage(block, path, path_full_field=field)
@dna_code('MC') @dna_code("MC")
def movie_clip(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]: def movie_clip(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
"""MovieClip data blocks.""" """MovieClip data blocks."""
path, field = block.get(b'name', return_field=True) path, field = block.get(b"name", return_field=True)
# TODO: The assumption that this is not a sequence may not be true for all modifiers. # TODO: The assumption that this is not a sequence may not be true for all modifiers.
yield result.BlockUsage(block, path, is_sequence=False, path_full_field=field) yield result.BlockUsage(block, path, is_sequence=False, path_full_field=field)
@dna_code('OB') @dna_code("OB")
def object_block(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]: def object_block(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
"""Object data blocks.""" """Object data blocks."""
ctx = modifier_walkers.ModifierContext(owner=block) ctx = modifier_walkers.ModifierContext(owner=block)
# 'ob->modifiers[...].filepath' # 'ob->modifiers[...].filepath'
for mod_idx, block_mod in enumerate(iterators.modifiers(block)): for mod_idx, block_mod in enumerate(iterators.modifiers(block)):
block_name = b'%s.modifiers[%d]' % (block.id_name, mod_idx) block_name = b"%s.modifiers[%d]" % (block.id_name, mod_idx)
mod_type = block_mod[b'modifier', b'type'] mod_type = block_mod[b"modifier", b"type"]
log.debug('Tracing modifier %s, type=%d', block_name.decode(), mod_type) log.debug("Tracing modifier %s, type=%d", block_name.decode(), mod_type)
try: try:
mod_handler = modifier_walkers.modifier_handlers[mod_type] mod_handler = modifier_walkers.modifier_handlers[mod_type]
@ -151,52 +155,59 @@ def object_block(block: blendfile.BlendFileBlock) -> typing.Iterator[result.Bloc
yield from mod_handler(ctx, block_mod, block_name) yield from mod_handler(ctx, block_mod, block_name)
@dna_code('SC') @dna_code("SC")
def scene(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]: def scene(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
"""Scene data blocks.""" """Scene data blocks."""
# Sequence editor is the only interesting bit. # Sequence editor is the only interesting bit.
block_ed = block.get_pointer(b'ed') block_ed = block.get_pointer(b"ed")
if block_ed is None: if block_ed is None:
return return
single_asset_types = {cdefs.SEQ_TYPE_MOVIE, cdefs.SEQ_TYPE_SOUND_RAM, cdefs.SEQ_TYPE_SOUND_HD} single_asset_types = {
cdefs.SEQ_TYPE_MOVIE,
cdefs.SEQ_TYPE_SOUND_RAM,
cdefs.SEQ_TYPE_SOUND_HD,
}
asset_types = single_asset_types.union({cdefs.SEQ_TYPE_IMAGE}) asset_types = single_asset_types.union({cdefs.SEQ_TYPE_IMAGE})
for seq, seq_type in iterators.sequencer_strips(block_ed): for seq, seq_type in iterators.sequencer_strips(block_ed):
if seq_type not in asset_types: if seq_type not in asset_types:
continue continue
seq_strip = seq.get_pointer(b'strip') seq_strip = seq.get_pointer(b"strip")
if seq_strip is None: if seq_strip is None:
continue continue
seq_stripdata = seq_strip.get_pointer(b'stripdata') seq_stripdata = seq_strip.get_pointer(b"stripdata")
if seq_stripdata is None: if seq_stripdata is None:
continue continue
dirname, dn_field = seq_strip.get(b'dir', return_field=True) dirname, dn_field = seq_strip.get(b"dir", return_field=True)
basename, bn_field = seq_stripdata.get(b'name', return_field=True) basename, bn_field = seq_stripdata.get(b"name", return_field=True)
asset_path = bpathlib.BlendPath(dirname) / basename asset_path = bpathlib.BlendPath(dirname) / basename
is_sequence = seq_type not in single_asset_types is_sequence = seq_type not in single_asset_types
yield result.BlockUsage(seq_strip, asset_path, yield result.BlockUsage(
is_sequence=is_sequence, seq_strip,
path_dir_field=dn_field, asset_path,
path_base_field=bn_field) is_sequence=is_sequence,
path_dir_field=dn_field,
path_base_field=bn_field,
)
@dna_code('SO') @dna_code("SO")
@skip_packed @skip_packed
def sound(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]: def sound(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
"""Sound data blocks.""" """Sound data blocks."""
path, field = block.get(b'name', return_field=True) path, field = block.get(b"name", return_field=True)
yield result.BlockUsage(block, path, path_full_field=field) yield result.BlockUsage(block, path, path_full_field=field)
@dna_code('VF') @dna_code("VF")
@skip_packed @skip_packed
def vector_font(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]: def vector_font(block: blendfile.BlendFileBlock) -> typing.Iterator[result.BlockUsage]:
"""Vector Font data blocks.""" """Vector Font data blocks."""
path, field = block.get(b'name', return_field=True) path, field = block.get(b"name", return_field=True)
if path == b'<builtin>': # builtin font if path == b"<builtin>": # builtin font
return return
yield result.BlockUsage(block, path, path_full_field=field) yield result.BlockUsage(block, path, path_full_field=field)

153
blender_asset_tracer/trace/expanders.py
View File

@ -30,23 +30,25 @@ from blender_asset_tracer import blendfile, cdefs
from blender_asset_tracer.blendfile import iterators from blender_asset_tracer.blendfile import iterators
# Don't warn about these types at all. # Don't warn about these types at all.
_warned_about_types = {b'LI', b'DATA'} _warned_about_types = {b"LI", b"DATA"}
_funcs_for_code = {} # type: typing.Dict[bytes, typing.Callable] _funcs_for_code = {} # type: typing.Dict[bytes, typing.Callable]
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
def expand_block(block: blendfile.BlendFileBlock) -> typing.Iterator[blendfile.BlendFileBlock]: def expand_block(
block: blendfile.BlendFileBlock,
) -> typing.Iterator[blendfile.BlendFileBlock]:
"""Generator, yield the data blocks used by this data block.""" """Generator, yield the data blocks used by this data block."""
try: try:
expander = _funcs_for_code[block.code] expander = _funcs_for_code[block.code]
except KeyError: except KeyError:
if block.code not in _warned_about_types: if block.code not in _warned_about_types:
log.debug('No expander implemented for block type %r', block.code.decode()) log.debug("No expander implemented for block type %r", block.code.decode())
_warned_about_types.add(block.code) _warned_about_types.add(block.code)
return return
log.debug('Expanding block %r', block) log.debug("Expanding block %r", block)
# Filter out falsy blocks, i.e. None values. # Filter out falsy blocks, i.e. None values.
# Allowing expanders to yield None makes them more consise. # Allowing expanders to yield None makes them more consise.
yield from filter(None, expander(block)) yield from filter(None, expander(block))
@ -65,83 +67,90 @@ def dna_code(block_code: str):
def _expand_generic_material(block: blendfile.BlendFileBlock): def _expand_generic_material(block: blendfile.BlendFileBlock):
array_len = block.get(b'totcol') array_len = block.get(b"totcol")
yield from block.iter_array_of_pointers(b'mat', array_len) yield from block.iter_array_of_pointers(b"mat", array_len)
def _expand_generic_mtex(block: blendfile.BlendFileBlock): def _expand_generic_mtex(block: blendfile.BlendFileBlock):
if not block.dna_type.has_field(b'mtex'): if not block.dna_type.has_field(b"mtex"):
# mtex was removed in Blender 2.8 # mtex was removed in Blender 2.8
return return
for mtex in block.iter_fixed_array_of_pointers(b'mtex'): for mtex in block.iter_fixed_array_of_pointers(b"mtex"):
yield mtex.get_pointer(b'tex') yield mtex.get_pointer(b"tex")
yield mtex.get_pointer(b'object') yield mtex.get_pointer(b"object")
def _expand_generic_nodetree(block: blendfile.BlendFileBlock): def _expand_generic_nodetree(block: blendfile.BlendFileBlock):
assert block.dna_type.dna_type_id == b'bNodeTree' assert block.dna_type.dna_type_id == b"bNodeTree"
nodes = block.get_pointer((b'nodes', b'first')) nodes = block.get_pointer((b"nodes", b"first"))
for node in iterators.listbase(nodes): for node in iterators.listbase(nodes):
if node[b'type'] == cdefs.CMP_NODE_R_LAYERS: if node[b"type"] == cdefs.CMP_NODE_R_LAYERS:
continue continue
yield node yield node
# The 'id' property points to whatever is used by the node # The 'id' property points to whatever is used by the node
# (like the image in an image texture node). # (like the image in an image texture node).
yield node.get_pointer(b'id') yield node.get_pointer(b"id")
def _expand_generic_nodetree_id(block: blendfile.BlendFileBlock): def _expand_generic_nodetree_id(block: blendfile.BlendFileBlock):
block_ntree = block.get_pointer(b'nodetree', None) block_ntree = block.get_pointer(b"nodetree", None)
if block_ntree is not None: if block_ntree is not None:
yield from _expand_generic_nodetree(block_ntree) yield from _expand_generic_nodetree(block_ntree)
def _expand_generic_animdata(block: blendfile.BlendFileBlock): def _expand_generic_animdata(block: blendfile.BlendFileBlock):
block_adt = block.get_pointer(b'adt') block_adt = block.get_pointer(b"adt")
if block_adt: if block_adt:
yield block_adt.get_pointer(b'action') yield block_adt.get_pointer(b"action")
# TODO, NLA # TODO, NLA
@dna_code('AR') @dna_code("AR")
def _expand_armature(block: blendfile.BlendFileBlock): def _expand_armature(block: blendfile.BlendFileBlock):
yield from _expand_generic_animdata(block) yield from _expand_generic_animdata(block)
@dna_code('CU') @dna_code("CU")
def _expand_curve(block: blendfile.BlendFileBlock): def _expand_curve(block: blendfile.BlendFileBlock):
yield from _expand_generic_animdata(block) yield from _expand_generic_animdata(block)
yield from _expand_generic_material(block) yield from _expand_generic_material(block)
for fieldname in (b'vfont', b'vfontb', b'vfonti', b'vfontbi', for fieldname in (
b'bevobj', b'taperobj', b'textoncurve'): b"vfont",
b"vfontb",
b"vfonti",
b"vfontbi",
b"bevobj",
b"taperobj",
b"textoncurve",
):
yield block.get_pointer(fieldname) yield block.get_pointer(fieldname)
@dna_code('GR') @dna_code("GR")
def _expand_group(block: blendfile.BlendFileBlock): def _expand_group(block: blendfile.BlendFileBlock):
log.debug('Collection/group Block: %s (name=%s)', block, block.id_name) log.debug("Collection/group Block: %s (name=%s)", block, block.id_name)
objects = block.get_pointer((b'gobject', b'first')) objects = block.get_pointer((b"gobject", b"first"))
for item in iterators.listbase(objects): for item in iterators.listbase(objects):
yield item.get_pointer(b'ob') yield item.get_pointer(b"ob")
# Recurse through child collections. # Recurse through child collections.
try: try:
children = block.get_pointer((b'children', b'first')) children = block.get_pointer((b"children", b"first"))
except KeyError: except KeyError:
# 'children' was introduced in Blender 2.8 collections # 'children' was introduced in Blender 2.8 collections
pass pass
else: else:
for child in iterators.listbase(children): for child in iterators.listbase(children):
subcoll = child.get_pointer(b'collection') subcoll = child.get_pointer(b"collection")
if subcoll is None: if subcoll is None:
continue continue
if subcoll.dna_type_id == b'ID': if subcoll.dna_type_id == b"ID":
# This issue happened while recursing a linked-in 'Hidden' # This issue happened while recursing a linked-in 'Hidden'
# collection in the Chimes set of the Spring project. Such # collection in the Chimes set of the Spring project. Such
# collections named 'Hidden' were apparently created while # collections named 'Hidden' were apparently created while
@ -150,127 +159,131 @@ def _expand_group(block: blendfile.BlendFileBlock):
yield subcoll yield subcoll
continue continue
log.debug('recursing into child collection %s (name=%r, type=%r)', log.debug(
subcoll, subcoll.id_name, subcoll.dna_type_name) "recursing into child collection %s (name=%r, type=%r)",
subcoll,
subcoll.id_name,
subcoll.dna_type_name,
)
yield from _expand_group(subcoll) yield from _expand_group(subcoll)
@dna_code('LA') @dna_code("LA")
def _expand_lamp(block: blendfile.BlendFileBlock): def _expand_lamp(block: blendfile.BlendFileBlock):
yield from _expand_generic_animdata(block) yield from _expand_generic_animdata(block)
yield from _expand_generic_nodetree_id(block) yield from _expand_generic_nodetree_id(block)
yield from _expand_generic_mtex(block) yield from _expand_generic_mtex(block)
@dna_code('MA') @dna_code("MA")
def _expand_material(block: blendfile.BlendFileBlock): def _expand_material(block: blendfile.BlendFileBlock):
yield from _expand_generic_animdata(block) yield from _expand_generic_animdata(block)
yield from _expand_generic_nodetree_id(block) yield from _expand_generic_nodetree_id(block)
yield from _expand_generic_mtex(block) yield from _expand_generic_mtex(block)
try: try:
yield block.get_pointer(b'group') yield block.get_pointer(b"group")
except KeyError: except KeyError:
# Groups were removed from Blender 2.8 # Groups were removed from Blender 2.8
pass pass
@dna_code('MB') @dna_code("MB")
def _expand_metaball(block: blendfile.BlendFileBlock): def _expand_metaball(block: blendfile.BlendFileBlock):
yield from _expand_generic_animdata(block) yield from _expand_generic_animdata(block)
yield from _expand_generic_material(block) yield from _expand_generic_material(block)
@dna_code('ME') @dna_code("ME")
def _expand_mesh(block: blendfile.BlendFileBlock): def _expand_mesh(block: blendfile.BlendFileBlock):
yield from _expand_generic_animdata(block) yield from _expand_generic_animdata(block)
yield from _expand_generic_material(block) yield from _expand_generic_material(block)
yield block.get_pointer(b'texcomesh') yield block.get_pointer(b"texcomesh")
# TODO, TexFace? - it will be slow, we could simply ignore :S # TODO, TexFace? - it will be slow, we could simply ignore :S
@dna_code('NT') @dna_code("NT")
def _expand_node_tree(block: blendfile.BlendFileBlock): def _expand_node_tree(block: blendfile.BlendFileBlock):
yield from _expand_generic_animdata(block) yield from _expand_generic_animdata(block)
yield from _expand_generic_nodetree(block) yield from _expand_generic_nodetree(block)
@dna_code('OB') @dna_code("OB")
def _expand_object(block: blendfile.BlendFileBlock): def _expand_object(block: blendfile.BlendFileBlock):
yield from _expand_generic_animdata(block) yield from _expand_generic_animdata(block)
yield from _expand_generic_material(block) yield from _expand_generic_material(block)
yield block.get_pointer(b'data') yield block.get_pointer(b"data")
if block[b'transflag'] & cdefs.OB_DUPLIGROUP: if block[b"transflag"] & cdefs.OB_DUPLIGROUP:
yield block.get_pointer(b'dup_group') yield block.get_pointer(b"dup_group")
yield block.get_pointer(b'proxy') yield block.get_pointer(b"proxy")
yield block.get_pointer(b'proxy_group') yield block.get_pointer(b"proxy_group")
# 'ob->pose->chanbase[...].custom' # 'ob->pose->chanbase[...].custom'
block_pose = block.get_pointer(b'pose') block_pose = block.get_pointer(b"pose")
if block_pose: if block_pose:
assert block_pose.dna_type.dna_type_id == b'bPose' assert block_pose.dna_type.dna_type_id == b"bPose"
# sdna_index_bPoseChannel = block_pose.file.sdna_index_from_id[b'bPoseChannel'] # sdna_index_bPoseChannel = block_pose.file.sdna_index_from_id[b'bPoseChannel']
channels = block_pose.get_pointer((b'chanbase', b'first')) channels = block_pose.get_pointer((b"chanbase", b"first"))
for pose_chan in iterators.listbase(channels): for pose_chan in iterators.listbase(channels):
yield pose_chan.get_pointer(b'custom') yield pose_chan.get_pointer(b"custom")
# Expand the objects 'ParticleSettings' via 'ob->particlesystem[...].part' # Expand the objects 'ParticleSettings' via 'ob->particlesystem[...].part'
# sdna_index_ParticleSystem = block.file.sdna_index_from_id.get(b'ParticleSystem') # sdna_index_ParticleSystem = block.file.sdna_index_from_id.get(b'ParticleSystem')
# if sdna_index_ParticleSystem is not None: # if sdna_index_ParticleSystem is not None:
psystems = block.get_pointer((b'particlesystem', b'first')) psystems = block.get_pointer((b"particlesystem", b"first"))
for psystem in iterators.listbase(psystems): for psystem in iterators.listbase(psystems):
yield psystem.get_pointer(b'part') yield psystem.get_pointer(b"part")
# Modifiers can also refer to other datablocks, which should also get expanded. # Modifiers can also refer to other datablocks, which should also get expanded.
for block_mod in iterators.modifiers(block): for block_mod in iterators.modifiers(block):
mod_type = block_mod[b'modifier', b'type'] mod_type = block_mod[b"modifier", b"type"]
# Currently only node groups are supported. If the support should expand # Currently only node groups are supported. If the support should expand
# to more types, something more intelligent than this should be made. # to more types, something more intelligent than this should be made.
if mod_type == cdefs.eModifierType_Nodes: if mod_type == cdefs.eModifierType_Nodes:
yield block_mod.get_pointer(b'node_group') yield block_mod.get_pointer(b"node_group")
@dna_code('PA') @dna_code("PA")
def _expand_particle_settings(block: blendfile.BlendFileBlock): def _expand_particle_settings(block: blendfile.BlendFileBlock):
yield from _expand_generic_animdata(block) yield from _expand_generic_animdata(block)
yield from _expand_generic_mtex(block) yield from _expand_generic_mtex(block)
block_ren_as = block[b'ren_as'] block_ren_as = block[b"ren_as"]
if block_ren_as == cdefs.PART_DRAW_GR: if block_ren_as == cdefs.PART_DRAW_GR:
yield block.get_pointer(b'dup_group') yield block.get_pointer(b"dup_group")
elif block_ren_as == cdefs.PART_DRAW_OB: elif block_ren_as == cdefs.PART_DRAW_OB:
yield block.get_pointer(b'dup_ob') yield block.get_pointer(b"dup_ob")
@dna_code('SC') @dna_code("SC")
def _expand_scene(block: blendfile.BlendFileBlock): def _expand_scene(block: blendfile.BlendFileBlock):
yield from _expand_generic_animdata(block) yield from _expand_generic_animdata(block)
yield from _expand_generic_nodetree_id(block) yield from _expand_generic_nodetree_id(block)
yield block.get_pointer(b'camera') yield block.get_pointer(b"camera")
yield block.get_pointer(b'world') yield block.get_pointer(b"world")
yield block.get_pointer(b'set', default=None) yield block.get_pointer(b"set", default=None)
yield block.get_pointer(b'clip', default=None) yield block.get_pointer(b"clip", default=None)
# sdna_index_Base = block.file.sdna_index_from_id[b'Base'] # sdna_index_Base = block.file.sdna_index_from_id[b'Base']
# for item in bf_utils.iter_ListBase(block.get_pointer((b'base', b'first'))): # for item in bf_utils.iter_ListBase(block.get_pointer((b'base', b'first'))):
# yield item.get_pointer(b'object', sdna_index_refine=sdna_index_Base) # yield item.get_pointer(b'object', sdna_index_refine=sdna_index_Base)
bases = block.get_pointer((b'base', b'first')) bases = block.get_pointer((b"base", b"first"))
for base in iterators.listbase(bases): for base in iterators.listbase(bases):
yield base.get_pointer(b'object') yield base.get_pointer(b"object")
# Sequence Editor # Sequence Editor
block_ed = block.get_pointer(b'ed') block_ed = block.get_pointer(b"ed")
if not block_ed: if not block_ed:
return return
strip_type_to_field = { strip_type_to_field = {
cdefs.SEQ_TYPE_SCENE: b'scene', cdefs.SEQ_TYPE_SCENE: b"scene",
cdefs.SEQ_TYPE_MOVIECLIP: b'clip', cdefs.SEQ_TYPE_MOVIECLIP: b"clip",
cdefs.SEQ_TYPE_MASK: b'mask', cdefs.SEQ_TYPE_MASK: b"mask",
cdefs.SEQ_TYPE_SOUND_RAM: b'sound', cdefs.SEQ_TYPE_SOUND_RAM: b"sound",
} }
for strip, strip_type in iterators.sequencer_strips(block_ed): for strip, strip_type in iterators.sequencer_strips(block_ed):
try: try:
@ -280,14 +293,14 @@ def _expand_scene(block: blendfile.BlendFileBlock):
yield strip.get_pointer(field_name) yield strip.get_pointer(field_name)
@dna_code('TE') @dna_code("TE")
def _expand_texture(block: blendfile.BlendFileBlock): def _expand_texture(block: blendfile.BlendFileBlock):
yield from _expand_generic_animdata(block) yield from _expand_generic_animdata(block)
yield from _expand_generic_nodetree_id(block) yield from _expand_generic_nodetree_id(block)
yield block.get_pointer(b'ima') yield block.get_pointer(b"ima")
@dna_code('WO') @dna_code("WO")
def _expand_world(block: blendfile.BlendFileBlock): def _expand_world(block: blendfile.BlendFileBlock):
yield from _expand_generic_animdata(block) yield from _expand_generic_animdata(block)
yield from _expand_generic_nodetree_id(block) yield from _expand_generic_nodetree_id(block)

43
blender_asset_tracer/trace/file2blocks.py
View File

@ -65,14 +65,15 @@ class BlockIterator:
self.progress_cb = progress.Callback() self.progress_cb = progress.Callback()
def iter_blocks(self, def iter_blocks(
bfile: blendfile.BlendFile, self,
limit_to: typing.Set[blendfile.BlendFileBlock] = set(), bfile: blendfile.BlendFile,
) -> typing.Iterator[blendfile.BlendFileBlock]: limit_to: typing.Set[blendfile.BlendFileBlock] = set(),
) -> typing.Iterator[blendfile.BlendFileBlock]:
"""Expand blocks with dependencies from other libraries.""" """Expand blocks with dependencies from other libraries."""
self.progress_cb.trace_blendfile(bfile.filepath) self.progress_cb.trace_blendfile(bfile.filepath)
log.info('inspecting: %s', bfile.filepath) log.info("inspecting: %s", bfile.filepath)
if limit_to: if limit_to:
self._queue_named_blocks(bfile, limit_to) self._queue_named_blocks(bfile, limit_to)
else: else:
@ -94,14 +95,14 @@ class BlockIterator:
if (bpath, block.addr_old) in self.blocks_yielded: if (bpath, block.addr_old) in self.blocks_yielded:
continue continue
if block.code == b'ID': if block.code == b"ID":
# ID blocks represent linked-in assets. Those are the ones that # ID blocks represent linked-in assets. Those are the ones that
# should be loaded from their own blend file and "expanded" to # should be loaded from their own blend file and "expanded" to
# the entire set of data blocks required to render them. We # the entire set of data blocks required to render them. We
# defer the handling of those so that we can work with one # defer the handling of those so that we can work with one
# blend file at a time. # blend file at a time.
lib = block.get_pointer(b'lib') lib = block.get_pointer(b"lib")
lib_bpath = bpathlib.BlendPath(lib[b'name']).absolute(root_dir) lib_bpath = bpathlib.BlendPath(lib[b"name"]).absolute(root_dir)
blocks_per_lib[lib_bpath].add(block) blocks_per_lib[lib_bpath].add(block)
# The library block itself should also be reported, because it # The library block itself should also be reported, because it
@ -126,25 +127,25 @@ class BlockIterator:
lib_path = bpathlib.make_absolute(lib_bpath.to_path()) lib_path = bpathlib.make_absolute(lib_bpath.to_path())
if not lib_path.exists(): if not lib_path.exists():
log.warning('Library %s does not exist', lib_path) log.warning("Library %s does not exist", lib_path)
continue continue
log.debug('Expanding %d blocks in %s', len(idblocks), lib_path) log.debug("Expanding %d blocks in %s", len(idblocks), lib_path)
libfile = blendfile.open_cached(lib_path) libfile = blendfile.open_cached(lib_path)
yield from self.iter_blocks(libfile, idblocks) yield from self.iter_blocks(libfile, idblocks)
def _queue_all_blocks(self, bfile: blendfile.BlendFile): def _queue_all_blocks(self, bfile: blendfile.BlendFile):
log.debug('Queueing all blocks from file %s', bfile.filepath) log.debug("Queueing all blocks from file %s", bfile.filepath)
for block in bfile.blocks: for block in bfile.blocks:
# Don't bother visiting DATA blocks, as we won't know what # Don't bother visiting DATA blocks, as we won't know what
# to do with them anyway. # to do with them anyway.
if block.code == b'DATA': if block.code == b"DATA":
continue continue
self.to_visit.put(block) self.to_visit.put(block)
def _queue_named_blocks(self, def _queue_named_blocks(
bfile: blendfile.BlendFile, self, bfile: blendfile.BlendFile, limit_to: typing.Set[blendfile.BlendFileBlock]
limit_to: typing.Set[blendfile.BlendFileBlock]): ):
"""Queue only the blocks referred to in limit_to. """Queue only the blocks referred to in limit_to.
:param bfile: :param bfile:
@ -154,14 +155,14 @@ class BlockIterator:
""" """
for to_find in limit_to: for to_find in limit_to:
assert to_find.code == b'ID' assert to_find.code == b"ID"
name_to_find = to_find[b'name'] name_to_find = to_find[b"name"]
code = name_to_find[:2] code = name_to_find[:2]
log.debug('Finding block %r with code %r', name_to_find, code) log.debug("Finding block %r with code %r", name_to_find, code)
same_code = bfile.find_blocks_from_code(code) same_code = bfile.find_blocks_from_code(code)
for block in same_code: for block in same_code:
if block.id_name == name_to_find: if block.id_name == name_to_find:
log.debug('Queueing %r from file %s', block, bfile.filepath) log.debug("Queueing %r from file %s", block, bfile.filepath)
self.to_visit.put(block) self.to_visit.put(block)
def _queue_dependencies(self, block: blendfile.BlendFileBlock): def _queue_dependencies(self, block: blendfile.BlendFileBlock):
@ -169,7 +170,9 @@ class BlockIterator:
self.to_visit.put(block) self.to_visit.put(block)
def iter_blocks(bfile: blendfile.BlendFile) -> typing.Iterator[blendfile.BlendFileBlock]: def iter_blocks(
bfile: blendfile.BlendFile,
) -> typing.Iterator[blendfile.BlendFileBlock]:
"""Generator, yield all blocks in this file + required blocks in libs.""" """Generator, yield all blocks in this file + required blocks in libs."""
bi = BlockIterator() bi = BlockIterator()
yield from bi.iter_blocks(bfile) yield from bi.iter_blocks(bfile)

10
blender_asset_tracer/trace/file_sequence.py
View File

@ -39,9 +39,10 @@ def expand_sequence(path: pathlib.Path) -> typing.Iterator[pathlib.Path]:
or the path of the first file in the sequence. or the path of the first file in the sequence.
""" """
if '*' in str(path): # assume it is a glob if "*" in str(path): # assume it is a glob
import glob import glob
log.debug('expanding glob %s', path)
log.debug("expanding glob %s", path)
for fname in sorted(glob.glob(str(path), recursive=True)): for fname in sorted(glob.glob(str(path), recursive=True)):
yield pathlib.Path(fname) yield pathlib.Path(fname)
return return
@ -53,9 +54,10 @@ def expand_sequence(path: pathlib.Path) -> typing.Iterator[pathlib.Path]:
yield path yield path
return return
log.debug('expanding file sequence %s', path) log.debug("expanding file sequence %s", path)
import string import string
stem_no_digits = path.stem.rstrip(string.digits) stem_no_digits = path.stem.rstrip(string.digits)
if stem_no_digits == path.stem: if stem_no_digits == path.stem:
# Just a single file, no digits here. # Just a single file, no digits here.
@ -65,5 +67,5 @@ def expand_sequence(path: pathlib.Path) -> typing.Iterator[pathlib.Path]:
# Return everything start starts with 'stem_no_digits' and ends with the # Return everything start starts with 'stem_no_digits' and ends with the
# same suffix as the first file. This may result in more files than used # same suffix as the first file. This may result in more files than used
# by Blender, but at least it shouldn't miss any. # by Blender, but at least it shouldn't miss any.
pattern = '%s*%s' % (stem_no_digits, path.suffix) pattern = "%s*%s" % (stem_no_digits, path.suffix)
yield from sorted(path.parent.glob(pattern)) yield from sorted(path.parent.glob(pattern))

208
blender_asset_tracer/trace/modifier_walkers.py
View File

@ -37,8 +37,9 @@ class ModifierContext:
Currently just contains the object on which the modifier is defined. Currently just contains the object on which the modifier is defined.
""" """
def __init__(self, owner: blendfile.BlendFileBlock) -> None: def __init__(self, owner: blendfile.BlendFileBlock) -> None:
assert owner.dna_type_name == 'Object' assert owner.dna_type_name == "Object"
self.owner = owner self.owner = owner
@ -55,45 +56,56 @@ def mod_handler(dna_num: int):
@mod_handler(cdefs.eModifierType_MeshCache) @mod_handler(cdefs.eModifierType_MeshCache)
def modifier_filepath(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes) \ def modifier_filepath(
-> typing.Iterator[result.BlockUsage]: ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes
) -> typing.Iterator[result.BlockUsage]:
"""Just yield the 'filepath' field.""" """Just yield the 'filepath' field."""
path, field = modifier.get(b'filepath', return_field=True) path, field = modifier.get(b"filepath", return_field=True)
yield result.BlockUsage(modifier, path, path_full_field=field, block_name=block_name) yield result.BlockUsage(
modifier, path, path_full_field=field, block_name=block_name
)
@mod_handler(cdefs.eModifierType_MeshSequenceCache) @mod_handler(cdefs.eModifierType_MeshSequenceCache)
def modifier_mesh_sequence_cache(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, def modifier_mesh_sequence_cache(
block_name: bytes) -> typing.Iterator[result.BlockUsage]: ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes
) -> typing.Iterator[result.BlockUsage]:
"""Yield the Alembic file(s) used by this modifier""" """Yield the Alembic file(s) used by this modifier"""
cache_file = modifier.get_pointer(b'cache_file') cache_file = modifier.get_pointer(b"cache_file")
if cache_file is None: if cache_file is None:
return return
is_sequence = bool(cache_file[b'is_sequence']) is_sequence = bool(cache_file[b"is_sequence"])
cache_block_name = cache_file.id_name cache_block_name = cache_file.id_name
assert cache_block_name is not None assert cache_block_name is not None
path, field = cache_file.get(b'filepath', return_field=True) path, field = cache_file.get(b"filepath", return_field=True)
yield result.BlockUsage(cache_file, path, path_full_field=field, yield result.BlockUsage(
is_sequence=is_sequence, cache_file,
block_name=cache_block_name) path,
path_full_field=field,
is_sequence=is_sequence,
block_name=cache_block_name,
)
@mod_handler(cdefs.eModifierType_Ocean) @mod_handler(cdefs.eModifierType_Ocean)
def modifier_ocean(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes) \ def modifier_ocean(
-> typing.Iterator[result.BlockUsage]: ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes
if not modifier[b'cached']: ) -> typing.Iterator[result.BlockUsage]:
if not modifier[b"cached"]:
return return
path, field = modifier.get(b'cachepath', return_field=True) path, field = modifier.get(b"cachepath", return_field=True)
# The path indicates the directory containing the cached files. # The path indicates the directory containing the cached files.
yield result.BlockUsage(modifier, path, is_sequence=True, path_full_field=field, yield result.BlockUsage(
block_name=block_name) modifier, path, is_sequence=True, path_full_field=field, block_name=block_name
)
def _get_texture(prop_name: bytes, dblock: blendfile.BlendFileBlock, block_name: bytes) \ def _get_texture(
-> typing.Iterator[result.BlockUsage]: prop_name: bytes, dblock: blendfile.BlendFileBlock, block_name: bytes
) -> typing.Iterator[result.BlockUsage]:
"""Yield block usages from a texture propery. """Yield block usages from a texture propery.
Assumes dblock[prop_name] is a texture data block. Assumes dblock[prop_name] is a texture data block.
@ -102,13 +114,14 @@ def _get_texture(prop_name: bytes, dblock: blendfile.BlendFileBlock, block_name:
return return
tx = dblock.get_pointer(prop_name) tx = dblock.get_pointer(prop_name)
yield from _get_image(b'ima', tx, block_name) yield from _get_image(b"ima", tx, block_name)
def _get_image(prop_name: bytes, def _get_image(
dblock: typing.Optional[blendfile.BlendFileBlock], prop_name: bytes,
block_name: bytes) \ dblock: typing.Optional[blendfile.BlendFileBlock],
-> typing.Iterator[result.BlockUsage]: block_name: bytes,
) -> typing.Iterator[result.BlockUsage]:
"""Yield block usages from an image propery. """Yield block usages from an image propery.
Assumes dblock[prop_name] is an image data block. Assumes dblock[prop_name] is an image data block.
@ -120,132 +133,159 @@ def _get_image(prop_name: bytes,
ima = dblock.get_pointer(prop_name) ima = dblock.get_pointer(prop_name)
except KeyError as ex: except KeyError as ex:
# No such property, just return. # No such property, just return.
log.debug('_get_image() called with non-existing property name: %s', ex) log.debug("_get_image() called with non-existing property name: %s", ex)
return return
if not ima: if not ima:
return return
path, field = ima.get(b'name', return_field=True) path, field = ima.get(b"name", return_field=True)
yield result.BlockUsage(ima, path, path_full_field=field, block_name=block_name) yield result.BlockUsage(ima, path, path_full_field=field, block_name=block_name)
@mod_handler(cdefs.eModifierType_Displace) @mod_handler(cdefs.eModifierType_Displace)
@mod_handler(cdefs.eModifierType_Wave) @mod_handler(cdefs.eModifierType_Wave)
def modifier_texture(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes) \ def modifier_texture(
-> typing.Iterator[result.BlockUsage]: ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes
return _get_texture(b'texture', modifier, block_name) ) -> typing.Iterator[result.BlockUsage]:
return _get_texture(b"texture", modifier, block_name)
@mod_handler(cdefs.eModifierType_WeightVGEdit) @mod_handler(cdefs.eModifierType_WeightVGEdit)
@mod_handler(cdefs.eModifierType_WeightVGMix) @mod_handler(cdefs.eModifierType_WeightVGMix)
@mod_handler(cdefs.eModifierType_WeightVGProximity) @mod_handler(cdefs.eModifierType_WeightVGProximity)
def modifier_mask_texture(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, def modifier_mask_texture(
block_name: bytes) \ ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes
-> typing.Iterator[result.BlockUsage]: ) -> typing.Iterator[result.BlockUsage]:
return _get_texture(b'mask_texture', modifier, block_name) return _get_texture(b"mask_texture", modifier, block_name)
@mod_handler(cdefs.eModifierType_UVProject) @mod_handler(cdefs.eModifierType_UVProject)
def modifier_image(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes) \ def modifier_image(
-> typing.Iterator[result.BlockUsage]: ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes
yield from _get_image(b'image', modifier, block_name) ) -> typing.Iterator[result.BlockUsage]:
yield from _get_image(b"image", modifier, block_name)
def _walk_point_cache(ctx: ModifierContext, def _walk_point_cache(
block_name: bytes, ctx: ModifierContext,
bfile: blendfile.BlendFile, block_name: bytes,
pointcache: blendfile.BlendFileBlock, bfile: blendfile.BlendFile,
extension: bytes): pointcache: blendfile.BlendFileBlock,
flag = pointcache[b'flag'] extension: bytes,
):
flag = pointcache[b"flag"]
if flag & cdefs.PTCACHE_EXTERNAL: if flag & cdefs.PTCACHE_EXTERNAL:
path, field = pointcache.get(b'path', return_field=True) path, field = pointcache.get(b"path", return_field=True)
log.info(' external cache at %s', path) log.info(" external cache at %s", path)
bpath = bpathlib.BlendPath(path) bpath = bpathlib.BlendPath(path)
yield result.BlockUsage(pointcache, bpath, path_full_field=field, yield result.BlockUsage(
is_sequence=True, block_name=block_name) pointcache,
bpath,
path_full_field=field,
is_sequence=True,
block_name=block_name,
)
elif flag & cdefs.PTCACHE_DISK_CACHE: elif flag & cdefs.PTCACHE_DISK_CACHE:
# See ptcache_path() in pointcache.c # See ptcache_path() in pointcache.c
name, field = pointcache.get(b'name', return_field=True) name, field = pointcache.get(b"name", return_field=True)
if not name: if not name:
# See ptcache_filename() in pointcache.c # See ptcache_filename() in pointcache.c
idname = ctx.owner[b'id', b'name'] idname = ctx.owner[b"id", b"name"]
name = idname[2:].hex().upper().encode() name = idname[2:].hex().upper().encode()
path = b'//%b%b/%b_*%b' % ( path = b"//%b%b/%b_*%b" % (
cdefs.PTCACHE_PATH, cdefs.PTCACHE_PATH,
bfile.filepath.stem.encode(), bfile.filepath.stem.encode(),
name, name,
extension) extension,
log.info(' disk cache at %s', path) )
log.info(" disk cache at %s", path)
bpath = bpathlib.BlendPath(path) bpath = bpathlib.BlendPath(path)
yield result.BlockUsage(pointcache, bpath, path_full_field=field, yield result.BlockUsage(
is_sequence=True, block_name=block_name) pointcache,
bpath,
path_full_field=field,
is_sequence=True,
block_name=block_name,
)
@mod_handler(cdefs.eModifierType_ParticleSystem) @mod_handler(cdefs.eModifierType_ParticleSystem)
def modifier_particle_system(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, def modifier_particle_system(
block_name: bytes) \ ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes
-> typing.Iterator[result.BlockUsage]: ) -> typing.Iterator[result.BlockUsage]:
psys = modifier.get_pointer(b'psys') psys = modifier.get_pointer(b"psys")
if psys is None: if psys is None:
return return
pointcache = psys.get_pointer(b'pointcache') pointcache = psys.get_pointer(b"pointcache")
if pointcache is None: if pointcache is None:
return return
yield from _walk_point_cache(ctx, block_name, modifier.bfile, pointcache, cdefs.PTCACHE_EXT) yield from _walk_point_cache(
ctx, block_name, modifier.bfile, pointcache, cdefs.PTCACHE_EXT
)
@mod_handler(cdefs.eModifierType_Fluidsim) @mod_handler(cdefs.eModifierType_Fluidsim)
def modifier_fluid_sim(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes) \ def modifier_fluid_sim(
-> typing.Iterator[result.BlockUsage]: ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes
my_log = log.getChild('modifier_fluid_sim') ) -> typing.Iterator[result.BlockUsage]:
my_log = log.getChild("modifier_fluid_sim")
fss = modifier.get_pointer(b'fss') fss = modifier.get_pointer(b"fss")
if fss is None: if fss is None:
my_log.debug('Modifier %r (%r) has no fss', my_log.debug(
modifier[b'modifier', b'name'], block_name) "Modifier %r (%r) has no fss", modifier[b"modifier", b"name"], block_name
)
return return
path, field = fss.get(b'surfdataPath', return_field=True) path, field = fss.get(b"surfdataPath", return_field=True)
# This may match more than is used by Blender, but at least it shouldn't # This may match more than is used by Blender, but at least it shouldn't
# miss any files. # miss any files.
# The 'fluidsurface' prefix is defined in source/blender/makesdna/DNA_object_fluidsim_types.h # The 'fluidsurface' prefix is defined in source/blender/makesdna/DNA_object_fluidsim_types.h
bpath = bpathlib.BlendPath(path) bpath = bpathlib.BlendPath(path)
yield result.BlockUsage(fss, bpath, path_full_field=field, yield result.BlockUsage(
is_sequence=True, block_name=block_name) fss, bpath, path_full_field=field, is_sequence=True, block_name=block_name
)
@mod_handler(cdefs.eModifierType_Smokesim) @mod_handler(cdefs.eModifierType_Smokesim)
def modifier_smoke_sim(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes) \ def modifier_smoke_sim(
-> typing.Iterator[result.BlockUsage]: ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes
my_log = log.getChild('modifier_smoke_sim') ) -> typing.Iterator[result.BlockUsage]:
my_log = log.getChild("modifier_smoke_sim")
domain = modifier.get_pointer(b'domain') domain = modifier.get_pointer(b"domain")
if domain is None: if domain is None:
my_log.debug('Modifier %r (%r) has no domain', my_log.debug(
modifier[b'modifier', b'name'], block_name) "Modifier %r (%r) has no domain", modifier[b"modifier", b"name"], block_name
)
return return
pointcache = domain.get_pointer(b'point_cache') pointcache = domain.get_pointer(b"point_cache")
if pointcache is None: if pointcache is None:
return return
format = domain.get(b'cache_file_format') format = domain.get(b"cache_file_format")
extensions = { extensions = {
cdefs.PTCACHE_FILE_PTCACHE: cdefs.PTCACHE_EXT, cdefs.PTCACHE_FILE_PTCACHE: cdefs.PTCACHE_EXT,
cdefs.PTCACHE_FILE_OPENVDB: cdefs.PTCACHE_EXT_VDB cdefs.PTCACHE_FILE_OPENVDB: cdefs.PTCACHE_EXT_VDB,
} }
yield from _walk_point_cache(ctx, block_name, modifier.bfile, pointcache, extensions[format]) yield from _walk_point_cache(
ctx, block_name, modifier.bfile, pointcache, extensions[format]
)
@mod_handler(cdefs.eModifierType_Cloth) @mod_handler(cdefs.eModifierType_Cloth)
def modifier_cloth(ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes) \ def modifier_cloth(
-> typing.Iterator[result.BlockUsage]: ctx: ModifierContext, modifier: blendfile.BlendFileBlock, block_name: bytes
pointcache = modifier.get_pointer(b'point_cache') ) -> typing.Iterator[result.BlockUsage]:
pointcache = modifier.get_pointer(b"point_cache")
if pointcache is None: if pointcache is None:
return return
yield from _walk_point_cache(ctx, block_name, modifier.bfile, pointcache, cdefs.PTCACHE_EXT) yield from _walk_point_cache(
ctx, block_name, modifier.bfile, pointcache, cdefs.PTCACHE_EXT
)

87
blender_asset_tracer/trace/result.py
View File

@ -52,32 +52,37 @@ class BlockUsage:
:ivar path_base_field: field containing the basename of this asset. :ivar path_base_field: field containing the basename of this asset.
""" """
def __init__(self, def __init__(
block: blendfile.BlendFileBlock, self,
asset_path: bpathlib.BlendPath, block: blendfile.BlendFileBlock,
is_sequence: bool = False, asset_path: bpathlib.BlendPath,
path_full_field: dna.Field = None, is_sequence: bool = False,
path_dir_field: dna.Field = None, path_full_field: dna.Field = None,
path_base_field: dna.Field = None, path_dir_field: dna.Field = None,
block_name: bytes = b'', path_base_field: dna.Field = None,
) -> None: block_name: bytes = b"",
) -> None:
if block_name: if block_name:
self.block_name = block_name self.block_name = block_name
else: else:
self.block_name = self.guess_block_name(block) self.block_name = self.guess_block_name(block)
assert isinstance(block, blendfile.BlendFileBlock) assert isinstance(block, blendfile.BlendFileBlock)
assert isinstance(asset_path, (bytes, bpathlib.BlendPath)), \ assert isinstance(
'asset_path should be BlendPath, not %r' % type(asset_path) asset_path, (bytes, bpathlib.BlendPath)
), "asset_path should be BlendPath, not %r" % type(asset_path)
if path_full_field is None: if path_full_field is None:
assert isinstance(path_dir_field, dna.Field), \ assert isinstance(
'path_dir_field should be dna.Field, not %r' % type(path_dir_field) path_dir_field, dna.Field
assert isinstance(path_base_field, dna.Field), \ ), "path_dir_field should be dna.Field, not %r" % type(path_dir_field)
'path_base_field should be dna.Field, not %r' % type(path_base_field) assert isinstance(
path_base_field, dna.Field
), "path_base_field should be dna.Field, not %r" % type(path_base_field)
else: else:
assert isinstance(path_full_field, dna.Field), \ assert isinstance(
'path_full_field should be dna.Field, not %r' % type(path_full_field) path_full_field, dna.Field
), "path_full_field should be dna.Field, not %r" % type(path_full_field)
if isinstance(asset_path, bytes): if isinstance(asset_path, bytes):
asset_path = bpathlib.BlendPath(asset_path) asset_path = bpathlib.BlendPath(asset_path)
@ -95,26 +100,30 @@ class BlockUsage:
@staticmethod @staticmethod
def guess_block_name(block: blendfile.BlendFileBlock) -> bytes: def guess_block_name(block: blendfile.BlendFileBlock) -> bytes:
try: try:
return block[b'id', b'name'] return block[b"id", b"name"]
except KeyError: except KeyError:
pass pass
try: try:
return block[b'name'] return block[b"name"]
except KeyError: except KeyError:
pass pass
return b'-unnamed-' return b"-unnamed-"
def __repr__(self): def __repr__(self):
if self.path_full_field is None: if self.path_full_field is None:
field_name = self.path_dir_field.name.name_full.decode() + \ field_name = (
'/' + \ self.path_dir_field.name.name_full.decode()
self.path_base_field.name.name_full.decode() + "/"
+ self.path_base_field.name.name_full.decode()
)
else: else:
field_name = self.path_full_field.name.name_full.decode() field_name = self.path_full_field.name.name_full.decode()
return '<BlockUsage name=%r type=%r field=%r asset=%r%s>' % ( return "<BlockUsage name=%r type=%r field=%r asset=%r%s>" % (
self.block_name, self.block.dna_type_name, self.block_name,
field_name, self.asset_path, self.block.dna_type_name,
' sequence' if self.is_sequence else '' field_name,
self.asset_path,
" sequence" if self.is_sequence else "",
) )
def files(self) -> typing.Iterator[pathlib.Path]: def files(self) -> typing.Iterator[pathlib.Path]:
@ -130,7 +139,7 @@ class BlockUsage:
path = self.__fspath__() path = self.__fspath__()
if not self.is_sequence: if not self.is_sequence:
if not path.exists(): if not path.exists():
log.warning('Path %s does not exist for %s', path, self) log.warning("Path %s does not exist for %s", path, self)
return return
yield path yield path
return return
@ -138,14 +147,18 @@ class BlockUsage:
try: try:
yield from file_sequence.expand_sequence(path) yield from file_sequence.expand_sequence(path)
except file_sequence.DoesNotExist: except file_sequence.DoesNotExist:
log.warning('Path %s does not exist for %s', path, self) log.warning("Path %s does not exist for %s", path, self)
def __fspath__(self) -> pathlib.Path: def __fspath__(self) -> pathlib.Path:
"""Determine the absolute path of the asset on the filesystem.""" """Determine the absolute path of the asset on the filesystem."""
if self._abspath is None: if self._abspath is None:
bpath = self.block.bfile.abspath(self.asset_path) bpath = self.block.bfile.abspath(self.asset_path)
log.info('Resolved %s rel to %s -> %s', log.info(
self.asset_path, self.block.bfile.filepath, bpath) "Resolved %s rel to %s -> %s",
self.asset_path,
self.block.bfile.filepath,
bpath,
)
as_path = pathlib.Path(bpath.to_path()) as_path = pathlib.Path(bpath.to_path())
@ -159,15 +172,19 @@ class BlockUsage:
else: else:
self._abspath = abs_parent / as_path.name self._abspath = abs_parent / as_path.name
log.info('Resolving %s rel to %s -> %s', log.info(
self.asset_path, self.block.bfile.filepath, self._abspath) "Resolving %s rel to %s -> %s",
self.asset_path,
self.block.bfile.filepath,
self._abspath,
)
else: else:
log.info('Reusing abspath %s', self._abspath) log.info("Reusing abspath %s", self._abspath)
return self._abspath return self._abspath
abspath = property(__fspath__) abspath = property(__fspath__)
def __lt__(self, other: 'BlockUsage'): def __lt__(self, other: "BlockUsage"):
"""Allow sorting for repeatable and predictable unit tests.""" """Allow sorting for repeatable and predictable unit tests."""
if not isinstance(other, BlockUsage): if not isinstance(other, BlockUsage):
raise NotImplemented() raise NotImplemented()

6
tests/abstract_test.py
View File

@ -24,14 +24,14 @@ import unittest
from blender_asset_tracer import blendfile from blender_asset_tracer import blendfile
logging.basicConfig( logging.basicConfig(
format='%(asctime)-15s %(levelname)8s %(name)s %(message)s', format="%(asctime)-15s %(levelname)8s %(name)s %(message)s", level=logging.INFO
level=logging.INFO) )
class AbstractBlendFileTest(unittest.TestCase): class AbstractBlendFileTest(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls.blendfiles = pathlib.Path(__file__).with_name('blendfiles') cls.blendfiles = pathlib.Path(__file__).with_name("blendfiles")
def setUp(self): def setUp(self):
self.bf = None self.bf = None

179
tests/test_blendfile_dna.py
View File

@ -8,66 +8,72 @@ from blender_asset_tracer.blendfile import dna, dna_io
class NameTest(unittest.TestCase): class NameTest(unittest.TestCase):
def test_simple_name(self): def test_simple_name(self):
n = dna.Name(b'Suzanne') n = dna.Name(b"Suzanne")
self.assertEqual(n.name_full, b'Suzanne') self.assertEqual(n.name_full, b"Suzanne")
self.assertEqual(n.name_only, b'Suzanne') self.assertEqual(n.name_only, b"Suzanne")
self.assertFalse(n.is_pointer) self.assertFalse(n.is_pointer)
self.assertFalse(n.is_method_pointer) self.assertFalse(n.is_method_pointer)
self.assertEqual(n.array_size, 1) self.assertEqual(n.array_size, 1)
def test_pointer(self): def test_pointer(self):
n = dna.Name(b'*marker') n = dna.Name(b"*marker")
self.assertEqual(n.name_full, b'*marker') self.assertEqual(n.name_full, b"*marker")
self.assertEqual(n.name_only, b'marker') self.assertEqual(n.name_only, b"marker")
self.assertTrue(n.is_pointer) self.assertTrue(n.is_pointer)
self.assertFalse(n.is_method_pointer) self.assertFalse(n.is_method_pointer)
self.assertEqual(n.array_size, 1) self.assertEqual(n.array_size, 1)
def test_method_pointer(self): def test_method_pointer(self):
n = dna.Name(b'(*delta_cache)()') n = dna.Name(b"(*delta_cache)()")
self.assertEqual(n.name_full, b'(*delta_cache)()') self.assertEqual(n.name_full, b"(*delta_cache)()")
self.assertEqual(n.name_only, b'delta_cache') self.assertEqual(n.name_only, b"delta_cache")
self.assertTrue(n.is_pointer) self.assertTrue(n.is_pointer)
self.assertTrue(n.is_method_pointer) self.assertTrue(n.is_method_pointer)
self.assertEqual(n.array_size, 1) self.assertEqual(n.array_size, 1)
def test_simple_array(self): def test_simple_array(self):
n = dna.Name(b'flame_smoke_color[3]') n = dna.Name(b"flame_smoke_color[3]")
self.assertEqual(n.name_full, b'flame_smoke_color[3]') self.assertEqual(n.name_full, b"flame_smoke_color[3]")
self.assertEqual(n.name_only, b'flame_smoke_color') self.assertEqual(n.name_only, b"flame_smoke_color")
self.assertFalse(n.is_pointer) self.assertFalse(n.is_pointer)
self.assertFalse(n.is_method_pointer) self.assertFalse(n.is_method_pointer)
self.assertEqual(n.array_size, 3) self.assertEqual(n.array_size, 3)
def test_nested_array(self): def test_nested_array(self):
n = dna.Name(b'pattern_corners[4][2]') n = dna.Name(b"pattern_corners[4][2]")
self.assertEqual(n.name_full, b'pattern_corners[4][2]') self.assertEqual(n.name_full, b"pattern_corners[4][2]")
self.assertEqual(n.name_only, b'pattern_corners') self.assertEqual(n.name_only, b"pattern_corners")
self.assertFalse(n.is_pointer) self.assertFalse(n.is_pointer)
self.assertFalse(n.is_method_pointer) self.assertFalse(n.is_method_pointer)
self.assertEqual(n.array_size, 8) self.assertEqual(n.array_size, 8)
def test_pointer_array(self): def test_pointer_array(self):
n = dna.Name(b'*mtex[18]') n = dna.Name(b"*mtex[18]")
self.assertEqual(n.name_full, b'*mtex[18]') self.assertEqual(n.name_full, b"*mtex[18]")
self.assertEqual(n.name_only, b'mtex') self.assertEqual(n.name_only, b"mtex")
self.assertTrue(n.is_pointer) self.assertTrue(n.is_pointer)
self.assertFalse(n.is_method_pointer) self.assertFalse(n.is_method_pointer)
self.assertEqual(n.array_size, 18) self.assertEqual(n.array_size, 18)
def test_repr(self): def test_repr(self):
self.assertEqual(repr(dna.Name(b'Suzanne')), "Name(b'Suzanne')") self.assertEqual(repr(dna.Name(b"Suzanne")), "Name(b'Suzanne')")
self.assertEqual(repr(dna.Name(b'*marker')), "Name(b'*marker')") self.assertEqual(repr(dna.Name(b"*marker")), "Name(b'*marker')")
self.assertEqual(repr(dna.Name(b'(*delta_cache)()')), "Name(b'(*delta_cache)()')") self.assertEqual(
self.assertEqual(repr(dna.Name(b'flame_smoke_color[3]')), "Name(b'flame_smoke_color[3]')") repr(dna.Name(b"(*delta_cache)()")), "Name(b'(*delta_cache)()')"
self.assertEqual(repr(dna.Name(b'pattern_corners[4][2]')), "Name(b'pattern_corners[4][2]')") )
self.assertEqual(repr(dna.Name(b'*mtex[18]')), "Name(b'*mtex[18]')") self.assertEqual(
repr(dna.Name(b"flame_smoke_color[3]")), "Name(b'flame_smoke_color[3]')"
)
self.assertEqual(
repr(dna.Name(b"pattern_corners[4][2]")), "Name(b'pattern_corners[4][2]')"
)
self.assertEqual(repr(dna.Name(b"*mtex[18]")), "Name(b'*mtex[18]')")
def test_as_reference(self): def test_as_reference(self):
n = dna.Name(b'(*delta_cache)()') n = dna.Name(b"(*delta_cache)()")
self.assertEqual(n.as_reference(None), b'delta_cache') self.assertEqual(n.as_reference(None), b"delta_cache")
self.assertEqual(n.as_reference(b''), b'delta_cache') self.assertEqual(n.as_reference(b""), b"delta_cache")
self.assertEqual(n.as_reference(b'parent'), b'parent.delta_cache') self.assertEqual(n.as_reference(b"parent"), b"parent.delta_cache")
class StructTest(unittest.TestCase): class StructTest(unittest.TestCase):
@ -76,20 +82,20 @@ class StructTest(unittest.TestCase):
endian = dna_io.BigEndianTypes endian = dna_io.BigEndianTypes
def setUp(self): def setUp(self):
self.s = dna.Struct(b'AlembicObjectPath') self.s = dna.Struct(b"AlembicObjectPath")
self.s_char = dna.Struct(b'char', 1) self.s_char = dna.Struct(b"char", 1)
self.s_float = dna.Struct(b'float', 4) self.s_float = dna.Struct(b"float", 4)
self.s_uint64 = dna.Struct(b'uint64_t', 8) self.s_uint64 = dna.Struct(b"uint64_t", 8)
self.s_uint128 = dna.Struct(b'uint128_t', 16) # non-supported type self.s_uint128 = dna.Struct(b"uint128_t", 16) # non-supported type
self.f_next = dna.Field(self.s, dna.Name(b'*next'), 8, 0) self.f_next = dna.Field(self.s, dna.Name(b"*next"), 8, 0)
self.f_prev = dna.Field(self.s, dna.Name(b'*prev'), 8, 8) self.f_prev = dna.Field(self.s, dna.Name(b"*prev"), 8, 8)
self.f_path = dna.Field(self.s_char, dna.Name(b'path[4096]'), 4096, 16) self.f_path = dna.Field(self.s_char, dna.Name(b"path[4096]"), 4096, 16)
self.f_pointer = dna.Field(self.s_char, dna.Name(b'*ptr'), 3 * 8, 4112) self.f_pointer = dna.Field(self.s_char, dna.Name(b"*ptr"), 3 * 8, 4112)
self.f_number = dna.Field(self.s_uint64, dna.Name(b'numbah'), 8, 4136) self.f_number = dna.Field(self.s_uint64, dna.Name(b"numbah"), 8, 4136)
self.f_floaty = dna.Field(self.s_float, dna.Name(b'floaty[2]'), 2 * 4, 4144) self.f_floaty = dna.Field(self.s_float, dna.Name(b"floaty[2]"), 2 * 4, 4144)
self.f_flag = dna.Field(self.s_char, dna.Name(b'bitflag'), 1, 4152) self.f_flag = dna.Field(self.s_char, dna.Name(b"bitflag"), 1, 4152)
self.f_bignum = dna.Field(self.s_uint128, dna.Name(b'bignum'), 16, 4153) self.f_bignum = dna.Field(self.s_uint128, dna.Name(b"bignum"), 16, 4153)
self.s.append_field(self.f_next) self.s.append_field(self.f_next)
self.s.append_field(self.f_prev) self.s.append_field(self.f_prev)
@ -108,13 +114,13 @@ class StructTest(unittest.TestCase):
# the size property is explicitly set anyway. The situation we # the size property is explicitly set anyway. The situation we
# test here is for manually created Struct instances that don't # test here is for manually created Struct instances that don't
# have any fields. # have any fields.
dna.Struct(b'EmptyStruct').size dna.Struct(b"EmptyStruct").size
# Create AlebicObjectPath as it is actually used in Blender 2.79a # Create AlebicObjectPath as it is actually used in Blender 2.79a
s = dna.Struct(b'AlembicObjectPath') s = dna.Struct(b"AlembicObjectPath")
f_next = dna.Field(s, dna.Name(b'*next'), 8, 0) f_next = dna.Field(s, dna.Name(b"*next"), 8, 0)
f_prev = dna.Field(s, dna.Name(b'*prev'), 8, 8) f_prev = dna.Field(s, dna.Name(b"*prev"), 8, 8)
f_path = dna.Field(self.s_char, dna.Name(b'path[4096]'), 4096, 16) f_path = dna.Field(self.s_char, dna.Name(b"path[4096]"), 4096, 16)
s.append_field(f_next) s.append_field(f_next)
s.append_field(f_prev) s.append_field(f_prev)
s.append_field(f_path) s.append_field(f_path)
@ -123,36 +129,42 @@ class StructTest(unittest.TestCase):
def test_field_from_path(self): def test_field_from_path(self):
psize = 8 psize = 8
self.assertEqual(self.s.field_from_path(psize, b'path'), self.assertEqual(self.s.field_from_path(psize, b"path"), (self.f_path, 16))
(self.f_path, 16)) self.assertEqual(
self.assertEqual(self.s.field_from_path(psize, (b'prev', b'path')), self.s.field_from_path(psize, (b"prev", b"path")), (self.f_path, 24)
(self.f_path, 24)) )
self.assertEqual(self.s.field_from_path(psize, (b'ptr', 2)), self.assertEqual(
(self.f_pointer, 16 + 4096 + 2 * psize)) self.s.field_from_path(psize, (b"ptr", 2)),
self.assertEqual(self.s.field_from_path(psize, (b'floaty', 1)), (self.f_pointer, 16 + 4096 + 2 * psize),
(self.f_floaty, 4144 + self.s_float.size)) )
self.assertEqual(
self.s.field_from_path(psize, (b"floaty", 1)),
(self.f_floaty, 4144 + self.s_float.size),
)
with self.assertRaises(OverflowError): with self.assertRaises(OverflowError):
self.s.field_from_path(psize, (b'floaty', 2)) self.s.field_from_path(psize, (b"floaty", 2))
with self.assertRaises(KeyError): with self.assertRaises(KeyError):
self.s.field_from_path(psize, b'non-existant') self.s.field_from_path(psize, b"non-existant")
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
self.s.field_from_path(psize, 'path') self.s.field_from_path(psize, "path")
def test_simple_field_get(self): def test_simple_field_get(self):
fileobj = mock.MagicMock(io.BufferedReader) fileobj = mock.MagicMock(io.BufferedReader)
fileobj.read.return_value = b'\x01\x02\x03\x04\xff\xfe\xfd\xfa' fileobj.read.return_value = b"\x01\x02\x03\x04\xff\xfe\xfd\xfa"
_, val = self.s.field_get(self.FakeHeader(), fileobj, b'numbah') _, val = self.s.field_get(self.FakeHeader(), fileobj, b"numbah")
self.assertEqual(val, 0x1020304fffefdfa) self.assertEqual(val, 0x1020304FFFEFDFA)
fileobj.seek.assert_called_with(4136, os.SEEK_CUR) fileobj.seek.assert_called_with(4136, os.SEEK_CUR)
def test_field_get_default(self): def test_field_get_default(self):
fileobj = mock.MagicMock(io.BufferedReader) fileobj = mock.MagicMock(io.BufferedReader)
fileobj.read.side_effect = RuntimeError fileobj.read.side_effect = RuntimeError
_, val = self.s.field_get(self.FakeHeader(), fileobj, b'nonexistant', default=519871531) _, val = self.s.field_get(
self.FakeHeader(), fileobj, b"nonexistant", default=519871531
)
self.assertEqual(val, 519871531) self.assertEqual(val, 519871531)
fileobj.seek.assert_not_called() fileobj.seek.assert_not_called()
@ -162,7 +174,7 @@ class StructTest(unittest.TestCase):
fileobj.read.side_effect = RuntimeError fileobj.read.side_effect = RuntimeError
with self.assertRaises(KeyError): with self.assertRaises(KeyError):
self.s.field_get(self.FakeHeader(), fileobj, b'nonexistant') self.s.field_get(self.FakeHeader(), fileobj, b"nonexistant")
fileobj.seek.assert_not_called() fileobj.seek.assert_not_called()
def test_field_get_unsupported_type(self): def test_field_get_unsupported_type(self):
@ -170,62 +182,63 @@ class StructTest(unittest.TestCase):
fileobj.read.side_effect = RuntimeError fileobj.read.side_effect = RuntimeError
with self.assertRaises(NotImplementedError): with self.assertRaises(NotImplementedError):
self.s.field_get(self.FakeHeader(), fileobj, b'bignum') self.s.field_get(self.FakeHeader(), fileobj, b"bignum")
fileobj.seek.assert_called_with(4153, os.SEEK_CUR) fileobj.seek.assert_called_with(4153, os.SEEK_CUR)
def test_pointer_field_get(self): def test_pointer_field_get(self):
fileobj = mock.MagicMock(io.BufferedReader) fileobj = mock.MagicMock(io.BufferedReader)
fileobj.read.return_value = b'\xf0\x9f\xa6\x87\x00dum' fileobj.read.return_value = b"\xf0\x9f\xa6\x87\x00dum"
_, val = self.s.field_get(self.FakeHeader(), fileobj, b'ptr') _, val = self.s.field_get(self.FakeHeader(), fileobj, b"ptr")
self.assertEqual(0xf09fa6870064756d, val) self.assertEqual(0xF09FA6870064756D, val)
fileobj.seek.assert_called_with(4112, os.SEEK_CUR) fileobj.seek.assert_called_with(4112, os.SEEK_CUR)
def test_string_field_get(self): def test_string_field_get(self):
fileobj = mock.MagicMock(io.BufferedReader) fileobj = mock.MagicMock(io.BufferedReader)
fileobj.read.return_value = b'\xf0\x9f\xa6\x87\x00dummydata' fileobj.read.return_value = b"\xf0\x9f\xa6\x87\x00dummydata"
_, val = self.s.field_get(self.FakeHeader(), fileobj, b'path', as_str=True) _, val = self.s.field_get(self.FakeHeader(), fileobj, b"path", as_str=True)
self.assertEqual('🦇', val) self.assertEqual("🦇", val)
fileobj.seek.assert_called_with(16, os.SEEK_CUR) fileobj.seek.assert_called_with(16, os.SEEK_CUR)
def test_string_field_get_single_char(self): def test_string_field_get_single_char(self):
fileobj = mock.MagicMock(io.BufferedReader) fileobj = mock.MagicMock(io.BufferedReader)
fileobj.read.return_value = b'\xf0' fileobj.read.return_value = b"\xf0"
_, val = self.s.field_get(self.FakeHeader(), fileobj, b'bitflag') _, val = self.s.field_get(self.FakeHeader(), fileobj, b"bitflag")
self.assertEqual(0xf0, val) self.assertEqual(0xF0, val)
fileobj.seek.assert_called_with(4152, os.SEEK_CUR) fileobj.seek.assert_called_with(4152, os.SEEK_CUR)
def test_string_field_get_invalid_utf8(self): def test_string_field_get_invalid_utf8(self):
fileobj = mock.MagicMock(io.BufferedReader) fileobj = mock.MagicMock(io.BufferedReader)
fileobj.read.return_value = b'\x01\x02\x03\x04\xff\xfe\xfd\xfa' fileobj.read.return_value = b"\x01\x02\x03\x04\xff\xfe\xfd\xfa"
with self.assertRaises(UnicodeDecodeError): with self.assertRaises(UnicodeDecodeError):
self.s.field_get(self.FakeHeader(), fileobj, b'path', as_str=True) self.s.field_get(self.FakeHeader(), fileobj, b"path", as_str=True)
def test_string_field_get_bytes_null_terminated(self): def test_string_field_get_bytes_null_terminated(self):
fileobj = mock.MagicMock(io.BufferedReader) fileobj = mock.MagicMock(io.BufferedReader)
fileobj.read.return_value = b'\x01\x02\x03\x04\xff\xfe\xfd\xfa\x00dummydata' fileobj.read.return_value = b"\x01\x02\x03\x04\xff\xfe\xfd\xfa\x00dummydata"
_, val = self.s.field_get(self.FakeHeader(), fileobj, b'path', as_str=False) _, val = self.s.field_get(self.FakeHeader(), fileobj, b"path", as_str=False)
self.assertEqual(b'\x01\x02\x03\x04\xff\xfe\xfd\xfa', val) self.assertEqual(b"\x01\x02\x03\x04\xff\xfe\xfd\xfa", val)
fileobj.seek.assert_called_with(16, os.SEEK_CUR) fileobj.seek.assert_called_with(16, os.SEEK_CUR)
def test_string_field_get_bytes(self): def test_string_field_get_bytes(self):
fileobj = mock.MagicMock(io.BufferedReader) fileobj = mock.MagicMock(io.BufferedReader)
fileobj.read.return_value = b'\x01\x02\x03\x04\xff\xfe\xfd\xfa\x00dummydata' fileobj.read.return_value = b"\x01\x02\x03\x04\xff\xfe\xfd\xfa\x00dummydata"
_, val = self.s.field_get(self.FakeHeader(), fileobj, b'path', _, val = self.s.field_get(
as_str=False, null_terminated=False) self.FakeHeader(), fileobj, b"path", as_str=False, null_terminated=False
self.assertEqual(b'\x01\x02\x03\x04\xff\xfe\xfd\xfa\x00dummydata', val) )
self.assertEqual(b"\x01\x02\x03\x04\xff\xfe\xfd\xfa\x00dummydata", val)
fileobj.seek.assert_called_with(16, os.SEEK_CUR) fileobj.seek.assert_called_with(16, os.SEEK_CUR)
def test_string_field_get_float_array(self): def test_string_field_get_float_array(self):
fileobj = mock.MagicMock(io.BufferedReader) fileobj = mock.MagicMock(io.BufferedReader)
fileobj.read.side_effect = (b'@333', b'@2\x8f\\') fileobj.read.side_effect = (b"@333", b"@2\x8f\\")
_, val = self.s.field_get(self.FakeHeader(), fileobj, b'floaty') _, val = self.s.field_get(self.FakeHeader(), fileobj, b"floaty")
self.assertAlmostEqual(2.8, val[0]) self.assertAlmostEqual(2.8, val[0])
self.assertAlmostEqual(2.79, val[1]) self.assertAlmostEqual(2.79, val[1])
fileobj.seek.assert_called_with(4144, os.SEEK_CUR) fileobj.seek.assert_called_with(4144, os.SEEK_CUR)

8
tests/test_blendfile_dna_io.py
View File

@ -9,16 +9,16 @@ class StringTest(unittest.TestCase):
fileobj = mock.Mock() fileobj = mock.Mock()
# Sinhala for 'beer'. This is exactly 15 bytes in UTF-8, so the last # Sinhala for 'beer'. This is exactly 15 bytes in UTF-8, so the last
# character won't fit in the field (due to the 0-byte required). # character won't fit in the field (due to the 0-byte required).
dna_io.BigEndianTypes.write_string(fileobj, 'බියර්', 15) dna_io.BigEndianTypes.write_string(fileobj, "බියර්", 15)
expect_bytes = ('බියර්'[:-1]).encode('utf8') + b'\0' expect_bytes = ("බියර්"[:-1]).encode("utf8") + b"\0"
fileobj.write.assert_called_with(expect_bytes) fileobj.write.assert_called_with(expect_bytes)
def test_utf8(self): def test_utf8(self):
fileobj = mock.Mock() fileobj = mock.Mock()
# Sinhala for 'beer'. This is exactly 15 bytes in UTF-8, # Sinhala for 'beer'. This is exactly 15 bytes in UTF-8,
# so with the 0-byte it just fits. # so with the 0-byte it just fits.
dna_io.BigEndianTypes.write_string(fileobj, 'බියර්', 16) dna_io.BigEndianTypes.write_string(fileobj, "බියර්", 16)
expect_bytes = 'බියර්'.encode('utf8') + b'\0' expect_bytes = "බියර්".encode("utf8") + b"\0"
fileobj.write.assert_called_with(expect_bytes) fileobj.write.assert_called_with(expect_bytes)

267
tests/test_blendfile_loading.py
View File

@ -9,64 +9,66 @@ from tests.abstract_test import AbstractBlendFileTest
class BlendFileBlockTest(AbstractBlendFileTest): class BlendFileBlockTest(AbstractBlendFileTest):
def setUp(self): def setUp(self):
self.bf = blendfile.BlendFile(self.blendfiles / 'basic_file.blend') self.bf = blendfile.BlendFile(self.blendfiles / "basic_file.blend")
def test_loading(self): def test_loading(self):
self.assertFalse(self.bf.is_compressed) self.assertFalse(self.bf.is_compressed)
def test_some_properties(self): def test_some_properties(self):
ob = self.bf.code_index[b'OB'][0] ob = self.bf.code_index[b"OB"][0]
self.assertEqual('Object', ob.dna_type_name) self.assertEqual("Object", ob.dna_type_name)
# Try low level operation to read a property. # Try low level operation to read a property.
self.bf.fileobj.seek(ob.file_offset, os.SEEK_SET) self.bf.fileobj.seek(ob.file_offset, os.SEEK_SET)
_, loc = ob.dna_type.field_get(self.bf.header, self.bf.fileobj, b'loc') _, loc = ob.dna_type.field_get(self.bf.header, self.bf.fileobj, b"loc")
self.assertEqual([2.0, 3.0, 5.0], loc) self.assertEqual([2.0, 3.0, 5.0], loc)
# Try low level operation to read an array element. # Try low level operation to read an array element.
self.bf.fileobj.seek(ob.file_offset, os.SEEK_SET) self.bf.fileobj.seek(ob.file_offset, os.SEEK_SET)
_, loc_z = ob.dna_type.field_get(self.bf.header, self.bf.fileobj, (b'loc', 2)) _, loc_z = ob.dna_type.field_get(self.bf.header, self.bf.fileobj, (b"loc", 2))
self.assertEqual(5.0, loc_z) self.assertEqual(5.0, loc_z)
# Try high level operation to read the same property. # Try high level operation to read the same property.
loc = ob.get(b'loc') loc = ob.get(b"loc")
self.assertEqual([2.0, 3.0, 5.0], loc) self.assertEqual([2.0, 3.0, 5.0], loc)
# Try getting a subproperty. # Try getting a subproperty.
name = ob.get((b'id', b'name'), as_str=True) name = ob.get((b"id", b"name"), as_str=True)
self.assertEqual('OBümlaut', name) self.assertEqual("OBümlaut", name)
loc_z = ob.get((b'loc', 2)) loc_z = ob.get((b"loc", 2))
self.assertEqual(5.0, loc_z) self.assertEqual(5.0, loc_z)
# Try following a pointer. # Try following a pointer.
mesh_ptr = ob.get(b'data') mesh_ptr = ob.get(b"data")
mesh = self.bf.block_from_addr[mesh_ptr] mesh = self.bf.block_from_addr[mesh_ptr]
mname = mesh.get((b'id', b'name'), as_str=True) mname = mesh.get((b"id", b"name"), as_str=True)
self.assertEqual('MECube³', mname) self.assertEqual("MECube³", mname)
def test_get_recursive_iter(self): def test_get_recursive_iter(self):
ob = self.bf.code_index[b'OB'][0] ob = self.bf.code_index[b"OB"][0]
assert isinstance(ob, blendfile.BlendFileBlock) assert isinstance(ob, blendfile.BlendFileBlock)
# No recursing, just an array property. # No recursing, just an array property.
gen = ob.get_recursive_iter(b'loc') gen = ob.get_recursive_iter(b"loc")
self.assertEqual([(b'loc', [2.0, 3.0, 5.0])], list(gen)) self.assertEqual([(b"loc", [2.0, 3.0, 5.0])], list(gen))
# Recurse into an object # Recurse into an object
gen = ob.get_recursive_iter(b'id') gen = ob.get_recursive_iter(b"id")
self.assertEqual( self.assertEqual(
[((b'id', b'next'), 0), [
((b'id', b'prev'), 0), ((b"id", b"next"), 0),
((b'id', b'newid'), 0), ((b"id", b"prev"), 0),
((b'id', b'lib'), 0), ((b"id", b"newid"), 0),
((b'id', b'name'), 'OBümlaut'), ((b"id", b"lib"), 0),
((b'id', b'flag'), 0), ((b"id", b"name"), "OBümlaut"),
], ((b"id", b"flag"), 0),
list(gen)[:6]) ],
list(gen)[:6],
)
def test_iter_recursive(self): def test_iter_recursive(self):
ob = self.bf.code_index[b'OB'][0] ob = self.bf.code_index[b"OB"][0]
assert isinstance(ob, blendfile.BlendFileBlock) assert isinstance(ob, blendfile.BlendFileBlock)
# We can't test all of them in a reliable way, but it shouldn't crash. # We can't test all of them in a reliable way, but it shouldn't crash.
@ -74,17 +76,19 @@ class BlendFileBlockTest(AbstractBlendFileTest):
# And we can check the first few items. # And we can check the first few items.
self.assertEqual( self.assertEqual(
[((b'id', b'next'), 0), [
((b'id', b'prev'), 0), ((b"id", b"next"), 0),
((b'id', b'newid'), 0), ((b"id", b"prev"), 0),
((b'id', b'lib'), 0), ((b"id", b"newid"), 0),
((b'id', b'name'), ((b"id", b"lib"), 0),
b'OB\xc3\xbcmlaut'), ((b"id", b"name"), b"OB\xc3\xbcmlaut"),
((b'id', b'flag'), 0), ((b"id", b"flag"), 0),
], all_items[:6]) ],
all_items[:6],
)
def test_items(self): def test_items(self):
ma = self.bf.code_index[b'MA'][0] ma = self.bf.code_index[b"MA"][0]
assert isinstance(ma, blendfile.BlendFileBlock) assert isinstance(ma, blendfile.BlendFileBlock)
# We can't test all of them in a reliable way, but it shouldn't crash. # We can't test all of them in a reliable way, but it shouldn't crash.
@ -92,18 +96,21 @@ class BlendFileBlockTest(AbstractBlendFileTest):
# And we can check the first few items. # And we can check the first few items.
self.assertEqual( self.assertEqual(
[(b'id', '<ID>'), # not recursed into. [
(b'adt', 0), (b"id", "<ID>"), # not recursed into.
(b'material_type', 0), (b"adt", 0),
(b'flag', 0), (b"material_type", 0),
(b'r', 0.8000000715255737), (b"flag", 0),
(b'g', 0.03218378871679306), (b"r", 0.8000000715255737),
(b'b', 0.36836329102516174), (b"g", 0.03218378871679306),
(b'specr', 1.0)], (b"b", 0.36836329102516174),
all_items[:8]) (b"specr", 1.0),
],
all_items[:8],
)
def test_keys(self): def test_keys(self):
ma = self.bf.code_index[b'MA'][0] ma = self.bf.code_index[b"MA"][0]
assert isinstance(ma, blendfile.BlendFileBlock) assert isinstance(ma, blendfile.BlendFileBlock)
# We can't test all of them in a reliable way, but it shouldn't crash. # We can't test all of them in a reliable way, but it shouldn't crash.
@ -111,11 +118,12 @@ class BlendFileBlockTest(AbstractBlendFileTest):
# And we can check the first few items. # And we can check the first few items.
self.assertEqual( self.assertEqual(
[b'id', b'adt', b'material_type', b'flag', b'r', b'g', b'b', b'specr'], [b"id", b"adt", b"material_type", b"flag", b"r", b"g", b"b", b"specr"],
all_keys[:8]) all_keys[:8],
)
def test_values(self): def test_values(self):
ma = self.bf.code_index[b'MA'][0] ma = self.bf.code_index[b"MA"][0]
assert isinstance(ma, blendfile.BlendFileBlock) assert isinstance(ma, blendfile.BlendFileBlock)
# We can't test all of them in a reliable way, but it shouldn't crash. # We can't test all of them in a reliable way, but it shouldn't crash.
@ -123,151 +131,156 @@ class BlendFileBlockTest(AbstractBlendFileTest):
# And we can check the first few items. # And we can check the first few items.
self.assertEqual( self.assertEqual(
['<ID>', [
0, "<ID>",
0, 0,
0, 0,
0.8000000715255737, 0,
0.03218378871679306, 0.8000000715255737,
0.36836329102516174, 0.03218378871679306,
1.0], 0.36836329102516174,
all_values[:8]) 1.0,
],
all_values[:8],
)
def test_get_via_dict_interface(self): def test_get_via_dict_interface(self):
ma = self.bf.code_index[b'MA'][0] ma = self.bf.code_index[b"MA"][0]
assert isinstance(ma, blendfile.BlendFileBlock) assert isinstance(ma, blendfile.BlendFileBlock)
self.assertAlmostEqual(0.8000000715255737, ma[b'r']) self.assertAlmostEqual(0.8000000715255737, ma[b"r"])
ob = self.bf.code_index[b'OB'][0] ob = self.bf.code_index[b"OB"][0]
assert isinstance(ob, blendfile.BlendFileBlock) assert isinstance(ob, blendfile.BlendFileBlock)
self.assertEqual('OBümlaut', ob.id_name.decode()) self.assertEqual("OBümlaut", ob.id_name.decode())
class PointerTest(AbstractBlendFileTest): class PointerTest(AbstractBlendFileTest):
def setUp(self): def setUp(self):
self.bf = blendfile.BlendFile(self.blendfiles / 'with_sequencer.blend') self.bf = blendfile.BlendFile(self.blendfiles / "with_sequencer.blend")
def test_get_pointer_and_listbase(self): def test_get_pointer_and_listbase(self):
scenes = self.bf.code_index[b'SC'] scenes = self.bf.code_index[b"SC"]
self.assertEqual(1, len(scenes), 'expecting 1 scene') self.assertEqual(1, len(scenes), "expecting 1 scene")
scene = scenes[0] scene = scenes[0]
self.assertEqual(b'SCScene', scene.id_name) self.assertEqual(b"SCScene", scene.id_name)
ed_ptr = scene[b'ed'] ed_ptr = scene[b"ed"]
self.assertEqual(140051431100936, ed_ptr) self.assertEqual(140051431100936, ed_ptr)
ed = scene.get_pointer(b'ed') ed = scene.get_pointer(b"ed")
self.assertEqual(140051431100936, ed.addr_old) self.assertEqual(140051431100936, ed.addr_old)
seqbase = ed.get_pointer((b'seqbase', b'first')) seqbase = ed.get_pointer((b"seqbase", b"first"))
self.assertIsNotNone(seqbase) self.assertIsNotNone(seqbase)
types = { types = {
b'SQBlack': 28, b"SQBlack": 28,
b'SQCross': 8, b"SQCross": 8,
b'SQPink': 28, b"SQPink": 28,
} }
seq = None seq = None
for seq in iterators.listbase(seqbase): for seq in iterators.listbase(seqbase):
seq.refine_type(b'Sequence') seq.refine_type(b"Sequence")
name = seq[b'name'] name = seq[b"name"]
expected_type = types[name] expected_type = types[name]
self.assertEqual(expected_type, seq[b'type']) self.assertEqual(expected_type, seq[b"type"])
# The last 'seq' from the loop should be the last in the list. # The last 'seq' from the loop should be the last in the list.
seq_next = seq.get_pointer(b'next') seq_next = seq.get_pointer(b"next")
self.assertIsNone(seq_next) self.assertIsNone(seq_next)
def test_refine_sdna_by_name(self): def test_refine_sdna_by_name(self):
scene = self.bf.code_index[b'SC'][0] scene = self.bf.code_index[b"SC"][0]
ed = scene.get_pointer(b'ed') ed = scene.get_pointer(b"ed")
seq = ed.get_pointer((b'seqbase', b'first')) seq = ed.get_pointer((b"seqbase", b"first"))
seq.refine_type(b'Sequence') seq.refine_type(b"Sequence")
self.assertEqual(b'SQBlack', seq[b'name']) self.assertEqual(b"SQBlack", seq[b"name"])
self.assertEqual(28, seq[b'type']) self.assertEqual(28, seq[b"type"])
def test_refine_sdna_by_idx(self): def test_refine_sdna_by_idx(self):
scene = self.bf.code_index[b'SC'][0] scene = self.bf.code_index[b"SC"][0]
ed = scene.get_pointer(b'ed') ed = scene.get_pointer(b"ed")
seq = ed.get_pointer((b'seqbase', b'first')) seq = ed.get_pointer((b"seqbase", b"first"))
sdna_idx_sequence = self.bf.sdna_index_from_id[b'Sequence'] sdna_idx_sequence = self.bf.sdna_index_from_id[b"Sequence"]
seq.refine_type_from_index(sdna_idx_sequence) seq.refine_type_from_index(sdna_idx_sequence)
self.assertEqual(b'SQBlack', seq[b'name']) self.assertEqual(b"SQBlack", seq[b"name"])
self.assertEqual(28, seq[b'type']) self.assertEqual(28, seq[b"type"])
def test_segfault(self): def test_segfault(self):
scene = self.bf.code_index[b'SC'][0] scene = self.bf.code_index[b"SC"][0]
ed_ptr = scene.get(b'ed') ed_ptr = scene.get(b"ed")
del self.bf.block_from_addr[ed_ptr] del self.bf.block_from_addr[ed_ptr]
with self.assertRaises(exceptions.SegmentationFault): with self.assertRaises(exceptions.SegmentationFault):
scene.get_pointer(b'ed') scene.get_pointer(b"ed")
def test_abs_offset(self): def test_abs_offset(self):
scene = self.bf.code_index[b'SC'][0] scene = self.bf.code_index[b"SC"][0]
ed = scene.get_pointer(b'ed') ed = scene.get_pointer(b"ed")
assert isinstance(ed, blendfile.BlendFileBlock) assert isinstance(ed, blendfile.BlendFileBlock)
abs_offset, field_size = ed.abs_offset((b'seqbase', b'first')) abs_offset, field_size = ed.abs_offset((b"seqbase", b"first"))
self.assertEqual(ed.file_offset + 8, abs_offset) self.assertEqual(ed.file_offset + 8, abs_offset)
self.assertEqual(1, field_size) self.assertEqual(1, field_size)
class ArrayTest(AbstractBlendFileTest): class ArrayTest(AbstractBlendFileTest):
def test_array_of_pointers(self): def test_array_of_pointers(self):
self.bf = blendfile.BlendFile(self.blendfiles / 'multiple_materials.blend') self.bf = blendfile.BlendFile(self.blendfiles / "multiple_materials.blend")
mesh = self.bf.code_index[b'ME'][0] mesh = self.bf.code_index[b"ME"][0]
assert isinstance(mesh, blendfile.BlendFileBlock) assert isinstance(mesh, blendfile.BlendFileBlock)
material_count = mesh[b'totcol'] material_count = mesh[b"totcol"]
self.assertEqual(4, material_count) self.assertEqual(4, material_count)
for i, material in enumerate(mesh.iter_array_of_pointers(b'mat', material_count)): for i, material in enumerate(
mesh.iter_array_of_pointers(b"mat", material_count)
):
if i == 0: if i == 0:
name = b'MAMaterial.000' name = b"MAMaterial.000"
elif i in {1, 3}: elif i in {1, 3}:
name = b'MAMaterial.001' name = b"MAMaterial.001"
else: else:
name = b'MAMaterial.002' name = b"MAMaterial.002"
self.assertEqual(name, material.id_name) self.assertEqual(name, material.id_name)
def test_array_of_lamp_textures(self): def test_array_of_lamp_textures(self):
self.bf = blendfile.BlendFile(self.blendfiles / 'lamp_textures.blend') self.bf = blendfile.BlendFile(self.blendfiles / "lamp_textures.blend")
lamp = self.bf.code_index[b'LA'][0] lamp = self.bf.code_index[b"LA"][0]
assert isinstance(lamp, blendfile.BlendFileBlock) assert isinstance(lamp, blendfile.BlendFileBlock)
mtex0 = lamp.get_pointer(b'mtex') mtex0 = lamp.get_pointer(b"mtex")
tex = mtex0.get_pointer(b'tex') tex = mtex0.get_pointer(b"tex")
self.assertEqual(b'TE', tex.code) self.assertEqual(b"TE", tex.code)
self.assertEqual(b'TEClouds', tex.id_name) self.assertEqual(b"TEClouds", tex.id_name)
for i, mtex in enumerate(lamp.iter_fixed_array_of_pointers(b'mtex')): for i, mtex in enumerate(lamp.iter_fixed_array_of_pointers(b"mtex")):
if i == 0: if i == 0:
name = b'TEClouds' name = b"TEClouds"
elif i == 1: elif i == 1:
name = b'TEVoronoi' name = b"TEVoronoi"
else: else:
self.fail('Too many textures reported: %r' % mtex) self.fail("Too many textures reported: %r" % mtex)
tex = mtex.get_pointer(b'tex') tex = mtex.get_pointer(b"tex")
self.assertEqual(b'TE', tex.code) self.assertEqual(b"TE", tex.code)
self.assertEqual(name, tex.id_name) self.assertEqual(name, tex.id_name)
class LoadCompressedTest(AbstractBlendFileTest): class LoadCompressedTest(AbstractBlendFileTest):
def test_loading(self): def test_loading(self):
self.bf = blendfile.BlendFile(self.blendfiles / 'basic_file_compressed.blend') self.bf = blendfile.BlendFile(self.blendfiles / "basic_file_compressed.blend")
self.assertTrue(self.bf.is_compressed) self.assertTrue(self.bf.is_compressed)
ob = self.bf.code_index[b'OB'][0] ob = self.bf.code_index[b"OB"][0]
name = ob.get((b'id', b'name'), as_str=True) name = ob.get((b"id", b"name"), as_str=True)
self.assertEqual('OBümlaut', name) self.assertEqual("OBümlaut", name)
def test_as_context(self): def test_as_context(self):
with blendfile.BlendFile(self.blendfiles / 'basic_file_compressed.blend') as bf: with blendfile.BlendFile(self.blendfiles / "basic_file_compressed.blend") as bf:
filepath = bf.filepath filepath = bf.filepath
raw_filepath = bf.raw_filepath raw_filepath = bf.raw_filepath
@ -283,7 +296,7 @@ class LoadNonBlendfileTest(AbstractBlendFileTest):
def test_no_datablocks(self): def test_no_datablocks(self):
with self.assertRaises(exceptions.NoDNA1Block): with self.assertRaises(exceptions.NoDNA1Block):
blendfile.BlendFile(self.blendfiles / 'corrupt_only_magic.blend') blendfile.BlendFile(self.blendfiles / "corrupt_only_magic.blend")
class BlendFileCacheTest(AbstractBlendFileTest): class BlendFileCacheTest(AbstractBlendFileTest):
@ -297,7 +310,7 @@ class BlendFileCacheTest(AbstractBlendFileTest):
self.tdir.cleanup() self.tdir.cleanup()
def test_open_cached(self): def test_open_cached(self):
infile = self.blendfiles / 'basic_file.blend' infile = self.blendfiles / "basic_file.blend"
bf1 = blendfile.open_cached(infile) bf1 = blendfile.open_cached(infile)
bf2 = blendfile.open_cached(infile) bf2 = blendfile.open_cached(infile)
@ -306,7 +319,7 @@ class BlendFileCacheTest(AbstractBlendFileTest):
self.assertIs(bf1, blendfile._cached_bfiles[infile]) self.assertIs(bf1, blendfile._cached_bfiles[infile])
def test_compressed(self): def test_compressed(self):
infile = self.blendfiles / 'linked_cube_compressed.blend' infile = self.blendfiles / "linked_cube_compressed.blend"
bf1 = blendfile.open_cached(infile) bf1 = blendfile.open_cached(infile)
bf2 = blendfile.open_cached(infile) bf2 = blendfile.open_cached(infile)
@ -315,7 +328,7 @@ class BlendFileCacheTest(AbstractBlendFileTest):
self.assertIs(bf1, blendfile._cached_bfiles[infile]) self.assertIs(bf1, blendfile._cached_bfiles[infile])
def test_closed(self): def test_closed(self):
infile = self.blendfiles / 'linked_cube_compressed.blend' infile = self.blendfiles / "linked_cube_compressed.blend"
bf = blendfile.open_cached(infile) bf = blendfile.open_cached(infile)
self.assertIs(bf, blendfile._cached_bfiles[infile]) self.assertIs(bf, blendfile._cached_bfiles[infile])
@ -324,8 +337,8 @@ class BlendFileCacheTest(AbstractBlendFileTest):
self.assertEqual({}, blendfile._cached_bfiles) self.assertEqual({}, blendfile._cached_bfiles)
def test_close_one_file(self): def test_close_one_file(self):
path1 = self.blendfiles / 'linked_cube_compressed.blend' path1 = self.blendfiles / "linked_cube_compressed.blend"
path2 = self.blendfiles / 'basic_file.blend' path2 = self.blendfiles / "basic_file.blend"
bf1 = blendfile.open_cached(path1) bf1 = blendfile.open_cached(path1)
bf2 = blendfile.open_cached(path2) bf2 = blendfile.open_cached(path2)
self.assertIs(bf1, blendfile._cached_bfiles[path1]) self.assertIs(bf1, blendfile._cached_bfiles[path1])
@ -336,13 +349,13 @@ class BlendFileCacheTest(AbstractBlendFileTest):
self.assertEqual({path2: bf2}, blendfile._cached_bfiles) self.assertEqual({path2: bf2}, blendfile._cached_bfiles)
def test_open_and_rebind(self): def test_open_and_rebind(self):
infile = self.blendfiles / 'linked_cube.blend' infile = self.blendfiles / "linked_cube.blend"
other = self.tpath / 'copy.blend' other = self.tpath / "copy.blend"
self._open_and_rebind_test(infile, other) self._open_and_rebind_test(infile, other)
def test_open_and_rebind_compressed(self): def test_open_and_rebind_compressed(self):
infile = self.blendfiles / 'linked_cube_compressed.blend' infile = self.blendfiles / "linked_cube_compressed.blend"
other = self.tpath / 'copy.blend' other = self.tpath / "copy.blend"
self._open_and_rebind_test(infile, other) self._open_and_rebind_test(infile, other)
def _open_and_rebind_test(self, infile: pathlib.Path, other: pathlib.Path): def _open_and_rebind_test(self, infile: pathlib.Path, other: pathlib.Path):
@ -357,7 +370,7 @@ class BlendFileCacheTest(AbstractBlendFileTest):
before_blocks = bf.blocks before_blocks = bf.blocks
before_compressed = bf.is_compressed before_compressed = bf.is_compressed
bf.copy_and_rebind(other, mode='rb+') bf.copy_and_rebind(other, mode="rb+")
self.assertTrue(other.exists()) self.assertTrue(other.exists())
self.assertEqual(before_compressed, bf.is_compressed) self.assertEqual(before_compressed, bf.is_compressed)

56
tests/test_blendfile_modification.py
View File

@ -8,11 +8,13 @@ from tests.abstract_test import AbstractBlendFileTest
class ModifyUncompressedTest(AbstractBlendFileTest): class ModifyUncompressedTest(AbstractBlendFileTest):
def setUp(self): def setUp(self):
self.orig = self.blendfiles / 'linked_cube.blend' self.orig = self.blendfiles / "linked_cube.blend"
self.to_modify = self.orig.with_name('linked_cube_modified.blend') self.to_modify = self.orig.with_name("linked_cube_modified.blend")
copyfile(str(self.orig), str(self.to_modify)) # TODO: when requiring Python 3.6+, remove str() copyfile(
self.bf = blendfile.BlendFile(self.to_modify, mode='r+b') str(self.orig), str(self.to_modify)
) # TODO: when requiring Python 3.6+, remove str()
self.bf = blendfile.BlendFile(self.to_modify, mode="r+b")
self.assertFalse(self.bf.is_compressed) self.assertFalse(self.bf.is_compressed)
@ -22,20 +24,20 @@ class ModifyUncompressedTest(AbstractBlendFileTest):
self.to_modify.unlink() self.to_modify.unlink()
def test_change_path(self): def test_change_path(self):
library = self.bf.code_index[b'LI'][0] library = self.bf.code_index[b"LI"][0]
# Change it from absolute to relative. # Change it from absolute to relative.
library[b'filepath'] = b'//basic_file.blend' library[b"filepath"] = b"//basic_file.blend"
library[b'name'] = b'//basic_file.blend' library[b"name"] = b"//basic_file.blend"
self.reload() self.reload()
library = self.bf.code_index[b'LI'][0] library = self.bf.code_index[b"LI"][0]
self.assertEqual(b'//basic_file.blend', library[b'filepath']) self.assertEqual(b"//basic_file.blend", library[b"filepath"])
self.assertEqual(b'//basic_file.blend', library[b'name']) self.assertEqual(b"//basic_file.blend", library[b"name"])
def test_block_hash(self): def test_block_hash(self):
scene = self.bf.code_index[b'SC'][0] scene = self.bf.code_index[b"SC"][0]
assert isinstance(scene, blendfile.BlendFileBlock) assert isinstance(scene, blendfile.BlendFileBlock)
pre_hash = scene.hash() pre_hash = scene.hash()
@ -43,28 +45,30 @@ class ModifyUncompressedTest(AbstractBlendFileTest):
# Change the 'ed' pointer to some arbitrary value by hacking the blend file. # Change the 'ed' pointer to some arbitrary value by hacking the blend file.
psize = self.bf.header.pointer_size psize = self.bf.header.pointer_size
field, field_offset = scene.dna_type.field_from_path(psize, b'ed') field, field_offset = scene.dna_type.field_from_path(psize, b"ed")
self.bf.fileobj.seek(scene.file_offset + field_offset, os.SEEK_SET) self.bf.fileobj.seek(scene.file_offset + field_offset, os.SEEK_SET)
self.bf.fileobj.write(b'12345678'[:psize]) self.bf.fileobj.write(b"12345678"[:psize])
self.reload() self.reload()
scene = self.bf.code_index[b'SC'][0] scene = self.bf.code_index[b"SC"][0]
post_hash = scene.hash() post_hash = scene.hash()
self.assertEqual(pre_hash, post_hash) self.assertEqual(pre_hash, post_hash)
def reload(self): def reload(self):
self.bf.close() self.bf.close()
self.bf = blendfile.BlendFile(self.to_modify, mode='r+b') self.bf = blendfile.BlendFile(self.to_modify, mode="r+b")
class ModifyCompressedTest(AbstractBlendFileTest): class ModifyCompressedTest(AbstractBlendFileTest):
def setUp(self): def setUp(self):
self.orig = self.blendfiles / 'linked_cube_compressed.blend' self.orig = self.blendfiles / "linked_cube_compressed.blend"
self.to_modify = self.orig.with_name('linked_cube_modified.blend') self.to_modify = self.orig.with_name("linked_cube_modified.blend")
copyfile(str(self.orig), str(self.to_modify)) # TODO: when requiring Python 3.6+, remove str() copyfile(
self.bf = blendfile.BlendFile(self.to_modify, mode='r+b') str(self.orig), str(self.to_modify)
) # TODO: when requiring Python 3.6+, remove str()
self.bf = blendfile.BlendFile(self.to_modify, mode="r+b")
self.assertTrue(self.bf.is_compressed) self.assertTrue(self.bf.is_compressed)
@ -73,16 +77,16 @@ class ModifyCompressedTest(AbstractBlendFileTest):
self.to_modify.unlink() self.to_modify.unlink()
def test_change_path(self): def test_change_path(self):
library = self.bf.code_index[b'LI'][0] library = self.bf.code_index[b"LI"][0]
# Change it from absolute to relative. # Change it from absolute to relative.
library[b'filepath'] = b'//basic_file.blend' library[b"filepath"] = b"//basic_file.blend"
library[b'name'] = b'//basic_file.blend' library[b"name"] = b"//basic_file.blend"
# Reload the blend file to inspect that it was written properly. # Reload the blend file to inspect that it was written properly.
self.bf.close() self.bf.close()
self.bf = blendfile.BlendFile(self.to_modify, mode='r+b') self.bf = blendfile.BlendFile(self.to_modify, mode="r+b")
library = self.bf.code_index[b'LI'][0] library = self.bf.code_index[b"LI"][0]
self.assertEqual(b'//basic_file.blend', library[b'filepath']) self.assertEqual(b"//basic_file.blend", library[b"filepath"])
self.assertEqual(b'//basic_file.blend', library[b'name']) self.assertEqual(b"//basic_file.blend", library[b"name"])

373
tests/test_bpathlib.py
View File

@ -10,140 +10,201 @@ from blender_asset_tracer.bpathlib import BlendPath, make_absolute, strip_root
class BlendPathTest(unittest.TestCase): class BlendPathTest(unittest.TestCase):
def test_string_path(self): def test_string_path(self):
p = BlendPath(PurePosixPath('//some/file.blend')) p = BlendPath(PurePosixPath("//some/file.blend"))
self.assertEqual('//some/file.blend', str(PurePosixPath('//some/file.blend'))) self.assertEqual("//some/file.blend", str(PurePosixPath("//some/file.blend")))
self.assertEqual(b'//some/file.blend', p) self.assertEqual(b"//some/file.blend", p)
p = BlendPath(Path(r'C:\some\file.blend')) p = BlendPath(Path(r"C:\some\file.blend"))
self.assertEqual(b'C:/some/file.blend', p) self.assertEqual(b"C:/some/file.blend", p)
def test_invalid_type(self): def test_invalid_type(self):
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
BlendPath('//some/file.blend') BlendPath("//some/file.blend")
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
BlendPath(47) BlendPath(47)
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
BlendPath(None) BlendPath(None)
def test_repr(self): def test_repr(self):
p = BlendPath(b'//some/file.blend') p = BlendPath(b"//some/file.blend")
self.assertEqual("BlendPath(b'//some/file.blend')", repr(p)) self.assertEqual("BlendPath(b'//some/file.blend')", repr(p))
p = BlendPath(PurePosixPath('//some/file.blend')) p = BlendPath(PurePosixPath("//some/file.blend"))
self.assertEqual("BlendPath(b'//some/file.blend')", repr(p)) self.assertEqual("BlendPath(b'//some/file.blend')", repr(p))
def test_to_path(self): def test_to_path(self):
self.assertEqual(PurePath('/some/file.blend'), self.assertEqual(
BlendPath(b'/some/file.blend').to_path()) PurePath("/some/file.blend"), BlendPath(b"/some/file.blend").to_path()
self.assertEqual(PurePath('C:/some/file.blend'), )
BlendPath(b'C:/some/file.blend').to_path()) self.assertEqual(
self.assertEqual(PurePath('C:/some/file.blend'), PurePath("C:/some/file.blend"), BlendPath(b"C:/some/file.blend").to_path()
BlendPath(br'C:\some\file.blend').to_path()) )
self.assertEqual(
PurePath("C:/some/file.blend"), BlendPath(br"C:\some\file.blend").to_path()
)
with mock.patch('sys.getfilesystemencoding') as mock_getfse: with mock.patch("sys.getfilesystemencoding") as mock_getfse:
mock_getfse.return_value = 'latin1' mock_getfse.return_value = "latin1"
# \xe9 is Latin-1 for é, and BlendPath should revert to using the # \xe9 is Latin-1 for é, and BlendPath should revert to using the
# (mocked) filesystem encoding when decoding as UTF-8 fails. # (mocked) filesystem encoding when decoding as UTF-8 fails.
self.assertEqual(PurePath('C:/some/filé.blend'), self.assertEqual(
BlendPath(b'C:\\some\\fil\xe9.blend').to_path()) PurePath("C:/some/filé.blend"),
BlendPath(b"C:\\some\\fil\xe9.blend").to_path(),
)
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
BlendPath(b'//relative/path.jpg').to_path() BlendPath(b"//relative/path.jpg").to_path()
def test_is_absolute(self): def test_is_absolute(self):
self.assertFalse(BlendPath(b'//some/file.blend').is_absolute()) self.assertFalse(BlendPath(b"//some/file.blend").is_absolute())
self.assertTrue(BlendPath(b'/some/file.blend').is_absolute()) self.assertTrue(BlendPath(b"/some/file.blend").is_absolute())
self.assertTrue(BlendPath(b'C:/some/file.blend').is_absolute()) self.assertTrue(BlendPath(b"C:/some/file.blend").is_absolute())
self.assertTrue(BlendPath(b'C:\\some\\file.blend').is_absolute()) self.assertTrue(BlendPath(b"C:\\some\\file.blend").is_absolute())
self.assertFalse(BlendPath(b'some/file.blend').is_absolute()) self.assertFalse(BlendPath(b"some/file.blend").is_absolute())
def test_is_blendfile_relative(self): def test_is_blendfile_relative(self):
self.assertTrue(BlendPath(b'//some/file.blend').is_blendfile_relative()) self.assertTrue(BlendPath(b"//some/file.blend").is_blendfile_relative())
self.assertFalse(BlendPath(b'/some/file.blend').is_blendfile_relative()) self.assertFalse(BlendPath(b"/some/file.blend").is_blendfile_relative())
self.assertFalse(BlendPath(b'C:/some/file.blend').is_blendfile_relative()) self.assertFalse(BlendPath(b"C:/some/file.blend").is_blendfile_relative())
self.assertFalse(BlendPath(b'some/file.blend').is_blendfile_relative()) self.assertFalse(BlendPath(b"some/file.blend").is_blendfile_relative())
def test_make_absolute(self): def test_make_absolute(self):
self.assertEqual(b'/root/to/some/file.blend', self.assertEqual(
BlendPath(b'//some/file.blend').absolute(b'/root/to')) b"/root/to/some/file.blend",
self.assertEqual(b'/root/to/some/file.blend', BlendPath(b"//some/file.blend").absolute(b"/root/to"),
BlendPath(b'some/file.blend').absolute(b'/root/to')) )
self.assertEqual(b'/root/to/../some/file.blend', self.assertEqual(
BlendPath(b'../some/file.blend').absolute(b'/root/to')) b"/root/to/some/file.blend",
self.assertEqual(b'/shared/some/file.blend', BlendPath(b"some/file.blend").absolute(b"/root/to"),
BlendPath(b'/shared/some/file.blend').absolute(b'/root/to')) )
self.assertEqual(
b"/root/to/../some/file.blend",
BlendPath(b"../some/file.blend").absolute(b"/root/to"),
)
self.assertEqual(
b"/shared/some/file.blend",
BlendPath(b"/shared/some/file.blend").absolute(b"/root/to"),
)
def test_slash(self): def test_slash(self):
self.assertEqual(b'/root/and/parent.blend', BlendPath(b'/root/and') / b'parent.blend') self.assertEqual(
b"/root/and/parent.blend", BlendPath(b"/root/and") / b"parent.blend"
)
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
BlendPath(b'/root/and') / b'/parent.blend' BlendPath(b"/root/and") / b"/parent.blend"
self.assertEqual(b'/root/and/parent.blend', b'/root/and' / BlendPath(b'parent.blend')) self.assertEqual(
b"/root/and/parent.blend", b"/root/and" / BlendPath(b"parent.blend")
)
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
b'/root/and' / BlendPath(b'/parent.blend') b"/root/and" / BlendPath(b"/parent.blend")
# On Windows+Python 3.5.4 this resulted in b'//root//parent.blend', # On Windows+Python 3.5.4 this resulted in b'//root//parent.blend',
# but only if the root is a single term (so not b'//root/and/'). # but only if the root is a single term (so not b'//root/and/').
self.assertEqual(BlendPath(b'//root/parent.blend'), self.assertEqual(
BlendPath(b'//root/') / b'parent.blend') BlendPath(b"//root/parent.blend"), BlendPath(b"//root/") / b"parent.blend"
)
@unittest.skipIf(platform.system() == 'Windows', "POSIX paths cannot be used on Windows") @unittest.skipIf(
platform.system() == "Windows", "POSIX paths cannot be used on Windows"
)
def test_mkrelative_posix(self): def test_mkrelative_posix(self):
self.assertEqual(b'//asset.png', BlendPath.mkrelative( self.assertEqual(
Path('/path/to/asset.png'), b"//asset.png",
PurePosixPath('/path/to/bfile.blend'), BlendPath.mkrelative(
)) Path("/path/to/asset.png"),
self.assertEqual(b'//to/asset.png', BlendPath.mkrelative( PurePosixPath("/path/to/bfile.blend"),
Path('/path/to/asset.png'), ),
PurePosixPath('/path/bfile.blend'), )
)) self.assertEqual(
self.assertEqual(b'//../of/asset.png', BlendPath.mkrelative( b"//to/asset.png",
Path('/path/of/asset.png'), BlendPath.mkrelative(
PurePosixPath('/path/to/bfile.blend'), Path("/path/to/asset.png"),
)) PurePosixPath("/path/bfile.blend"),
self.assertEqual(b'//../../path/of/asset.png', BlendPath.mkrelative( ),
Path('/path/of/asset.png'), )
PurePosixPath('/some/weird/bfile.blend'), self.assertEqual(
)) b"//../of/asset.png",
self.assertEqual(b'//very/very/very/very/very/deep/asset.png', BlendPath.mkrelative( BlendPath.mkrelative(
Path('/path/to/very/very/very/very/very/deep/asset.png'), Path("/path/of/asset.png"),
PurePosixPath('/path/to/bfile.blend'), PurePosixPath("/path/to/bfile.blend"),
)) ),
self.assertEqual(b'//../../../../../../../../shallow/asset.png', BlendPath.mkrelative( )
Path('/shallow/asset.png'), self.assertEqual(
PurePosixPath('/path/to/very/very/very/very/very/deep/bfile.blend'), b"//../../path/of/asset.png",
)) BlendPath.mkrelative(
Path("/path/of/asset.png"),
PurePosixPath("/some/weird/bfile.blend"),
),
)
self.assertEqual(
b"//very/very/very/very/very/deep/asset.png",
BlendPath.mkrelative(
Path("/path/to/very/very/very/very/very/deep/asset.png"),
PurePosixPath("/path/to/bfile.blend"),
),
)
self.assertEqual(
b"//../../../../../../../../shallow/asset.png",
BlendPath.mkrelative(
Path("/shallow/asset.png"),
PurePosixPath("/path/to/very/very/very/very/very/deep/bfile.blend"),
),
)
@unittest.skipIf(platform.system() != 'Windows', "Windows paths cannot be used on POSIX") @unittest.skipIf(
platform.system() != "Windows", "Windows paths cannot be used on POSIX"
)
def test_mkrelative_windows(self): def test_mkrelative_windows(self):
self.assertEqual(b'//asset.png', BlendPath.mkrelative( self.assertEqual(
Path('Q:/path/to/asset.png'), b"//asset.png",
PureWindowsPath('Q:/path/to/bfile.blend'), BlendPath.mkrelative(
)) Path("Q:/path/to/asset.png"),
self.assertEqual(b'//to/asset.png', BlendPath.mkrelative( PureWindowsPath("Q:/path/to/bfile.blend"),
Path('Q:/path/to/asset.png'), ),
PureWindowsPath('Q:/path/bfile.blend'), )
)) self.assertEqual(
self.assertEqual(b'//../of/asset.png', BlendPath.mkrelative( b"//to/asset.png",
Path('Q:/path/of/asset.png'), BlendPath.mkrelative(
PureWindowsPath('Q:/path/to/bfile.blend'), Path("Q:/path/to/asset.png"),
)) PureWindowsPath("Q:/path/bfile.blend"),
self.assertEqual(b'//../../path/of/asset.png', BlendPath.mkrelative( ),
Path('Q:/path/of/asset.png'), )
PureWindowsPath('Q:/some/weird/bfile.blend'), self.assertEqual(
)) b"//../of/asset.png",
self.assertEqual(b'//very/very/very/very/very/deep/asset.png', BlendPath.mkrelative( BlendPath.mkrelative(
Path('Q:/path/to/very/very/very/very/very/deep/asset.png'), Path("Q:/path/of/asset.png"),
PureWindowsPath('Q:/path/to/bfile.blend'), PureWindowsPath("Q:/path/to/bfile.blend"),
)) ),
self.assertEqual(b'//../../../../../../../../shallow/asset.png', BlendPath.mkrelative( )
Path('Q:/shallow/asset.png'), self.assertEqual(
PureWindowsPath('Q:/path/to/very/very/very/very/very/deep/bfile.blend'), b"//../../path/of/asset.png",
)) BlendPath.mkrelative(
self.assertEqual(b'D:/path/to/asset.png', BlendPath.mkrelative( Path("Q:/path/of/asset.png"),
Path('D:/path/to/asset.png'), PureWindowsPath("Q:/some/weird/bfile.blend"),
PureWindowsPath('Q:/path/to/bfile.blend'), ),
)) )
self.assertEqual(
b"//very/very/very/very/very/deep/asset.png",
BlendPath.mkrelative(
Path("Q:/path/to/very/very/very/very/very/deep/asset.png"),
PureWindowsPath("Q:/path/to/bfile.blend"),
),
)
self.assertEqual(
b"//../../../../../../../../shallow/asset.png",
BlendPath.mkrelative(
Path("Q:/shallow/asset.png"),
PureWindowsPath("Q:/path/to/very/very/very/very/very/deep/bfile.blend"),
),
)
self.assertEqual(
b"D:/path/to/asset.png",
BlendPath.mkrelative(
Path("D:/path/to/asset.png"),
PureWindowsPath("Q:/path/to/bfile.blend"),
),
)
class MakeAbsoluteTest(unittest.TestCase): class MakeAbsoluteTest(unittest.TestCase):
@ -152,101 +213,127 @@ class MakeAbsoluteTest(unittest.TestCase):
cwd = os.getcwd() cwd = os.getcwd()
try: try:
os.chdir(my_dir) os.chdir(my_dir)
self.assertEqual(my_dir / 'blendfiles/Cube.btx', self.assertEqual(
make_absolute(Path('blendfiles/Cube.btx'))) my_dir / "blendfiles/Cube.btx",
make_absolute(Path("blendfiles/Cube.btx")),
)
except Exception: except Exception:
os.chdir(cwd) os.chdir(cwd)
raise raise
@unittest.skipIf(platform.system() != 'Windows', "This test uses drive letters") @unittest.skipIf(platform.system() != "Windows", "This test uses drive letters")
def test_relative_drive(self): def test_relative_drive(self):
cwd = os.getcwd() cwd = os.getcwd()
my_drive = Path(f'{Path(cwd).drive}/') my_drive = Path(f"{Path(cwd).drive}/")
self.assertEqual(my_drive / 'blendfiles/Cube.btx', self.assertEqual(
make_absolute(Path('/blendfiles/Cube.btx'))) my_drive / "blendfiles/Cube.btx",
make_absolute(Path("/blendfiles/Cube.btx")),
)
def test_drive_letters(self): def test_drive_letters(self):
"""PureWindowsPath should be accepted and work well on POSIX systems too.""" """PureWindowsPath should be accepted and work well on POSIX systems too."""
in_path = PureWindowsPath('R:/wrongroot/oops/../../path/to/a/file') in_path = PureWindowsPath("R:/wrongroot/oops/../../path/to/a/file")
expect_path = Path('R:/path/to/a/file') expect_path = Path("R:/path/to/a/file")
self.assertNotEqual(expect_path, in_path, 'pathlib should not automatically resolve ../') self.assertNotEqual(
expect_path, in_path, "pathlib should not automatically resolve ../"
)
self.assertEqual(expect_path, make_absolute(in_path)) self.assertEqual(expect_path, make_absolute(in_path))
@unittest.skipIf(platform.system() == 'Windows', "This test ignores drive letters") @unittest.skipIf(platform.system() == "Windows", "This test ignores drive letters")
def test_dotdot_dotdot_posix(self): def test_dotdot_dotdot_posix(self):
in_path = Path('/wrongroot/oops/../../path/to/a/file') in_path = Path("/wrongroot/oops/../../path/to/a/file")
expect_path = Path('/path/to/a/file') expect_path = Path("/path/to/a/file")
self.assertNotEqual(expect_path, in_path, 'pathlib should not automatically resolve ../') self.assertNotEqual(
expect_path, in_path, "pathlib should not automatically resolve ../"
)
self.assertEqual(expect_path, make_absolute(in_path)) self.assertEqual(expect_path, make_absolute(in_path))
@unittest.skipIf(platform.system() != 'Windows', "This test uses drive letters") @unittest.skipIf(platform.system() != "Windows", "This test uses drive letters")
def test_dotdot_dotdot_windows(self): def test_dotdot_dotdot_windows(self):
in_path = Path('Q:/wrongroot/oops/../../path/to/a/file') in_path = Path("Q:/wrongroot/oops/../../path/to/a/file")
expect_path = Path('Q:/path/to/a/file') expect_path = Path("Q:/path/to/a/file")
self.assertNotEqual(expect_path, in_path, 'pathlib should not automatically resolve ../') self.assertNotEqual(
expect_path, in_path, "pathlib should not automatically resolve ../"
)
self.assertEqual(expect_path, make_absolute(in_path)) self.assertEqual(expect_path, make_absolute(in_path))
@unittest.skipIf(platform.system() == 'Windows', "This test ignores drive letters") @unittest.skipIf(platform.system() == "Windows", "This test ignores drive letters")
def test_way_too_many_dotdot_posix(self): def test_way_too_many_dotdot_posix(self):
in_path = Path('/webroot/../../../../../etc/passwd') in_path = Path("/webroot/../../../../../etc/passwd")
expect_path = Path('/etc/passwd') expect_path = Path("/etc/passwd")
self.assertEqual(expect_path, make_absolute(in_path)) self.assertEqual(expect_path, make_absolute(in_path))
@unittest.skipIf(platform.system() != 'Windows', "This test uses drive letters") @unittest.skipIf(platform.system() != "Windows", "This test uses drive letters")
def test_way_too_many_dotdot_windows(self): def test_way_too_many_dotdot_windows(self):
in_path = Path('G:/webroot/../../../../../etc/passwd') in_path = Path("G:/webroot/../../../../../etc/passwd")
expect_path = Path('G:/etc/passwd') expect_path = Path("G:/etc/passwd")
self.assertEqual(expect_path, make_absolute(in_path)) self.assertEqual(expect_path, make_absolute(in_path))
@unittest.skipIf(platform.system() == 'Windows', @unittest.skipIf(
"Symlinks on Windows require Administrator rights") platform.system() == "Windows",
"Symlinks on Windows require Administrator rights",
)
def test_symlinks(self): def test_symlinks(self):
with tempfile.TemporaryDirectory(suffix="-bat-symlink-test") as tmpdir_str: with tempfile.TemporaryDirectory(suffix="-bat-symlink-test") as tmpdir_str:
tmpdir = Path(tmpdir_str) tmpdir = Path(tmpdir_str)
orig_path = tmpdir / 'some_file.txt' orig_path = tmpdir / "some_file.txt"
with orig_path.open('w') as outfile: with orig_path.open("w") as outfile:
outfile.write('this file exists now') outfile.write("this file exists now")
symlink = tmpdir / 'subdir' / 'linked.txt' symlink = tmpdir / "subdir" / "linked.txt"
symlink.parent.mkdir() symlink.parent.mkdir()
symlink.symlink_to(orig_path) symlink.symlink_to(orig_path)
self.assertEqual(symlink, make_absolute(symlink), 'Symlinks should not be resolved') self.assertEqual(
symlink, make_absolute(symlink), "Symlinks should not be resolved"
)
@unittest.skipIf(platform.system() != 'Windows', @unittest.skipIf(
"Drive letters mapped to network share can only be tested on Windows") platform.system() != "Windows",
@unittest.skip('Mapped drive letter testing should be mocked, but that is hard to do') "Drive letters mapped to network share can only be tested on Windows",
)
@unittest.skip(
"Mapped drive letter testing should be mocked, but that is hard to do"
)
def test_mapped_drive_letters(self): def test_mapped_drive_letters(self):
pass pass
def test_path_types(self): def test_path_types(self):
platorm_path = type(PurePath()) platorm_path = type(PurePath())
self.assertIsInstance(make_absolute(PureWindowsPath('/some/path')), platorm_path) self.assertIsInstance(
self.assertIsInstance(make_absolute(PurePosixPath('/some/path')), platorm_path) make_absolute(PureWindowsPath("/some/path")), platorm_path
)
self.assertIsInstance(make_absolute(PurePosixPath("/some/path")), platorm_path)
class StripRootTest(unittest.TestCase): class StripRootTest(unittest.TestCase):
def test_windows_paths(self): def test_windows_paths(self):
self.assertEqual(PurePosixPath(), strip_root(PureWindowsPath())) self.assertEqual(PurePosixPath(), strip_root(PureWindowsPath()))
self.assertEqual( self.assertEqual(
PurePosixPath('C/Program Files/Blender'), PurePosixPath("C/Program Files/Blender"),
strip_root(PureWindowsPath('C:/Program Files/Blender'))) strip_root(PureWindowsPath("C:/Program Files/Blender")),
)
self.assertEqual( self.assertEqual(
PurePosixPath('C/Program Files/Blender'), PurePosixPath("C/Program Files/Blender"),
strip_root(PureWindowsPath('C:\\Program Files\\Blender'))) strip_root(PureWindowsPath("C:\\Program Files\\Blender")),
)
self.assertEqual( self.assertEqual(
PurePosixPath('C/Program Files/Blender'), PurePosixPath("C/Program Files/Blender"),
strip_root(PureWindowsPath('C\\Program Files\\Blender'))) strip_root(PureWindowsPath("C\\Program Files\\Blender")),
)
def test_posix_paths(self): def test_posix_paths(self):
self.assertEqual(PurePosixPath(), strip_root(PurePosixPath())) self.assertEqual(PurePosixPath(), strip_root(PurePosixPath()))
self.assertEqual( self.assertEqual(
PurePosixPath('C/path/to/blender'), PurePosixPath("C/path/to/blender"),
strip_root(PurePosixPath('C:/path/to/blender'))) strip_root(PurePosixPath("C:/path/to/blender")),
)
self.assertEqual( self.assertEqual(
PurePosixPath('C/path/to/blender'), PurePosixPath("C/path/to/blender"),
strip_root(PurePosixPath('C/path/to/blender'))) strip_root(PurePosixPath("C/path/to/blender")),
)
self.assertEqual( self.assertEqual(
PurePosixPath('C/path/to/blender'), PurePosixPath("C/path/to/blender"),
strip_root(PurePosixPath('/C/path/to/blender'))) strip_root(PurePosixPath("/C/path/to/blender")),
)

30
tests/test_compressor.py
View File

@ -12,8 +12,8 @@ class CompressorTest(AbstractBlendFileTest):
def setUp(self): def setUp(self):
self.temp = tempfile.TemporaryDirectory() self.temp = tempfile.TemporaryDirectory()
tempdir = pathlib.Path(self.temp.name) tempdir = pathlib.Path(self.temp.name)
self.srcdir = tempdir / 'src' self.srcdir = tempdir / "src"
self.destdir = tempdir / 'dest' self.destdir = tempdir / "dest"
self.srcdir.mkdir() self.srcdir.mkdir()
self.destdir.mkdir() self.destdir.mkdir()
@ -43,31 +43,33 @@ class CompressorTest(AbstractBlendFileTest):
self.assertEqual(source_must_remain, srcfile.exists()) self.assertEqual(source_must_remain, srcfile.exists())
self.assertTrue(destfile.exists()) self.assertTrue(destfile.exists())
if destfile.suffix == '.blend': if destfile.suffix == ".blend":
self.bf = blendfile.BlendFile(destfile) self.bf = blendfile.BlendFile(destfile)
self.assertTrue(self.bf.is_compressed) self.assertTrue(self.bf.is_compressed)
return return
with destfile.open('rb') as infile: with destfile.open("rb") as infile:
magic = infile.read(3) magic = infile.read(3)
if destfile.suffix == '.jpg': if destfile.suffix == ".jpg":
self.assertEqual(b'\xFF\xD8\xFF', magic, self.assertEqual(
'Expected %s to be a JPEG' % destfile) b"\xFF\xD8\xFF", magic, "Expected %s to be a JPEG" % destfile
)
else: else:
self.assertNotEqual(b'\x1f\x8b', magic[:2], self.assertNotEqual(
'Expected %s to be NOT compressed' % destfile) b"\x1f\x8b", magic[:2], "Expected %s to be NOT compressed" % destfile
)
def test_move_already_compressed(self): def test_move_already_compressed(self):
self._test('basic_file_ñønæščii.blend', False) self._test("basic_file_ñønæščii.blend", False)
def test_move_compress_on_the_fly(self): def test_move_compress_on_the_fly(self):
self._test('basic_file.blend', False) self._test("basic_file.blend", False)
def test_copy_already_compressed(self): def test_copy_already_compressed(self):
self._test('basic_file_ñønæščii.blend', True) self._test("basic_file_ñønæščii.blend", True)
def test_copy_compress_on_the_fly(self): def test_copy_compress_on_the_fly(self):
self._test('basic_file.blend', True) self._test("basic_file.blend", True)
def test_move_jpeg(self): def test_move_jpeg(self):
self._test('textures/Bricks/brick_dotted_04-color.jpg', False) self._test("textures/Bricks/brick_dotted_04-color.jpg", False)

10
tests/test_mypy.py
View File

@ -13,11 +13,11 @@ class MypyRunnerTest(unittest.TestCase):
# /path/to/blender-asset-tracer/.tox/py37/lib/python3.7/site-packages is in the PYTHONPATH. # /path/to/blender-asset-tracer/.tox/py37/lib/python3.7/site-packages is in the PYTHONPATH.
# Please change directory so it is not. # Please change directory so it is not.
for path in sys.path: for path in sys.path:
if '/.tox/' in path and path.endswith('/site-packages'): if "/.tox/" in path and path.endswith("/site-packages"):
self.skipTest("Mypy doesn't like Tox") self.skipTest("Mypy doesn't like Tox")
path = pathlib.Path(blender_asset_tracer.__file__).parent path = pathlib.Path(blender_asset_tracer.__file__).parent
result = mypy.api.run(['--incremental', '--ignore-missing-imports', str(path)]) result = mypy.api.run(["--incremental", "--ignore-missing-imports", str(path)])
stdout, stderr, status = result stdout, stderr, status = result
@ -27,6 +27,6 @@ class MypyRunnerTest(unittest.TestCase):
if stdout: if stdout:
messages.append(stdout) messages.append(stdout)
if status: if status:
messages.append('Mypy failed with status %d' % status) messages.append("Mypy failed with status %d" % status)
if messages and not all(msg.startswith('Success: ') for msg in messages): if messages and not all(msg.startswith("Success: ") for msg in messages):
self.fail('\n'.join(['Mypy errors:'] + messages)) self.fail("\n".join(["Mypy errors:"] + messages))

437
tests/test_pack.py
View File

@ -17,15 +17,21 @@ class AbstractPackTest(AbstractBlendFileTest):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
super().setUpClass() super().setUpClass()
logging.getLogger('blender_asset_tracer.compressor').setLevel(logging.DEBUG) logging.getLogger("blender_asset_tracer.compressor").setLevel(logging.DEBUG)
logging.getLogger('blender_asset_tracer.pack').setLevel(logging.DEBUG) logging.getLogger("blender_asset_tracer.pack").setLevel(logging.DEBUG)
logging.getLogger('blender_asset_tracer.blendfile.open_cached').setLevel(logging.DEBUG) logging.getLogger("blender_asset_tracer.blendfile.open_cached").setLevel(
logging.getLogger('blender_asset_tracer.blendfile.open_cached').setLevel(logging.DEBUG) logging.DEBUG
logging.getLogger('blender_asset_tracer.blendfile.BlendFile').setLevel(logging.DEBUG) )
logging.getLogger("blender_asset_tracer.blendfile.open_cached").setLevel(
logging.DEBUG
)
logging.getLogger("blender_asset_tracer.blendfile.BlendFile").setLevel(
logging.DEBUG
)
def setUp(self): def setUp(self):
super().setUp() super().setUp()
self.tdir = tempfile.TemporaryDirectory(suffix='-packtest') self.tdir = tempfile.TemporaryDirectory(suffix="-packtest")
self.tpath = Path(self.tdir.name) self.tpath = Path(self.tdir.name)
def tearDown(self): def tearDown(self):
@ -34,120 +40,135 @@ class AbstractPackTest(AbstractBlendFileTest):
@staticmethod @staticmethod
def rewrites(packer: pack.Packer): def rewrites(packer: pack.Packer):
return {path: action.rewrites return {
for path, action in packer._actions.items() path: action.rewrites
if action.rewrites} for path, action in packer._actions.items()
if action.rewrites
}
def outside_project(self) -> Path: def outside_project(self) -> Path:
"""Return the '_outside_project' path for files in self.blendfiles.""" """Return the '_outside_project' path for files in self.blendfiles."""
# /tmp/target + /workspace/bat/tests/blendfiles → /tmp/target/workspace/bat/tests/blendfiles # /tmp/target + /workspace/bat/tests/blendfiles → /tmp/target/workspace/bat/tests/blendfiles
# /tmp/target + C:\workspace\bat\tests\blendfiles → /tmp/target/C/workspace/bat/tests/blendfiles # /tmp/target + C:\workspace\bat\tests\blendfiles → /tmp/target/C/workspace/bat/tests/blendfiles
extpath = Path(self.tpath, '_outside_project', bpathlib.strip_root(self.blendfiles)) extpath = Path(
self.tpath, "_outside_project", bpathlib.strip_root(self.blendfiles)
)
return extpath return extpath
class PackTest(AbstractPackTest): class PackTest(AbstractPackTest):
def test_strategise_no_rewrite_required(self): def test_strategise_no_rewrite_required(self):
infile = self.blendfiles / 'doubly_linked.blend' infile = self.blendfiles / "doubly_linked.blend"
packer = pack.Packer(infile, self.blendfiles, self.tpath) packer = pack.Packer(infile, self.blendfiles, self.tpath)
packer.strategise() packer.strategise()
packed_files = ( packed_files = (
'doubly_linked.blend', "doubly_linked.blend",
'linked_cube.blend', "linked_cube.blend",
'basic_file.blend', "basic_file.blend",
'material_textures.blend', "material_textures.blend",
'textures/Bricks/brick_dotted_04-bump.jpg', "textures/Bricks/brick_dotted_04-bump.jpg",
'textures/Bricks/brick_dotted_04-color.jpg', "textures/Bricks/brick_dotted_04-color.jpg",
) )
for pf in packed_files: for pf in packed_files:
path = self.blendfiles / pf path = self.blendfiles / pf
act = packer._actions[path] act = packer._actions[path]
self.assertEqual(pack.PathAction.KEEP_PATH, act.path_action, 'for %s' % pf) self.assertEqual(pack.PathAction.KEEP_PATH, act.path_action, "for %s" % pf)
self.assertEqual(self.tpath / pf, act.new_path, 'for %s' % pf) self.assertEqual(self.tpath / pf, act.new_path, "for %s" % pf)
self.assertEqual({}, self.rewrites(packer)) self.assertEqual({}, self.rewrites(packer))
self.assertEqual(len(packed_files), len(packer._actions)) self.assertEqual(len(packed_files), len(packer._actions))
def test_strategise_rewrite(self): def test_strategise_rewrite(self):
ppath = self.blendfiles / 'subdir' ppath = self.blendfiles / "subdir"
infile = ppath / 'doubly_linked_up-windows.blend' infile = ppath / "doubly_linked_up-windows.blend"
packer = pack.Packer(infile, ppath, self.tpath) packer = pack.Packer(infile, ppath, self.tpath)
packer.strategise() packer.strategise()
external_files = ( external_files = (
'linked_cube.blend', "linked_cube.blend",
'basic_file.blend', "basic_file.blend",
'material_textures.blend', "material_textures.blend",
'textures/Bricks/brick_dotted_04-bump.jpg', "textures/Bricks/brick_dotted_04-bump.jpg",
'textures/Bricks/brick_dotted_04-color.jpg', "textures/Bricks/brick_dotted_04-color.jpg",
) )
extpath = self.outside_project() extpath = self.outside_project()
act = packer._actions[infile] act = packer._actions[infile]
self.assertEqual(pack.PathAction.KEEP_PATH, act.path_action, 'for %s' % infile.name) self.assertEqual(
self.assertEqual(self.tpath / infile.name, act.new_path, 'for %s' % infile.name) pack.PathAction.KEEP_PATH, act.path_action, "for %s" % infile.name
)
self.assertEqual(self.tpath / infile.name, act.new_path, "for %s" % infile.name)
for fn in external_files: for fn in external_files:
path = self.blendfiles / fn path = self.blendfiles / fn
act = packer._actions[path] act = packer._actions[path]
self.assertEqual(pack.PathAction.FIND_NEW_LOCATION, act.path_action, 'for %s' % fn) self.assertEqual(
self.assertEqual(extpath / fn, act.new_path, pack.PathAction.FIND_NEW_LOCATION, act.path_action, "for %s" % fn
f'\nEXPECT: {extpath / fn}\nACTUAL: {act.new_path}\nfor {fn}') )
self.assertEqual(
extpath / fn,
act.new_path,
f"\nEXPECT: {extpath / fn}\nACTUAL: {act.new_path}\nfor {fn}",
)
to_rewrite = ( to_rewrite = (
'linked_cube.blend', "linked_cube.blend",
'material_textures.blend', "material_textures.blend",
str(infile.relative_to(self.blendfiles)), str(infile.relative_to(self.blendfiles)),
) )
rewrites = self.rewrites(packer) rewrites = self.rewrites(packer)
self.assertEqual([self.blendfiles / fn for fn in to_rewrite], self.assertEqual(
sorted(rewrites.keys())) [self.blendfiles / fn for fn in to_rewrite], sorted(rewrites.keys())
)
# Library link referencing basic_file.blend should (maybe) be rewritten. # Library link referencing basic_file.blend should (maybe) be rewritten.
rw_linked_cube = rewrites[self.blendfiles / 'linked_cube.blend'] rw_linked_cube = rewrites[self.blendfiles / "linked_cube.blend"]
self.assertEqual(1, len(rw_linked_cube)) self.assertEqual(1, len(rw_linked_cube))
self.assertEqual(b'LILib', rw_linked_cube[0].block_name) self.assertEqual(b"LILib", rw_linked_cube[0].block_name)
self.assertEqual(b'//basic_file.blend', rw_linked_cube[0].asset_path) self.assertEqual(b"//basic_file.blend", rw_linked_cube[0].asset_path)
# Texture links to image assets should (maybe) be rewritten. # Texture links to image assets should (maybe) be rewritten.
rw_mattex = rewrites[self.blendfiles / 'material_textures.blend'] rw_mattex = rewrites[self.blendfiles / "material_textures.blend"]
self.assertEqual(2, len(rw_mattex)) self.assertEqual(2, len(rw_mattex))
rw_mattex.sort() # for repeatable tests rw_mattex.sort() # for repeatable tests
self.assertEqual(b'IMbrick_dotted_04-bump', rw_mattex[0].block_name) self.assertEqual(b"IMbrick_dotted_04-bump", rw_mattex[0].block_name)
self.assertEqual(b'//textures/Bricks/brick_dotted_04-bump.jpg', rw_mattex[0].asset_path) self.assertEqual(
self.assertEqual(b'IMbrick_dotted_04-color', rw_mattex[1].block_name) b"//textures/Bricks/brick_dotted_04-bump.jpg", rw_mattex[0].asset_path
self.assertEqual(b'//textures/Bricks/brick_dotted_04-color.jpg', rw_mattex[1].asset_path) )
self.assertEqual(b"IMbrick_dotted_04-color", rw_mattex[1].block_name)
self.assertEqual(
b"//textures/Bricks/brick_dotted_04-color.jpg", rw_mattex[1].asset_path
)
# Library links from doubly_linked_up.blend to the above to blend files should be rewritten. # Library links from doubly_linked_up.blend to the above to blend files should be rewritten.
rw_dbllink = rewrites[infile] rw_dbllink = rewrites[infile]
self.assertEqual(2, len(rw_dbllink)) self.assertEqual(2, len(rw_dbllink))
rw_dbllink.sort() # for repeatable tests rw_dbllink.sort() # for repeatable tests
self.assertEqual(b'LILib', rw_dbllink[0].block_name) self.assertEqual(b"LILib", rw_dbllink[0].block_name)
self.assertEqual(b'//../linked_cube.blend', rw_dbllink[0].asset_path) self.assertEqual(b"//../linked_cube.blend", rw_dbllink[0].asset_path)
self.assertEqual(b'LILib.002', rw_dbllink[1].block_name) self.assertEqual(b"LILib.002", rw_dbllink[1].block_name)
self.assertEqual(b'//../material_textures.blend', rw_dbllink[1].asset_path) self.assertEqual(b"//../material_textures.blend", rw_dbllink[1].asset_path)
def test_strategise_relative_only(self): def test_strategise_relative_only(self):
infile = self.blendfiles / 'absolute_path.blend' infile = self.blendfiles / "absolute_path.blend"
packer = pack.Packer(infile, self.blendfiles, self.tpath, packer = pack.Packer(infile, self.blendfiles, self.tpath, relative_only=True)
relative_only=True)
packer.strategise() packer.strategise()
packed_files = ( packed_files = (
'absolute_path.blend', "absolute_path.blend",
# Linked with a relative path: # Linked with a relative path:
'textures/Bricks/brick_dotted_04-color.jpg', "textures/Bricks/brick_dotted_04-color.jpg",
# This file links to textures/Textures/Buildings/buildings_roof_04-color.png, # This file links to textures/Textures/Buildings/buildings_roof_04-color.png,
# but using an absolute path, so that file should be skipped. # but using an absolute path, so that file should be skipped.
) )
for pf in packed_files: for pf in packed_files:
path = self.blendfiles / pf path = self.blendfiles / pf
act = packer._actions[path] act = packer._actions[path]
self.assertEqual(pack.PathAction.KEEP_PATH, act.path_action, 'for %s' % pf) self.assertEqual(pack.PathAction.KEEP_PATH, act.path_action, "for %s" % pf)
self.assertEqual(self.tpath / pf, act.new_path, 'for %s' % pf) self.assertEqual(self.tpath / pf, act.new_path, "for %s" % pf)
self.assertEqual(len(packed_files), len(packer._actions)) self.assertEqual(len(packed_files), len(packer._actions))
self.assertEqual({}, self.rewrites(packer)) self.assertEqual({}, self.rewrites(packer))
@ -157,31 +178,33 @@ class PackTest(AbstractPackTest):
# The original file shouldn't be touched. # The original file shouldn't be touched.
bfile = blendfile.open_cached(infile, assert_cached=False) bfile = blendfile.open_cached(infile, assert_cached=False)
libs = sorted(bfile.code_index[b'LI']) libs = sorted(bfile.code_index[b"LI"])
self.assertEqual(b'LILib', libs[0].id_name) self.assertEqual(b"LILib", libs[0].id_name)
self.assertEqual(b'//../linked_cube.blend', libs[0][b'name']) self.assertEqual(b"//../linked_cube.blend", libs[0][b"name"])
self.assertEqual(b'LILib.002', libs[1].id_name) self.assertEqual(b"LILib.002", libs[1].id_name)
self.assertEqual(b'//../material_textures.blend', libs[1][b'name']) self.assertEqual(b"//../material_textures.blend", libs[1][b"name"])
def test_execute_rewrite(self): def test_execute_rewrite(self):
infile, _ = self._pack_with_rewrite() infile, _ = self._pack_with_rewrite()
if platform.system() == 'Windows': if platform.system() == "Windows":
extpath = PurePosixPath('//_outside_project', extpath = PurePosixPath(
self.blendfiles.drive[0], "//_outside_project",
*self.blendfiles.parts[1:]) self.blendfiles.drive[0],
*self.blendfiles.parts[1:],
)
else: else:
extpath = PurePosixPath('//_outside_project', *self.blendfiles.parts[1:]) extpath = PurePosixPath("//_outside_project", *self.blendfiles.parts[1:])
extbpath = bpathlib.BlendPath(extpath) extbpath = bpathlib.BlendPath(extpath)
# Those libraries should be properly rewritten. # Those libraries should be properly rewritten.
bfile = blendfile.open_cached(self.tpath / infile.name, assert_cached=False) bfile = blendfile.open_cached(self.tpath / infile.name, assert_cached=False)
libs = sorted(bfile.code_index[b'LI']) libs = sorted(bfile.code_index[b"LI"])
self.assertEqual(b'LILib', libs[0].id_name) self.assertEqual(b"LILib", libs[0].id_name)
self.assertEqual(extbpath / b'linked_cube.blend', libs[0][b'name']) self.assertEqual(extbpath / b"linked_cube.blend", libs[0][b"name"])
self.assertEqual(b'LILib.002', libs[1].id_name) self.assertEqual(b"LILib.002", libs[1].id_name)
self.assertEqual(extbpath / b'material_textures.blend', libs[1][b'name']) self.assertEqual(extbpath / b"material_textures.blend", libs[1][b"name"])
def test_execute_rewrite_cleanup(self): def test_execute_rewrite_cleanup(self):
infile, packer = self._pack_with_rewrite() infile, packer = self._pack_with_rewrite()
@ -195,8 +218,10 @@ class PackTest(AbstractPackTest):
packer.close() packer.close()
self.assertFalse(packer._rewrite_in.exists()) self.assertFalse(packer._rewrite_in.exists())
@unittest.skipIf(platform.system() == 'Windows', @unittest.skipIf(
"Symlinks on Windows require Administrator rights") platform.system() == "Windows",
"Symlinks on Windows require Administrator rights",
)
def test_symlinked_files(self): def test_symlinked_files(self):
"""Test that symlinks are NOT resolved. """Test that symlinks are NOT resolved.
@ -206,7 +231,7 @@ class PackTest(AbstractPackTest):
As a concrete example, a directory structure with only symlinked files As a concrete example, a directory structure with only symlinked files
in it should still be BAT-packable and produce the same structure. in it should still be BAT-packable and produce the same structure.
""" """
orig_ppath = self.blendfiles / 'subdir' orig_ppath = self.blendfiles / "subdir"
# This is the original structure when packing subdir/doubly_linked_up.blend: # This is the original structure when packing subdir/doubly_linked_up.blend:
# . # .
@ -225,12 +250,12 @@ class PackTest(AbstractPackTest):
# should be no different than packing the originals. # should be no different than packing the originals.
orig_paths = [ orig_paths = [
Path('basic_file.blend'), Path("basic_file.blend"),
Path('linked_cube.blend'), Path("linked_cube.blend"),
Path('material_textures.blend'), Path("material_textures.blend"),
Path('subdir/doubly_linked_up.blend'), Path("subdir/doubly_linked_up.blend"),
Path('textures/Bricks/brick_dotted_04-bump.jpg'), Path("textures/Bricks/brick_dotted_04-bump.jpg"),
Path('textures/Bricks/brick_dotted_04-color.jpg'), Path("textures/Bricks/brick_dotted_04-color.jpg"),
] ]
import hashlib import hashlib
@ -238,14 +263,14 @@ class PackTest(AbstractPackTest):
with tempfile.TemporaryDirectory(suffix="-bat-symlink") as tmpdir_str: with tempfile.TemporaryDirectory(suffix="-bat-symlink") as tmpdir_str:
tmpdir = Path(tmpdir_str) tmpdir = Path(tmpdir_str)
real_file_dir = tmpdir / 'real' real_file_dir = tmpdir / "real"
symlinked_dir = tmpdir / 'symlinked' symlinked_dir = tmpdir / "symlinked"
real_file_dir.mkdir() real_file_dir.mkdir()
symlinked_dir.mkdir() symlinked_dir.mkdir()
for orig_path in orig_paths: for orig_path in orig_paths:
hashed_name = hashlib.new('md5', bytes(orig_path)).hexdigest() hashed_name = hashlib.new("md5", bytes(orig_path)).hexdigest()
# Copy the file to the temporary project, under a hashed name. # Copy the file to the temporary project, under a hashed name.
# This will break Blendfile linking. # This will break Blendfile linking.
real_file_path = real_file_dir / hashed_name real_file_path = real_file_dir / hashed_name
@ -259,20 +284,22 @@ class PackTest(AbstractPackTest):
symlink.symlink_to(real_file_path) symlink.symlink_to(real_file_path)
# Pack the symlinked directory structure. # Pack the symlinked directory structure.
pack_dir = tmpdir / 'packed' pack_dir = tmpdir / "packed"
packer = pack.Packer(self.blendfiles / 'subdir/doubly_linked_up.blend', packer = pack.Packer(
self.blendfiles, self.blendfiles / "subdir/doubly_linked_up.blend",
pack_dir) self.blendfiles,
pack_dir,
)
packer.strategise() packer.strategise()
packer.execute() packer.execute()
for orig_path in orig_paths: for orig_path in orig_paths:
packed_path = pack_dir / orig_path packed_path = pack_dir / orig_path
self.assertTrue(packed_path.exists(), f'{packed_path} should exist') self.assertTrue(packed_path.exists(), f"{packed_path} should exist")
def _pack_with_rewrite(self): def _pack_with_rewrite(self):
ppath = self.blendfiles / 'subdir' ppath = self.blendfiles / "subdir"
infile = ppath / 'doubly_linked_up.blend' infile = ppath / "doubly_linked_up.blend"
packer = pack.Packer(infile, ppath, self.tpath) packer = pack.Packer(infile, ppath, self.tpath)
packer.strategise() packer.strategise()
@ -281,34 +308,34 @@ class PackTest(AbstractPackTest):
return infile, packer return infile, packer
def test_rewrite_sequence(self): def test_rewrite_sequence(self):
ppath = self.blendfiles / 'subdir' ppath = self.blendfiles / "subdir"
infile = ppath / 'image_sequence_dir_up.blend' infile = ppath / "image_sequence_dir_up.blend"
with pack.Packer(infile, ppath, self.tpath) as packer: with pack.Packer(infile, ppath, self.tpath) as packer:
packer.strategise() packer.strategise()
packer.execute() packer.execute()
bf = blendfile.open_cached(self.tpath / infile.name, assert_cached=False) bf = blendfile.open_cached(self.tpath / infile.name, assert_cached=False)
scene = bf.code_index[b'SC'][0] scene = bf.code_index[b"SC"][0]
ed = scene.get_pointer(b'ed') ed = scene.get_pointer(b"ed")
seq = ed.get_pointer((b'seqbase', b'first')) seq = ed.get_pointer((b"seqbase", b"first"))
seq_strip = seq.get_pointer(b'strip') seq_strip = seq.get_pointer(b"strip")
imgseq_path = bpathlib.make_absolute(self.blendfiles / 'imgseq') imgseq_path = bpathlib.make_absolute(self.blendfiles / "imgseq")
print(f'imgseq_path: {imgseq_path!r}') print(f"imgseq_path: {imgseq_path!r}")
print(f' anchor: {imgseq_path.anchor!r}') print(f" anchor: {imgseq_path.anchor!r}")
as_bytes = bpathlib.strip_root(imgseq_path).as_posix().encode() as_bytes = bpathlib.strip_root(imgseq_path).as_posix().encode()
print(f'as_bytes: {as_bytes!r}') print(f"as_bytes: {as_bytes!r}")
relpath = bpathlib.BlendPath(b'//_outside_project') / as_bytes relpath = bpathlib.BlendPath(b"//_outside_project") / as_bytes
print(f'relpath: {relpath!r}') print(f"relpath: {relpath!r}")
# The image sequence base path should be rewritten. # The image sequence base path should be rewritten.
self.assertEqual(b'SQ000210.png', seq[b'name']) self.assertEqual(b"SQ000210.png", seq[b"name"])
self.assertEqual(relpath, seq_strip[b'dir']) self.assertEqual(relpath, seq_strip[b"dir"])
def test_noop(self): def test_noop(self):
ppath = self.blendfiles / 'subdir' ppath = self.blendfiles / "subdir"
infile = ppath / 'doubly_linked_up.blend' infile = ppath / "doubly_linked_up.blend"
packer = pack.Packer(infile, ppath, self.tpath, noop=True) packer = pack.Packer(infile, ppath, self.tpath, noop=True)
packer.strategise() packer.strategise()
@ -318,71 +345,77 @@ class PackTest(AbstractPackTest):
# The original file shouldn't be touched. # The original file shouldn't be touched.
bfile = blendfile.open_cached(infile) bfile = blendfile.open_cached(infile)
libs = sorted(bfile.code_index[b'LI']) libs = sorted(bfile.code_index[b"LI"])
self.assertEqual(b'LILib', libs[0].id_name) self.assertEqual(b"LILib", libs[0].id_name)
self.assertEqual(b'//../linked_cube.blend', libs[0][b'name']) self.assertEqual(b"//../linked_cube.blend", libs[0][b"name"])
self.assertEqual(b'LILib.002', libs[1].id_name) self.assertEqual(b"LILib.002", libs[1].id_name)
self.assertEqual(b'//../material_textures.blend', libs[1][b'name']) self.assertEqual(b"//../material_textures.blend", libs[1][b"name"])
def test_missing_files(self): def test_missing_files(self):
infile = self.blendfiles / 'missing_textures.blend' infile = self.blendfiles / "missing_textures.blend"
packer = pack.Packer(infile, self.blendfiles, self.tpath) packer = pack.Packer(infile, self.blendfiles, self.tpath)
packer.strategise() packer.strategise()
self.assertEqual( self.assertEqual(
[self.blendfiles / 'textures/HDRI/Myanmar/Golden Palace 2, Old Bagan-1k.exr', [
self.blendfiles / 'textures/Textures/Marble/marble_decoration-color.png'], self.blendfiles
sorted(packer.missing_files) / "textures/HDRI/Myanmar/Golden Palace 2, Old Bagan-1k.exr",
self.blendfiles
/ "textures/Textures/Marble/marble_decoration-color.png",
],
sorted(packer.missing_files),
) )
def test_exclude_filter(self): def test_exclude_filter(self):
# Files shouldn't be reported missing if they should be ignored. # Files shouldn't be reported missing if they should be ignored.
infile = self.blendfiles / 'image_sequencer.blend' infile = self.blendfiles / "image_sequencer.blend"
with pack.Packer(infile, self.blendfiles, self.tpath) as packer: with pack.Packer(infile, self.blendfiles, self.tpath) as packer:
packer.exclude('*.png', '*.nonsense') packer.exclude("*.png", "*.nonsense")
packer.strategise() packer.strategise()
packer.execute() packer.execute()
self.assertFalse((self.tpath / 'imgseq').exists()) self.assertFalse((self.tpath / "imgseq").exists())
def test_exclude_filter_missing_files(self): def test_exclude_filter_missing_files(self):
# Files shouldn't be reported missing if they should be ignored. # Files shouldn't be reported missing if they should be ignored.
infile = self.blendfiles / 'missing_textures.blend' infile = self.blendfiles / "missing_textures.blend"
with pack.Packer(infile, self.blendfiles, self.tpath) as packer: with pack.Packer(infile, self.blendfiles, self.tpath) as packer:
packer.exclude('*.png') packer.exclude("*.png")
packer.strategise() packer.strategise()
self.assertEqual( self.assertEqual(
[self.blendfiles / 'textures/HDRI/Myanmar/Golden Palace 2, Old Bagan-1k.exr'], [
list(packer.missing_files) self.blendfiles
/ "textures/HDRI/Myanmar/Golden Palace 2, Old Bagan-1k.exr"
],
list(packer.missing_files),
) )
def test_output_path(self): def test_output_path(self):
infile = self.blendfiles / 'basic_file.blend' infile = self.blendfiles / "basic_file.blend"
packer = pack.Packer(infile, self.blendfiles.parent, self.tpath) packer = pack.Packer(infile, self.blendfiles.parent, self.tpath)
packer.strategise() packer.strategise()
self.assertEqual( self.assertEqual(
self.tpath / self.blendfiles.name / infile.name, self.tpath / self.blendfiles.name / infile.name, packer.output_path
packer.output_path
) )
def test_infofile(self): def test_infofile(self):
blendname = 'subdir/doubly_linked_up.blend' blendname = "subdir/doubly_linked_up.blend"
infile = self.blendfiles / blendname infile = self.blendfiles / blendname
packer = pack.Packer(infile, self.blendfiles, self.tpath) packer = pack.Packer(infile, self.blendfiles, self.tpath)
packer.strategise() packer.strategise()
packer.execute() packer.execute()
infopath = self.tpath / 'pack-info.txt' infopath = self.tpath / "pack-info.txt"
self.assertTrue(infopath.exists()) self.assertTrue(infopath.exists())
info = infopath.open().read().splitlines(keepends=False) info = infopath.open().read().splitlines(keepends=False)
self.assertEqual(blendname, info[-1].strip()) self.assertEqual(blendname, info[-1].strip())
def test_compression(self): def test_compression(self):
blendname = 'subdir/doubly_linked_up.blend' blendname = "subdir/doubly_linked_up.blend"
imgfile = self.blendfiles / blendname imgfile = self.blendfiles / blendname
packer = pack.Packer(imgfile, self.blendfiles, self.tpath, compress=True) packer = pack.Packer(imgfile, self.blendfiles, self.tpath, compress=True)
@ -393,31 +426,34 @@ class PackTest(AbstractPackTest):
self.assertTrue(dest.exists()) self.assertTrue(dest.exists())
self.assertTrue(blendfile.open_cached(dest).is_compressed) self.assertTrue(blendfile.open_cached(dest).is_compressed)
for bpath in self.tpath.rglob('*.blend'): for bpath in self.tpath.rglob("*.blend"):
if bpath == dest: if bpath == dest:
# Only test files that were bundled as dependency; the main # Only test files that were bundled as dependency; the main
# file was tested above already. # file was tested above already.
continue continue
self.assertTrue(blendfile.open_cached(bpath).is_compressed, self.assertTrue(
'Expected %s to be compressed' % bpath) blendfile.open_cached(bpath).is_compressed,
"Expected %s to be compressed" % bpath,
)
break break
else: else:
self.fail(f'Expected to have Blend files in the BAT pack at {self.tpath}.') self.fail(f"Expected to have Blend files in the BAT pack at {self.tpath}.")
for imgpath in self.tpath.rglob('*.jpg'): for imgpath in self.tpath.rglob("*.jpg"):
with imgpath.open('rb') as imgfile: with imgpath.open("rb") as imgfile:
magic = imgfile.read(3) magic = imgfile.read(3)
self.assertEqual(b'\xFF\xD8\xFF', magic, self.assertEqual(
'Expected %s to NOT be compressed' % imgpath) b"\xFF\xD8\xFF", magic, "Expected %s to NOT be compressed" % imgpath
)
break break
else: else:
self.fail(f'Expected to have JPEG files in the BAT pack at {self.tpath}.') self.fail(f"Expected to have JPEG files in the BAT pack at {self.tpath}.")
class ProgressTest(AbstractPackTest): class ProgressTest(AbstractPackTest):
def test_strategise(self): def test_strategise(self):
cb = mock.Mock(progress.Callback) cb = mock.Mock(progress.Callback)
infile = self.blendfiles / 'subdir/doubly_linked_up.blend' infile = self.blendfiles / "subdir/doubly_linked_up.blend"
with pack.Packer(infile, self.blendfiles, self.tpath) as packer: with pack.Packer(infile, self.blendfiles, self.tpath) as packer:
packer.progress_cb = cb packer.progress_cb = cb
packer.strategise() packer.strategise()
@ -426,20 +462,20 @@ class ProgressTest(AbstractPackTest):
self.assertEqual(0, cb.pack_done.call_count) self.assertEqual(0, cb.pack_done.call_count)
expected_calls = [ expected_calls = [
mock.call(self.blendfiles / 'subdir/doubly_linked_up.blend'), mock.call(self.blendfiles / "subdir/doubly_linked_up.blend"),
mock.call(self.blendfiles / 'linked_cube.blend'), mock.call(self.blendfiles / "linked_cube.blend"),
mock.call(self.blendfiles / 'basic_file.blend'), mock.call(self.blendfiles / "basic_file.blend"),
mock.call(self.blendfiles / 'material_textures.blend'), mock.call(self.blendfiles / "material_textures.blend"),
] ]
cb.trace_blendfile.assert_has_calls(expected_calls, any_order=True) cb.trace_blendfile.assert_has_calls(expected_calls, any_order=True)
self.assertEqual(len(expected_calls), cb.trace_blendfile.call_count) self.assertEqual(len(expected_calls), cb.trace_blendfile.call_count)
expected_calls = [ expected_calls = [
mock.call(self.blendfiles / 'linked_cube.blend'), mock.call(self.blendfiles / "linked_cube.blend"),
mock.call(self.blendfiles / 'basic_file.blend'), mock.call(self.blendfiles / "basic_file.blend"),
mock.call(self.blendfiles / 'material_textures.blend'), mock.call(self.blendfiles / "material_textures.blend"),
mock.call(self.blendfiles / 'textures/Bricks/brick_dotted_04-color.jpg'), mock.call(self.blendfiles / "textures/Bricks/brick_dotted_04-color.jpg"),
mock.call(self.blendfiles / 'textures/Bricks/brick_dotted_04-bump.jpg'), mock.call(self.blendfiles / "textures/Bricks/brick_dotted_04-bump.jpg"),
] ]
cb.trace_asset.assert_has_calls(expected_calls, any_order=True) cb.trace_asset.assert_has_calls(expected_calls, any_order=True)
self.assertEqual(len(expected_calls), cb.trace_asset.call_count) self.assertEqual(len(expected_calls), cb.trace_asset.call_count)
@ -452,7 +488,7 @@ class ProgressTest(AbstractPackTest):
def test_execute_with_rewrite(self): def test_execute_with_rewrite(self):
cb = mock.Mock(progress.Callback) cb = mock.Mock(progress.Callback)
infile = self.blendfiles / 'subdir/doubly_linked_up.blend' infile = self.blendfiles / "subdir/doubly_linked_up.blend"
with pack.Packer(infile, infile.parent, self.tpath) as packer: with pack.Packer(infile, infile.parent, self.tpath) as packer:
packer.progress_cb = cb packer.progress_cb = cb
packer.strategise() packer.strategise()
@ -463,34 +499,43 @@ class ProgressTest(AbstractPackTest):
# rewrite_blendfile should only be called paths in a blendfile are # rewrite_blendfile should only be called paths in a blendfile are
# actually rewritten. # actually rewritten.
cb.rewrite_blendfile.assert_called_with(self.blendfiles / 'subdir/doubly_linked_up.blend') cb.rewrite_blendfile.assert_called_with(
self.blendfiles / "subdir/doubly_linked_up.blend"
)
self.assertEqual(1, cb.rewrite_blendfile.call_count) self.assertEqual(1, cb.rewrite_blendfile.call_count)
# mock.ANY is used for temporary files in temporary paths, because they # mock.ANY is used for temporary files in temporary paths, because they
# are hard to predict. # are hard to predict.
extpath = self.outside_project() extpath = self.outside_project()
expected_calls = [ expected_calls = [
mock.call(mock.ANY, self.tpath / 'doubly_linked_up.blend'), mock.call(mock.ANY, self.tpath / "doubly_linked_up.blend"),
mock.call(mock.ANY, self.tpath / 'pack-info.txt'), mock.call(mock.ANY, self.tpath / "pack-info.txt"),
mock.call(mock.ANY, extpath / 'linked_cube.blend'), mock.call(mock.ANY, extpath / "linked_cube.blend"),
mock.call(mock.ANY, extpath / 'basic_file.blend'), mock.call(mock.ANY, extpath / "basic_file.blend"),
mock.call(mock.ANY, extpath / 'material_textures.blend'), mock.call(mock.ANY, extpath / "material_textures.blend"),
mock.call(self.blendfiles / 'textures/Bricks/brick_dotted_04-color.jpg', mock.call(
extpath / 'textures/Bricks/brick_dotted_04-color.jpg'), self.blendfiles / "textures/Bricks/brick_dotted_04-color.jpg",
mock.call(self.blendfiles / 'textures/Bricks/brick_dotted_04-bump.jpg', extpath / "textures/Bricks/brick_dotted_04-color.jpg",
extpath / 'textures/Bricks/brick_dotted_04-bump.jpg'), ),
mock.call(
self.blendfiles / "textures/Bricks/brick_dotted_04-bump.jpg",
extpath / "textures/Bricks/brick_dotted_04-bump.jpg",
),
] ]
cb.transfer_file.assert_has_calls(expected_calls, any_order=True) cb.transfer_file.assert_has_calls(expected_calls, any_order=True)
self.assertEqual(len(expected_calls), cb.transfer_file.call_count) self.assertEqual(len(expected_calls), cb.transfer_file.call_count)
self.assertEqual(0, cb.transfer_file_skipped.call_count) self.assertEqual(0, cb.transfer_file_skipped.call_count)
self.assertGreaterEqual(cb.transfer_progress.call_count, 6, self.assertGreaterEqual(
'transfer_progress() should be called at least once per asset') cb.transfer_progress.call_count,
6,
"transfer_progress() should be called at least once per asset",
)
self.assertEqual(0, cb.missing_file.call_count) self.assertEqual(0, cb.missing_file.call_count)
def test_missing_files(self): def test_missing_files(self):
cb = mock.Mock(progress.Callback) cb = mock.Mock(progress.Callback)
infile = self.blendfiles / 'missing_textures.blend' infile = self.blendfiles / "missing_textures.blend"
with pack.Packer(infile, self.blendfiles, self.tpath) as packer: with pack.Packer(infile, self.blendfiles, self.tpath) as packer:
packer.progress_cb = cb packer.progress_cb = cb
packer.strategise() packer.strategise()
@ -500,18 +545,29 @@ class ProgressTest(AbstractPackTest):
self.assertEqual(1, cb.pack_done.call_count) self.assertEqual(1, cb.pack_done.call_count)
cb.rewrite_blendfile.assert_not_called() cb.rewrite_blendfile.assert_not_called()
cb.transfer_file.assert_has_calls([ cb.transfer_file.assert_has_calls(
mock.call(infile, self.tpath / 'missing_textures.blend'), [
mock.call(mock.ANY, self.tpath / 'pack-info.txt'), mock.call(infile, self.tpath / "missing_textures.blend"),
], any_order=True) mock.call(mock.ANY, self.tpath / "pack-info.txt"),
],
any_order=True,
)
self.assertEqual(0, cb.transfer_file_skipped.call_count) self.assertEqual(0, cb.transfer_file_skipped.call_count)
self.assertGreaterEqual(cb.transfer_progress.call_count, 1, self.assertGreaterEqual(
'transfer_progress() should be called at least once per asset') cb.transfer_progress.call_count,
1,
"transfer_progress() should be called at least once per asset",
)
expected_calls = [ expected_calls = [
mock.call(self.blendfiles / 'textures/HDRI/Myanmar/Golden Palace 2, Old Bagan-1k.exr'), mock.call(
mock.call(self.blendfiles / 'textures/Textures/Marble/marble_decoration-color.png'), self.blendfiles
/ "textures/HDRI/Myanmar/Golden Palace 2, Old Bagan-1k.exr"
),
mock.call(
self.blendfiles / "textures/Textures/Marble/marble_decoration-color.png"
),
] ]
cb.missing_file.assert_has_calls(expected_calls, any_order=True) cb.missing_file.assert_has_calls(expected_calls, any_order=True)
self.assertEqual(len(expected_calls), cb.missing_file.call_count) self.assertEqual(len(expected_calls), cb.missing_file.call_count)
@ -519,64 +575,79 @@ class ProgressTest(AbstractPackTest):
def test_particle_cache(self): def test_particle_cache(self):
# The particle cache uses a glob to indicate which files to pack. # The particle cache uses a glob to indicate which files to pack.
cb = mock.Mock(progress.Callback) cb = mock.Mock(progress.Callback)
infile = self.blendfiles / 'T55539-particles/particle.blend' infile = self.blendfiles / "T55539-particles/particle.blend"
with pack.Packer(infile, self.blendfiles, self.tpath) as packer: with pack.Packer(infile, self.blendfiles, self.tpath) as packer:
packer.progress_cb = cb packer.progress_cb = cb
packer.strategise() packer.strategise()
packer.execute() packer.execute()
# We should have all the *.bphys files now. # We should have all the *.bphys files now.
count = len(list((self.tpath / 'T55539-particles/blendcache_particle').glob('*.bphys'))) count = len(
list((self.tpath / "T55539-particles/blendcache_particle").glob("*.bphys"))
)
self.assertEqual(27, count) self.assertEqual(27, count)
# Physics files + particle.blend + pack_info.txt # Physics files + particle.blend + pack_info.txt
self.assertGreaterEqual(cb.transfer_progress.call_count, 29, self.assertGreaterEqual(
'transfer_progress() should be called at least once per asset') cb.transfer_progress.call_count,
29,
"transfer_progress() should be called at least once per asset",
)
def test_particle_cache_with_ignore_glob(self): def test_particle_cache_with_ignore_glob(self):
cb = mock.Mock(progress.Callback) cb = mock.Mock(progress.Callback)
infile = self.blendfiles / 'T55539-particles/particle.blend' infile = self.blendfiles / "T55539-particles/particle.blend"
with pack.Packer(infile, self.blendfiles, self.tpath) as packer: with pack.Packer(infile, self.blendfiles, self.tpath) as packer:
packer.progress_cb = cb packer.progress_cb = cb
packer.exclude('*.bphys') packer.exclude("*.bphys")
packer.strategise() packer.strategise()
packer.execute() packer.execute()
# We should have none of the *.bphys files now. # We should have none of the *.bphys files now.
count = len(list((self.tpath / 'T55539-particles/blendcache_particle').glob('*.bphys'))) count = len(
list((self.tpath / "T55539-particles/blendcache_particle").glob("*.bphys"))
)
self.assertEqual(0, count) self.assertEqual(0, count)
# Just particle.blend + pack_info.txt # Just particle.blend + pack_info.txt
self.assertGreaterEqual(cb.transfer_progress.call_count, 2, self.assertGreaterEqual(
'transfer_progress() should be called at least once per asset') cb.transfer_progress.call_count,
2,
"transfer_progress() should be called at least once per asset",
)
def test_smoke_cache(self): def test_smoke_cache(self):
# The smoke cache uses a glob to indicate which files to pack. # The smoke cache uses a glob to indicate which files to pack.
cb = mock.Mock(progress.Callback) cb = mock.Mock(progress.Callback)
infile = self.blendfiles / 'T55542-smoke/smoke_cache.blend' infile = self.blendfiles / "T55542-smoke/smoke_cache.blend"
with pack.Packer(infile, self.blendfiles, self.tpath) as packer: with pack.Packer(infile, self.blendfiles, self.tpath) as packer:
packer.progress_cb = cb packer.progress_cb = cb
packer.strategise() packer.strategise()
packer.execute() packer.execute()
# We should have all the *.bphys files now. # We should have all the *.bphys files now.
count = len(list((self.tpath / 'T55542-smoke/blendcache_smoke_cache').glob('*.bphys'))) count = len(
list((self.tpath / "T55542-smoke/blendcache_smoke_cache").glob("*.bphys"))
)
self.assertEqual(10, count) self.assertEqual(10, count)
# Physics files + smoke_cache.blend + pack_info.txt # Physics files + smoke_cache.blend + pack_info.txt
self.assertGreaterEqual(cb.transfer_progress.call_count, 12, self.assertGreaterEqual(
'transfer_progress() should be called at least once per asset') cb.transfer_progress.call_count,
12,
"transfer_progress() should be called at least once per asset",
)
class AbortTest(AbstractPackTest): class AbortTest(AbstractPackTest):
def test_abort_strategise(self): def test_abort_strategise(self):
infile = self.blendfiles / 'subdir/doubly_linked_up.blend' infile = self.blendfiles / "subdir/doubly_linked_up.blend"
packer = pack.Packer(infile, self.blendfiles, self.tpath) packer = pack.Packer(infile, self.blendfiles, self.tpath)
class AbortingCallback(progress.Callback): class AbortingCallback(progress.Callback):
def trace_blendfile(self, filename: Path): def trace_blendfile(self, filename: Path):
# Call abort() somewhere during the strategise() call. # Call abort() somewhere during the strategise() call.
if filename.name == 'linked_cube.blend': if filename.name == "linked_cube.blend":
packer.abort() packer.abort()
packer.progress_cb = AbortingCallback() packer.progress_cb = AbortingCallback()
@ -584,7 +655,7 @@ class AbortTest(AbstractPackTest):
packer.strategise() packer.strategise()
def test_abort_transfer(self): def test_abort_transfer(self):
infile = self.blendfiles / 'subdir/doubly_linked_up.blend' infile = self.blendfiles / "subdir/doubly_linked_up.blend"
packer = pack.Packer(infile, self.blendfiles, self.tpath) packer = pack.Packer(infile, self.blendfiles, self.tpath)
first_file_size = infile.stat().st_size first_file_size = infile.stat().st_size

20
tests/test_pack_progress.py
View File

@ -21,12 +21,12 @@ class ThreadedProgressTest(unittest.TestCase):
def thread(): def thread():
tscb.pack_start() tscb.pack_start()
tscb.pack_done(Path('one'), {Path('two'), Path('three')}) tscb.pack_done(Path("one"), {Path("two"), Path("three")})
tscb.trace_asset(Path('four')) tscb.trace_asset(Path("four"))
tscb.transfer_file(Path('five'), Path('six')) tscb.transfer_file(Path("five"), Path("six"))
tscb.transfer_file_skipped(Path('seven'), Path('eight')) tscb.transfer_file_skipped(Path("seven"), Path("eight"))
tscb.transfer_progress(327, 47) tscb.transfer_progress(327, 47)
tscb.missing_file(Path('nine')) tscb.missing_file(Path("nine"))
t = threading.Thread(target=thread) t = threading.Thread(target=thread)
t.start() t.start()
@ -34,9 +34,9 @@ class ThreadedProgressTest(unittest.TestCase):
tscb.flush(timeout=3) tscb.flush(timeout=3)
cb.pack_start.assert_called_with() cb.pack_start.assert_called_with()
cb.pack_done.assert_called_with(Path('one'), {Path('two'), Path('three')}) cb.pack_done.assert_called_with(Path("one"), {Path("two"), Path("three")})
cb.trace_asset.assert_called_with(Path('four')) cb.trace_asset.assert_called_with(Path("four"))
cb.transfer_file.assert_called_with(Path('five'), Path('six')) cb.transfer_file.assert_called_with(Path("five"), Path("six"))
cb.transfer_file_skipped.assert_called_with(Path('seven'), Path('eight')) cb.transfer_file_skipped.assert_called_with(Path("seven"), Path("eight"))
cb.transfer_progress.assert_called_with(327, 47) cb.transfer_progress.assert_called_with(327, 47)
cb.missing_file.assert_called_with(Path('nine')) cb.missing_file.assert_called_with(Path("nine"))

29
tests/test_pack_shaman.py
View File

@ -30,23 +30,32 @@ httpmock = responses.RequestsMock()
class ShamanPackTest(AbstractPackTest): class ShamanPackTest(AbstractPackTest):
@httpmock.activate @httpmock.activate
def test_all_files_already_uploaded(self): def test_all_files_already_uploaded(self):
infile = self.blendfiles / 'basic_file_ñønæščii.blend' infile = self.blendfiles / "basic_file_ñønæščii.blend"
packer = shaman.ShamanPacker(infile, infile.parent, '/', packer = shaman.ShamanPacker(
endpoint='http://shaman.local', infile,
checkout_id='DA-JOBBY-ID') infile.parent,
"/",
endpoint="http://shaman.local",
checkout_id="DA-JOBBY-ID",
)
# Temporary hack # Temporary hack
httpmock.add('GET', 'http://shaman.local/get-token', body='AUTH-TOKEN') httpmock.add("GET", "http://shaman.local/get-token", body="AUTH-TOKEN")
# Just fake that everything is already available on the server. # Just fake that everything is already available on the server.
httpmock.add('POST', 'http://shaman.local/checkout/requirements', body='') httpmock.add("POST", "http://shaman.local/checkout/requirements", body="")
httpmock.add('POST', 'http://shaman.local/checkout/create/DA-JOBBY-ID', httpmock.add(
body='DA/-JOBBY-ID') "POST",
"http://shaman.local/checkout/create/DA-JOBBY-ID",
body="DA/-JOBBY-ID",
)
with packer: with packer:
packer.strategise() packer.strategise()
packer.execute() packer.execute()
self.assertEqual(pathlib.PurePosixPath('DA/-JOBBY-ID/basic_file_ñønæščii.blend'), self.assertEqual(
packer.output_path) pathlib.PurePosixPath("DA/-JOBBY-ID/basic_file_ñønæščii.blend"),
packer.output_path,
)

8
tests/test_pack_zipped.py
View File

@ -25,8 +25,8 @@ from blender_asset_tracer.pack import zipped
class ZippedPackTest(AbstractPackTest): class ZippedPackTest(AbstractPackTest):
def test_basic_file(self): def test_basic_file(self):
infile = self.blendfiles / 'basic_file_ñønæščii.blend' infile = self.blendfiles / "basic_file_ñønæščii.blend"
zippath = self.tpath / 'target.zip' zippath = self.tpath / "target.zip"
with zipped.ZipPacker(infile, infile.parent, zippath) as packer: with zipped.ZipPacker(infile, infile.parent, zippath) as packer:
packer.strategise() packer.strategise()
packer.execute() packer.execute()
@ -34,4 +34,6 @@ class ZippedPackTest(AbstractPackTest):
self.assertTrue(zippath.exists()) self.assertTrue(zippath.exists())
with zipfile.ZipFile(str(zippath)) as inzip: with zipfile.ZipFile(str(zippath)) as inzip:
inzip.testzip() inzip.testzip()
self.assertEqual({'pack-info.txt', 'basic_file_ñønæščii.blend'}, set(inzip.namelist())) self.assertEqual(
{"pack-info.txt", "basic_file_ñønæščii.blend"}, set(inzip.namelist())
)

32
tests/test_shaman.py
View File

@ -6,40 +6,40 @@ from blender_asset_tracer.pack import shaman
class ParseEndpointTest(unittest.TestCase): class ParseEndpointTest(unittest.TestCase):
def test_path_slashyness(self): def test_path_slashyness(self):
self.assertEqual( self.assertEqual(
('https://endpoint/', '123'), ("https://endpoint/", "123"),
shaman.parse_endpoint('shaman://endpoint#123'), shaman.parse_endpoint("shaman://endpoint#123"),
) )
self.assertEqual( self.assertEqual(
('https://endpoint/', '123'), ("https://endpoint/", "123"),
shaman.parse_endpoint('shaman://endpoint/#123'), shaman.parse_endpoint("shaman://endpoint/#123"),
) )
self.assertEqual( self.assertEqual(
('https://endpoint/root', '123'), ("https://endpoint/root", "123"),
shaman.parse_endpoint('shaman://endpoint/root#123'), shaman.parse_endpoint("shaman://endpoint/root#123"),
) )
self.assertEqual( self.assertEqual(
('https://endpoint/root/is/longer/', '123'), ("https://endpoint/root/is/longer/", "123"),
shaman.parse_endpoint('shaman://endpoint/root/is/longer/#123'), shaman.parse_endpoint("shaman://endpoint/root/is/longer/#123"),
) )
def test_schemes_with_plus(self): def test_schemes_with_plus(self):
self.assertEqual( self.assertEqual(
('https://endpoint/', '123'), ("https://endpoint/", "123"),
shaman.parse_endpoint('shaman+https://endpoint/#123'), shaman.parse_endpoint("shaman+https://endpoint/#123"),
) )
self.assertEqual( self.assertEqual(
('http://endpoint/', '123'), ("http://endpoint/", "123"),
shaman.parse_endpoint('shaman+http://endpoint/#123'), shaman.parse_endpoint("shaman+http://endpoint/#123"),
) )
def test_checkout_ids(self): def test_checkout_ids(self):
self.assertEqual( self.assertEqual(
('https://endpoint/', ''), ("https://endpoint/", ""),
shaman.parse_endpoint('shaman+https://endpoint/'), shaman.parse_endpoint("shaman+https://endpoint/"),
) )
# Not a valid ID, but the parser should handle it gracefully anyway # Not a valid ID, but the parser should handle it gracefully anyway
self.assertEqual( self.assertEqual(
('http://endpoint/', 'ïđ'), ("http://endpoint/", "ïđ"),
shaman.parse_endpoint('shaman+http://endpoint/#%C3%AF%C4%91'), shaman.parse_endpoint("shaman+http://endpoint/#%C3%AF%C4%91"),
) )

42
tests/test_shaman_cache.py
View File

@ -29,8 +29,10 @@ class AbstractChecksumTest(AbstractBlendFileTest):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
super().setUpClass() super().setUpClass()
cls.test_file = cls.blendfiles / 'linked_cube_compressed.blend' cls.test_file = cls.blendfiles / "linked_cube_compressed.blend"
cls.expected_checksum = '3c525e3a01ece11f26ded1e05e43284c4cce575c8074b97c6bdbc414fa2802ab' cls.expected_checksum = (
"3c525e3a01ece11f26ded1e05e43284c4cce575c8074b97c6bdbc414fa2802ab"
)
class ChecksumTest(AbstractChecksumTest): class ChecksumTest(AbstractChecksumTest):
@ -39,49 +41,49 @@ class ChecksumTest(AbstractChecksumTest):
class CachedChecksumTest(AbstractChecksumTest): class CachedChecksumTest(AbstractChecksumTest):
@mock.patch('blender_asset_tracer.pack.shaman.cache._cache_path') @mock.patch("blender_asset_tracer.pack.shaman.cache._cache_path")
@mock.patch('blender_asset_tracer.pack.shaman.cache.compute_checksum') @mock.patch("blender_asset_tracer.pack.shaman.cache.compute_checksum")
def test_cache_invalid_json(self, mock_compute_checksum, mock_cache_path): def test_cache_invalid_json(self, mock_compute_checksum, mock_cache_path):
mock_path = mock.MagicMock(spec=pathlib.Path) mock_path = mock.MagicMock(spec=pathlib.Path)
mock_path.open().__enter__().read.return_value = 'je moeder' mock_path.open().__enter__().read.return_value = "je moeder"
mock_cache_path.return_value = mock_path mock_cache_path.return_value = mock_path
mock_compute_checksum.return_value = 'computed-checksum' mock_compute_checksum.return_value = "computed-checksum"
checksum = cache.compute_cached_checksum(self.test_file) checksum = cache.compute_cached_checksum(self.test_file)
self.assertEqual('computed-checksum', checksum) self.assertEqual("computed-checksum", checksum)
@mock.patch('blender_asset_tracer.pack.shaman.cache._cache_path') @mock.patch("blender_asset_tracer.pack.shaman.cache._cache_path")
@mock.patch('blender_asset_tracer.pack.shaman.cache.compute_checksum') @mock.patch("blender_asset_tracer.pack.shaman.cache.compute_checksum")
def test_cache_valid_json(self, mock_compute_checksum, mock_cache_path): def test_cache_valid_json(self, mock_compute_checksum, mock_cache_path):
stat = self.test_file.stat() stat = self.test_file.stat()
cache_info = { cache_info = {
'checksum': 'cached-checksum', "checksum": "cached-checksum",
'file_mtime': stat.st_mtime + 0.0001, # mimick a slight clock skew "file_mtime": stat.st_mtime + 0.0001, # mimick a slight clock skew
'file_size': stat.st_size, "file_size": stat.st_size,
} }
mock_path = mock.MagicMock(spec=pathlib.Path) mock_path = mock.MagicMock(spec=pathlib.Path)
mock_path.open().__enter__().read.return_value = json.dumps(cache_info) mock_path.open().__enter__().read.return_value = json.dumps(cache_info)
mock_cache_path.return_value = mock_path mock_cache_path.return_value = mock_path
mock_compute_checksum.return_value = 'computed-checksum' mock_compute_checksum.return_value = "computed-checksum"
checksum = cache.compute_cached_checksum(self.test_file) checksum = cache.compute_cached_checksum(self.test_file)
self.assertEqual('cached-checksum', checksum) self.assertEqual("cached-checksum", checksum)
@mock.patch('blender_asset_tracer.pack.shaman.cache._cache_path') @mock.patch("blender_asset_tracer.pack.shaman.cache._cache_path")
@mock.patch('blender_asset_tracer.pack.shaman.cache.compute_checksum') @mock.patch("blender_asset_tracer.pack.shaman.cache.compute_checksum")
def test_cache_not_exists(self, mock_compute_checksum, mock_cache_path): def test_cache_not_exists(self, mock_compute_checksum, mock_cache_path):
mock_path = mock.MagicMock(spec=pathlib.Path) mock_path = mock.MagicMock(spec=pathlib.Path)
mock_path.open.side_effect = [ mock_path.open.side_effect = [
FileNotFoundError('Testing absent cache file'), FileNotFoundError("Testing absent cache file"),
FileExistsError('Testing I/O error when writing'), FileExistsError("Testing I/O error when writing"),
] ]
mock_cache_path.return_value = mock_path mock_cache_path.return_value = mock_path
mock_compute_checksum.return_value = 'computed-checksum' mock_compute_checksum.return_value = "computed-checksum"
# This should not raise the FileExistsError # This should not raise the FileExistsError
checksum = cache.compute_cached_checksum(self.test_file) checksum = cache.compute_cached_checksum(self.test_file)
self.assertEqual('computed-checksum', checksum) self.assertEqual("computed-checksum", checksum)

8
tests/test_shaman_time_tracker.py
View File

@ -5,25 +5,25 @@ from blender_asset_tracer.pack.shaman import time_tracker
class TimeTrackerTest(unittest.TestCase): class TimeTrackerTest(unittest.TestCase):
@mock.patch('time.monotonic') @mock.patch("time.monotonic")
def test_empty_class(self, mock_monotonic): def test_empty_class(self, mock_monotonic):
class TestClass: class TestClass:
pass pass
mock_monotonic.side_effect = [1.25, 4.75] mock_monotonic.side_effect = [1.25, 4.75]
with time_tracker.track_time(TestClass, 'some_attr'): with time_tracker.track_time(TestClass, "some_attr"):
pass pass
# noinspection PyUnresolvedReferences # noinspection PyUnresolvedReferences
self.assertEqual(3.5, TestClass.some_attr) self.assertEqual(3.5, TestClass.some_attr)
@mock.patch('time.monotonic') @mock.patch("time.monotonic")
def test_with_value(self, mock_monotonic): def test_with_value(self, mock_monotonic):
class TestClass: class TestClass:
some_attr = 4.125 some_attr = 4.125
mock_monotonic.side_effect = [1.25, 4.75] mock_monotonic.side_effect = [1.25, 4.75]
with time_tracker.track_time(TestClass, 'some_attr'): with time_tracker.track_time(TestClass, "some_attr"):
pass pass
self.assertEqual(3.5 + 4.125, TestClass.some_attr) self.assertEqual(3.5 + 4.125, TestClass.some_attr)

51
tests/test_shaman_transfer.py
View File

@ -32,19 +32,19 @@ class ShamanTransferTest(AbstractBlendFileTest):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
super().setUpClass() super().setUpClass()
cls.test_file1 = cls.blendfiles / 'linked_cube_compressed.blend' cls.test_file1 = cls.blendfiles / "linked_cube_compressed.blend"
cls.test_file2 = cls.blendfiles / 'basic_file.blend' cls.test_file2 = cls.blendfiles / "basic_file.blend"
cls.expected_checksums = { cls.expected_checksums = {
cls.test_file1: '3c525e3a01ece11f26ded1e05e43284c4cce575c8074b97c6bdbc414fa2802ab', cls.test_file1: "3c525e3a01ece11f26ded1e05e43284c4cce575c8074b97c6bdbc414fa2802ab",
cls.test_file2: 'd5283988d95f259069d4cd3c25a40526090534b8d188577b6c6fb36c3d481454', cls.test_file2: "d5283988d95f259069d4cd3c25a40526090534b8d188577b6c6fb36c3d481454",
} }
cls.file_sizes = { cls.file_sizes = {
cls.test_file1: cls.test_file1.stat().st_size, cls.test_file1: cls.test_file1.stat().st_size,
cls.test_file2: cls.test_file2.stat().st_size, cls.test_file2: cls.test_file2.stat().st_size,
} }
cls.packed_names = { cls.packed_names = {
cls.test_file1: pathlib.PurePosixPath('path/in/pack/test1.blend'), cls.test_file1: pathlib.PurePosixPath("path/in/pack/test1.blend"),
cls.test_file2: pathlib.PurePosixPath('path/in/pack/test2.blend'), cls.test_file2: pathlib.PurePosixPath("path/in/pack/test2.blend"),
} }
def assertValidCheckoutDef(self, definition_file: bytes): def assertValidCheckoutDef(self, definition_file: bytes):
@ -54,8 +54,8 @@ class ShamanTransferTest(AbstractBlendFileTest):
checksum = self.expected_checksums[filepath] checksum = self.expected_checksums[filepath]
fsize = self.file_sizes[filepath] fsize = self.file_sizes[filepath]
relpath = str(self.packed_names[filepath]) relpath = str(self.packed_names[filepath])
expect_lines.add(b'%s %d %s' % (checksum.encode(), fsize, relpath.encode())) expect_lines.add(b"%s %d %s" % (checksum.encode(), fsize, relpath.encode()))
self.assertEqual(expect_lines, set(definition_file.split(b'\n'))) self.assertEqual(expect_lines, set(definition_file.split(b"\n")))
@httpmock.activate @httpmock.activate
def test_checkout_happy(self): def test_checkout_happy(self):
@ -63,26 +63,35 @@ class ShamanTransferTest(AbstractBlendFileTest):
fsize1 = self.file_sizes[self.test_file1] fsize1 = self.file_sizes[self.test_file1]
def mock_requirements(request): def mock_requirements(request):
self.assertEqual('text/plain', request.headers['Content-Type']) self.assertEqual("text/plain", request.headers["Content-Type"])
self.assertValidCheckoutDef(request.body) self.assertValidCheckoutDef(request.body)
body = 'file-unknown path/in/pack/test1.blend\n' body = "file-unknown path/in/pack/test1.blend\n"
return 200, {'Content-Type': 'text/plain'}, body return 200, {"Content-Type": "text/plain"}, body
def mock_checkout_create(request): def mock_checkout_create(request):
self.assertEqual('text/plain', request.headers['Content-Type']) self.assertEqual("text/plain", request.headers["Content-Type"])
self.assertValidCheckoutDef(request.body) self.assertValidCheckoutDef(request.body)
return 200, {'Content-Type': 'text/plain'}, 'DA/-JOB-ID' return 200, {"Content-Type": "text/plain"}, "DA/-JOB-ID"
httpmock.add_callback('POST', 'http://unittest.local:1234/checkout/requirements', httpmock.add_callback(
callback=mock_requirements) "POST",
"http://unittest.local:1234/checkout/requirements",
callback=mock_requirements,
)
httpmock.add('POST', 'http://unittest.local:1234/files/%s/%d' % (checksum1, fsize1)) httpmock.add(
httpmock.add_callback('POST', 'http://unittest.local:1234/checkout/create/DA-JOB-ID', "POST", "http://unittest.local:1234/files/%s/%d" % (checksum1, fsize1)
callback=mock_checkout_create) )
httpmock.add_callback(
"POST",
"http://unittest.local:1234/checkout/create/DA-JOB-ID",
callback=mock_checkout_create,
)
trans = transfer.ShamanTransferrer('auth-token', self.blendfiles, trans = transfer.ShamanTransferrer(
'http://unittest.local:1234/', 'DA-JOB-ID') "auth-token", self.blendfiles, "http://unittest.local:1234/", "DA-JOB-ID"
)
trans.start() trans.start()
trans.queue_copy(self.test_file1, self.packed_names[self.test_file1]) trans.queue_copy(self.test_file1, self.packed_names[self.test_file1])
@ -90,4 +99,4 @@ class ShamanTransferTest(AbstractBlendFileTest):
trans.done_and_join() trans.done_and_join()
self.assertFalse(trans.has_error, trans.error_message()) self.assertFalse(trans.has_error, trans.error_message())
self.assertEqual('DA/-JOB-ID', trans.checkout_location) self.assertEqual("DA/-JOB-ID", trans.checkout_location)

442
tests/test_tracer.py
View File

@ -9,20 +9,20 @@ from tests.abstract_test import AbstractBlendFileTest
# Mimicks a BlockUsage, but without having to set the block to an expected value. # Mimicks a BlockUsage, but without having to set the block to an expected value.
Expect = collections.namedtuple( Expect = collections.namedtuple(
'Expect', "Expect", "type full_field dirname_field basename_field asset_path is_sequence"
'type full_field dirname_field basename_field asset_path is_sequence') )
class AbstractTracerTest(AbstractBlendFileTest): class AbstractTracerTest(AbstractBlendFileTest):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
super().setUpClass() super().setUpClass()
logging.getLogger('blender_asset_tracer.tracer').setLevel(logging.DEBUG) logging.getLogger("blender_asset_tracer.tracer").setLevel(logging.DEBUG)
class AssetHoldingBlocksTest(AbstractTracerTest): class AssetHoldingBlocksTest(AbstractTracerTest):
def setUp(self): def setUp(self):
self.bf = blendfile.BlendFile(self.blendfiles / 'basic_file.blend') self.bf = blendfile.BlendFile(self.blendfiles / "basic_file.blend")
def test_simple_file(self): def test_simple_file(self):
# This file should not depend on external assets. # This file should not depend on external assets.
@ -38,17 +38,17 @@ class AssetHoldingBlocksTest(AbstractTracerTest):
self.assertEqual(2, len(block.code)) self.assertEqual(2, len(block.code))
# World blocks should not yielded either. # World blocks should not yielded either.
self.assertNotEqual(b'WO', block.code) self.assertNotEqual(b"WO", block.code)
# Do some arbitrary tests that convince us stuff is read well. # Do some arbitrary tests that convince us stuff is read well.
if block.code == b'SC': if block.code == b"SC":
seen_scene = True seen_scene = True
self.assertEqual(b'SCScene', block.id_name) self.assertEqual(b"SCScene", block.id_name)
continue continue
if block.code == b'OB': if block.code == b"OB":
seen_ob = True seen_ob = True
self.assertEqual('OBümlaut', block.get((b'id', b'name'), as_str=True)) self.assertEqual("OBümlaut", block.get((b"id", b"name"), as_str=True))
continue continue
self.assertTrue(seen_scene) self.assertTrue(seen_scene)
@ -77,95 +77,150 @@ class DepsTest(AbstractTracerTest):
actual_dirname = self.field_name(dep.path_dir_field) actual_dirname = self.field_name(dep.path_dir_field)
actual_basename = self.field_name(dep.path_base_field) actual_basename = self.field_name(dep.path_base_field)
actual = Expect(actual_type, actual_full_field, actual_dirname, actual_basename, actual = Expect(
dep.asset_path, dep.is_sequence) actual_type,
actual_full_field,
actual_dirname,
actual_basename,
dep.asset_path,
dep.is_sequence,
)
exp = expects.get(dep.block_name, None) exp = expects.get(dep.block_name, None)
if isinstance(exp, (set, list)): if isinstance(exp, (set, list)):
self.assertIn(actual, exp, msg='for block %s' % dep.block_name) self.assertIn(actual, exp, msg="for block %s" % dep.block_name)
exp.remove(actual) exp.remove(actual)
if not exp: if not exp:
# Don't leave empty sets in expects. # Don't leave empty sets in expects.
del expects[dep.block_name] del expects[dep.block_name]
elif exp is None: elif exp is None:
self.assertIsNone(actual, msg='unexpected dependency of block %s' % dep.block_name) self.assertIsNone(
actual, msg="unexpected dependency of block %s" % dep.block_name
)
del expects[dep.block_name] del expects[dep.block_name]
else: else:
self.assertEqual(exp, actual, msg='for block %s' % dep.block_name) self.assertEqual(exp, actual, msg="for block %s" % dep.block_name)
del expects[dep.block_name] del expects[dep.block_name]
# All expected uses should have been seen. # All expected uses should have been seen.
self.assertEqual(expects, {}, 'Expected results were not seen.') self.assertEqual(expects, {}, "Expected results were not seen.")
def test_no_deps(self): def test_no_deps(self):
self.assert_deps('basic_file.blend', {}) self.assert_deps("basic_file.blend", {})
def test_ob_mat_texture(self): def test_ob_mat_texture(self):
expects = { expects = {
b'IMbrick_dotted_04-bump': Expect( b"IMbrick_dotted_04-bump": Expect(
'Image', 'name[1024]', None, None, "Image",
b'//textures/Bricks/brick_dotted_04-bump.jpg', False), "name[1024]",
b'IMbrick_dotted_04-color': Expect( None,
'Image', 'name[1024]', None, None, None,
b'//textures/Bricks/brick_dotted_04-color.jpg', False), b"//textures/Bricks/brick_dotted_04-bump.jpg",
False,
),
b"IMbrick_dotted_04-color": Expect(
"Image",
"name[1024]",
None,
None,
b"//textures/Bricks/brick_dotted_04-color.jpg",
False,
),
# This data block is in there, but the image is packed, so it # This data block is in there, but the image is packed, so it
# shouldn't be in the results. # shouldn't be in the results.
# b'IMbrick_dotted_04-specular': Expect( # b'IMbrick_dotted_04-specular': Expect(
# 'Image', 'name[1024]', None, None, # 'Image', 'name[1024]', None, None,
# b'//textures/Bricks/brick_dotted_04-specular.jpg', False), # b'//textures/Bricks/brick_dotted_04-specular.jpg', False),
b'IMbuildings_roof_04-color': Expect( b"IMbuildings_roof_04-color": Expect(
'Image', 'name[1024]', None, None, "Image",
b'//textures/Textures/Buildings/buildings_roof_04-color.png', False), "name[1024]",
None,
None,
b"//textures/Textures/Buildings/buildings_roof_04-color.png",
False,
),
} }
self.assert_deps('material_textures.blend', expects) self.assert_deps("material_textures.blend", expects)
def test_seq_image_sequence(self): def test_seq_image_sequence(self):
expects = { expects = {
b'-unnamed-': [ b"-unnamed-": [
Expect('Strip', None, 'dir[768]', 'name[256]', b'//imgseq/000210.png', True), Expect(
"Strip", None, "dir[768]", "name[256]", b"//imgseq/000210.png", True
),
# Video strip reference. # Video strip reference.
Expect('Strip', None, 'dir[768]', 'name[256]', Expect(
b'//../../../../cloud/pillar/testfiles/video-tiny.mkv', False), "Strip",
None,
"dir[768]",
"name[256]",
b"//../../../../cloud/pillar/testfiles/video-tiny.mkv",
False,
),
# The sound will be referenced twice, from the sequence strip and an SO data block. # The sound will be referenced twice, from the sequence strip and an SO data block.
Expect('Strip', None, 'dir[768]', 'name[256]', Expect(
b'//../../../../cloud/pillar/testfiles/video-tiny.mkv', False), "Strip",
None,
"dir[768]",
"name[256]",
b"//../../../../cloud/pillar/testfiles/video-tiny.mkv",
False,
),
], ],
b'SOvideo-tiny.mkv': Expect( b"SOvideo-tiny.mkv": Expect(
'bSound', 'name[1024]', None, None, "bSound",
b'//../../../../cloud/pillar/testfiles/video-tiny.mkv', False), "name[1024]",
None,
None,
b"//../../../../cloud/pillar/testfiles/video-tiny.mkv",
False,
),
} }
self.assert_deps('image_sequencer.blend', expects) self.assert_deps("image_sequencer.blend", expects)
# Test the filename expansion. # Test the filename expansion.
expected = [self.blendfiles / ('imgseq/%06d.png' % num) expected = [
for num in range(210, 215)] self.blendfiles / ("imgseq/%06d.png" % num) for num in range(210, 215)
for dep in trace.deps(self.blendfiles / 'image_sequencer.blend'): ]
if dep.block_name != b'SQ000210.png': for dep in trace.deps(self.blendfiles / "image_sequencer.blend"):
if dep.block_name != b"SQ000210.png":
continue continue
actual = list(dep.files()) actual = list(dep.files())
self.assertEqual(actual, expected) self.assertEqual(actual, expected)
def test_block_cf(self): def test_block_cf(self):
self.assert_deps('alembic-user.blend', { self.assert_deps(
b'CFclothsim.abc': Expect('CacheFile', 'filepath[1024]', None, None, "alembic-user.blend",
b'//clothsim.abc', False), {
}) b"CFclothsim.abc": Expect(
"CacheFile", "filepath[1024]", None, None, b"//clothsim.abc", False
),
},
)
def test_alembic_sequence(self): def test_alembic_sequence(self):
self.assert_deps('alembic-sequence-user.blend', { self.assert_deps(
b'CFclothsim_alembic': "alembic-sequence-user.blend",
Expect('CacheFile', 'filepath[1024]', None, None, b'//clothsim.030.abc', True), {
}) b"CFclothsim_alembic": Expect(
"CacheFile",
"filepath[1024]",
None,
None,
b"//clothsim.030.abc",
True,
),
},
)
# Test the filename expansion. # Test the filename expansion.
expected = [self.blendfiles / ('clothsim.%03d.abc' % num) expected = [
for num in range(30, 36)] self.blendfiles / ("clothsim.%03d.abc" % num) for num in range(30, 36)
]
performed_test = False performed_test = False
for dep in trace.deps(self.blendfiles / 'alembic-sequence-user.blend'): for dep in trace.deps(self.blendfiles / "alembic-sequence-user.blend"):
if dep.block_name != b'CFclothsim_alembic': if dep.block_name != b"CFclothsim_alembic":
continue continue
actual = list(dep.files()) actual = list(dep.files())
@ -174,114 +229,241 @@ class DepsTest(AbstractTracerTest):
self.assertTrue(performed_test) self.assertTrue(performed_test)
def test_block_mc(self): def test_block_mc(self):
self.assert_deps('movieclip.blend', { self.assert_deps(
b'MCvideo.mov': Expect('MovieClip', 'name[1024]', None, None, "movieclip.blend",
b'//../../../../cloud/pillar/testfiles/video.mov', False), {
}) b"MCvideo.mov": Expect(
"MovieClip",
"name[1024]",
None,
None,
b"//../../../../cloud/pillar/testfiles/video.mov",
False,
),
},
)
def test_block_me(self): def test_block_me(self):
self.assert_deps('multires_external.blend', { self.assert_deps(
b'MECube': Expect('Mesh', 'filename[1024]', None, None, b'//Cube.btx', False), "multires_external.blend",
}) {
b"MECube": Expect(
"Mesh", "filename[1024]", None, None, b"//Cube.btx", False
),
},
)
def test_ocean(self): def test_ocean(self):
self.assert_deps('ocean_modifier.blend', { self.assert_deps(
b'OBPlane.modifiers[0]': Expect('OceanModifierData', 'cachepath[1024]', None, None, "ocean_modifier.blend",
b'//cache_ocean', True), {
}) b"OBPlane.modifiers[0]": Expect(
"OceanModifierData",
"cachepath[1024]",
None,
None,
b"//cache_ocean",
True,
),
},
)
def test_particles(self): def test_particles(self):
# This file has an empty name for the cache, which should result in some hex magic # This file has an empty name for the cache, which should result in some hex magic
# to create a name. See ptcache_filename() in pointcache.c. # to create a name. See ptcache_filename() in pointcache.c.
self.assert_deps('T55539-particles/particle.blend', { self.assert_deps(
b'OBCube.modifiers[0]': Expect('PointCache', 'name[64]', None, None, "T55539-particles/particle.blend",
b'//blendcache_particle/43756265_*.bphys', True), {
}) b"OBCube.modifiers[0]": Expect(
"PointCache",
"name[64]",
None,
None,
b"//blendcache_particle/43756265_*.bphys",
True,
),
},
)
def test_smoke(self): def test_smoke(self):
# This file has an empty name for the cache, which should result in some hex magic # This file has an empty name for the cache, which should result in some hex magic
# to create a name. See ptcache_filename() in pointcache.c. # to create a name. See ptcache_filename() in pointcache.c.
self.assert_deps('T55542-smoke/smoke_cache.blend', { self.assert_deps(
b'OBSmoke Domain.modifiers[0]': Expect('PointCache', 'name[64]', None, None, "T55542-smoke/smoke_cache.blend",
b'//blendcache_smoke_cache/536D6F6B6520446F6D61696E_*.bphys', True), {
}) b"OBSmoke Domain.modifiers[0]": Expect(
self.assert_deps('T55542-smoke/smoke_cache_vdb.blend', { "PointCache",
b'OBSmoke Domain.modifiers[0]': Expect('PointCache', 'name[64]', None, None, "name[64]",
b'//blendcache_smoke_cache_vdb/536D6F6B6520446F6D61696E_*.vdb', True), None,
}) None,
b"//blendcache_smoke_cache/536D6F6B6520446F6D61696E_*.bphys",
True,
),
},
)
self.assert_deps(
"T55542-smoke/smoke_cache_vdb.blend",
{
b"OBSmoke Domain.modifiers[0]": Expect(
"PointCache",
"name[64]",
None,
None,
b"//blendcache_smoke_cache_vdb/536D6F6B6520446F6D61696E_*.vdb",
True,
),
},
)
def test_mesh_cache(self): def test_mesh_cache(self):
self.assert_deps('meshcache-user.blend', { self.assert_deps(
b'OBPlane.modifiers[0]': Expect('MeshCacheModifierData', 'filepath[1024]', None, None, "meshcache-user.blend",
b'//meshcache.mdd', False), {
}) b"OBPlane.modifiers[0]": Expect(
"MeshCacheModifierData",
"filepath[1024]",
None,
None,
b"//meshcache.mdd",
False,
),
},
)
def test_block_vf(self): def test_block_vf(self):
self.assert_deps('with_font.blend', { self.assert_deps(
b'VFHack-Bold': Expect('VFont', 'name[1024]', None, None, "with_font.blend",
b'/usr/share/fonts/truetype/hack/Hack-Bold.ttf', False), {
}) b"VFHack-Bold": Expect(
"VFont",
"name[1024]",
None,
None,
b"/usr/share/fonts/truetype/hack/Hack-Bold.ttf",
False,
),
},
)
def test_block_li(self): def test_block_li(self):
self.assert_deps('linked_cube.blend', { self.assert_deps(
b'LILib': Expect('Library', 'name[1024]', None, None, b'//basic_file.blend', False), "linked_cube.blend",
}) {
b"LILib": Expect(
"Library", "name[1024]", None, None, b"//basic_file.blend", False
),
},
)
def test_deps_recursive(self): def test_deps_recursive(self):
self.assert_deps('doubly_linked.blend', { self.assert_deps(
b'LILib': { "doubly_linked.blend",
# From doubly_linked.blend {
Expect('Library', 'name[1024]', None, None, b'//linked_cube.blend', False), b"LILib": {
# From doubly_linked.blend
# From linked_cube.blend Expect(
Expect('Library', 'name[1024]', None, None, b'//basic_file.blend', False), "Library",
"name[1024]",
None,
None,
b"//linked_cube.blend",
False,
),
# From linked_cube.blend
Expect(
"Library",
"name[1024]",
None,
None,
b"//basic_file.blend",
False,
),
},
b"LILib.002": Expect(
"Library",
"name[1024]",
None,
None,
b"//material_textures.blend",
False,
),
# From material_texture.blend
b"IMbrick_dotted_04-bump": Expect(
"Image",
"name[1024]",
None,
None,
b"//textures/Bricks/brick_dotted_04-bump.jpg",
False,
),
b"IMbrick_dotted_04-color": Expect(
"Image",
"name[1024]",
None,
None,
b"//textures/Bricks/brick_dotted_04-color.jpg",
False,
),
# This data block is in the basic_file.blend file, but not used by
# any of the objects linked in from linked_cube.blend or
# doubly_linked.blend, hence it should *not* be reported:
# b'IMbuildings_roof_04-color': Expect(
# 'Image', 'name[1024]', None, None,
# b'//textures/Textures/Buildings/buildings_roof_04-color.png', False),
}, },
b'LILib.002': Expect('Library', 'name[1024]', None, None, )
b'//material_textures.blend', False),
# From material_texture.blend
b'IMbrick_dotted_04-bump': Expect(
'Image', 'name[1024]', None, None,
b'//textures/Bricks/brick_dotted_04-bump.jpg', False),
b'IMbrick_dotted_04-color': Expect(
'Image', 'name[1024]', None, None,
b'//textures/Bricks/brick_dotted_04-color.jpg', False),
# This data block is in the basic_file.blend file, but not used by
# any of the objects linked in from linked_cube.blend or
# doubly_linked.blend, hence it should *not* be reported:
# b'IMbuildings_roof_04-color': Expect(
# 'Image', 'name[1024]', None, None,
# b'//textures/Textures/Buildings/buildings_roof_04-color.png', False),
})
def test_geometry_nodes(self): def test_geometry_nodes(self):
self.assert_deps('geometry-nodes/file_to_pack.blend', { self.assert_deps(
b'LInode_lib.blend': Expect( "geometry-nodes/file_to_pack.blend",
type='Library', full_field='name[1024]', dirname_field=None, {
basename_field=None, asset_path=b'//node_lib.blend', is_sequence=False), b"LInode_lib.blend": Expect(
b'LIobject_lib.blend': Expect( type="Library",
type='Library', full_field='name[1024]', dirname_field=None, full_field="name[1024]",
basename_field=None, asset_path=b'//object_lib.blend', is_sequence=False), dirname_field=None,
}) basename_field=None,
asset_path=b"//node_lib.blend",
is_sequence=False,
),
b"LIobject_lib.blend": Expect(
type="Library",
full_field="name[1024]",
dirname_field=None,
basename_field=None,
asset_path=b"//object_lib.blend",
is_sequence=False,
),
},
)
def test_usage_abspath(self): def test_usage_abspath(self):
deps = [dep for dep in trace.deps(self.blendfiles / 'doubly_linked.blend') deps = [
if dep.asset_path == b'//material_textures.blend'] dep
for dep in trace.deps(self.blendfiles / "doubly_linked.blend")
if dep.asset_path == b"//material_textures.blend"
]
usage = deps[0] usage = deps[0]
expect = self.blendfiles / 'material_textures.blend' expect = self.blendfiles / "material_textures.blend"
self.assertEqual(expect, usage.abspath) self.assertEqual(expect, usage.abspath)
def test_sim_data(self): def test_sim_data(self):
self.assert_deps('T53562/bam_pack_bug.blend', { self.assert_deps(
b'OBEmitter.modifiers[0]': Expect( "T53562/bam_pack_bug.blend",
'PointCache', 'name[64]', None, None, {
b'//blendcache_bam_pack_bug/particles_*.bphys', True), b"OBEmitter.modifiers[0]": Expect(
}) "PointCache",
"name[64]",
None,
None,
b"//blendcache_bam_pack_bug/particles_*.bphys",
True,
),
},
)
def test_recursion_loop(self): def test_recursion_loop(self):
infinite_bfile = self.blendfiles / 'recursive_dependency_1.blend' infinite_bfile = self.blendfiles / "recursive_dependency_1.blend"
reclim = sys.getrecursionlimit() reclim = sys.getrecursionlimit()
try: try:

18
tests/test_tracer_file2blocks.py
View File

@ -6,7 +6,7 @@ from tests.test_tracer import AbstractTracerTest
class File2BlocksTest(AbstractTracerTest): class File2BlocksTest(AbstractTracerTest):
def test_id_blocks(self): def test_id_blocks(self):
self.bf = blendfile.BlendFile(self.blendfiles / 'doubly_linked.blend') self.bf = blendfile.BlendFile(self.blendfiles / "doubly_linked.blend")
foreign_blocks = {} foreign_blocks = {}
for block in file2blocks.iter_blocks(self.bf): for block in file2blocks.iter_blocks(self.bf):
@ -18,19 +18,19 @@ class File2BlocksTest(AbstractTracerTest):
self.assertNotEqual({}, foreign_blocks) self.assertNotEqual({}, foreign_blocks)
# It should find directly linked blocks (GRCubes and MABrick) as well # It should find directly linked blocks (GRCubes and MABrick) as well
# as indirectly linked (MECube³). # as indirectly linked (MECube³).
self.assertIn(b'GRCubes', foreign_blocks) self.assertIn(b"GRCubes", foreign_blocks)
self.assertIn(b'MABrick', foreign_blocks) self.assertIn(b"MABrick", foreign_blocks)
self.assertIn('MECube³'.encode(), foreign_blocks) self.assertIn("MECube³".encode(), foreign_blocks)
self.assertIn('OBümlaut'.encode(), foreign_blocks) self.assertIn("OBümlaut".encode(), foreign_blocks)
def test_circular_files(self): def test_circular_files(self):
self.bf = blendfile.BlendFile(self.blendfiles / 'recursive_dependency_1.blend') self.bf = blendfile.BlendFile(self.blendfiles / "recursive_dependency_1.blend")
blocks = {} blocks = {}
for block in file2blocks.iter_blocks(self.bf): for block in file2blocks.iter_blocks(self.bf):
blocks[block.id_name] = block blocks[block.id_name] = block
self.assertNotEqual({}, blocks) self.assertNotEqual({}, blocks)
self.assertIn(b'MAMaterial', blocks) self.assertIn(b"MAMaterial", blocks)
self.assertIn(b'OBCube', blocks) self.assertIn(b"OBCube", blocks)
self.assertIn(b'MECube', blocks) self.assertIn(b"MECube", blocks)

15
tests/test_tracer_file_sequence.py
View File

@ -6,28 +6,29 @@ from blender_asset_tracer.trace import file_sequence
class ExpandFileSequenceTest(AbstractBlendFileTest): class ExpandFileSequenceTest(AbstractBlendFileTest):
def setUp(self): def setUp(self):
super().setUp() super().setUp()
self.imgseq = [self.blendfiles / ('imgseq/%06d.png' % num) self.imgseq = [
for num in range(210, 215)] self.blendfiles / ("imgseq/%06d.png" % num) for num in range(210, 215)
]
def test_glob(self): def test_glob(self):
path = self.blendfiles / 'imgseq/*.png' path = self.blendfiles / "imgseq/*.png"
actual = list(file_sequence.expand_sequence(path)) actual = list(file_sequence.expand_sequence(path))
self.assertEqual(self.imgseq, actual) self.assertEqual(self.imgseq, actual)
def test_first_file(self): def test_first_file(self):
path = self.blendfiles / 'imgseq/000210.png' path = self.blendfiles / "imgseq/000210.png"
actual = list(file_sequence.expand_sequence(path)) actual = list(file_sequence.expand_sequence(path))
self.assertEqual(self.imgseq, actual) self.assertEqual(self.imgseq, actual)
def test_nonexistent(self): def test_nonexistent(self):
path = self.blendfiles / 'nonexistant' path = self.blendfiles / "nonexistant"
with self.assertRaises(file_sequence.DoesNotExist) as raises: with self.assertRaises(file_sequence.DoesNotExist) as raises:
for result in file_sequence.expand_sequence(path): for result in file_sequence.expand_sequence(path):
self.fail('unexpected result %r' % result) self.fail("unexpected result %r" % result)
self.assertEqual(path, raises.exception.path) self.assertEqual(path, raises.exception.path)
def test_non_sequence_file(self): def test_non_sequence_file(self):
path = self.blendfiles / 'imgseq/LICENSE.txt' path = self.blendfiles / "imgseq/LICENSE.txt"
actual = list(file_sequence.expand_sequence(path)) actual = list(file_sequence.expand_sequence(path))
self.assertEqual([path], actual) self.assertEqual([path], actual)

2
update_version.sh
View File

@ -8,7 +8,7 @@ fi
poetry version $1 poetry version $1
sed "s/version = '[^']*'/version = '$1'/" -i docs/conf.py sed "s/version = '[^']*'/version = '$1'/" -i docs/conf.py
sed "s/release = '[^']*'/release = '$1'/" -i docs/conf.py sed "s/release = '[^']*'/release = '$1'/" -i docs/conf.py
sed "s/__version__\s*=\s*'[^']*'/__version__ = '$1'/" -i blender_asset_tracer/__init__.py sed "s/__version__\s*=\s*\"[^']*\"/__version__ = \"$1\"/" -i blender_asset_tracer/__init__.py
git diff git diff
echo echo