autour_de_minuit/blender_asset_tracer
Archived
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update BAT module to 1.15 version

Browse Source
This commit is contained in:
pullusb
2023-01-10 11:41:55 +01:00
parent 85037c027a
commit fc32ebd453
34 changed files with 1878 additions and 1042 deletions
Download Patch File Download Diff File Expand all files Collapse all files
blendfile/__init__.py
+184 -131
View File
@@ -32,40 +32,39 @@ import shutil
import tempfile
import typing
from . import exceptions, dna_io, dna, header
from . import exceptions, dna, header, magic_compression
from blender_asset_tracer import bpathlib
log = logging.getLogger(__name__)
FILE_BUFFER_SIZE = 1024 * 1024
BLENDFILE_MAGIC = b'BLENDER'
GZIP_MAGIC = b'\x1f\x8b'
BFBList = typing.List['BlendFileBlock']
BFBList = typing.List["BlendFileBlock"]
_cached_bfiles = {} # type: typing.Dict[pathlib.Path, BlendFile]
def open_cached(path: pathlib.Path, mode='rb',
assert_cached: typing.Optional[bool] = None) -> 'BlendFile':
def open_cached(
path: pathlib.Path, mode="rb", assert_cached: typing.Optional[bool] = None
) -> "BlendFile":
"""Open a blend file, ensuring it is only opened once."""
my_log = log.getChild('open_cached')
my_log = log.getChild("open_cached")
bfile_path = bpathlib.make_absolute(path)
if assert_cached is not None:
is_cached = bfile_path in _cached_bfiles
if assert_cached and not is_cached:
raise AssertionError('File %s was not cached' % bfile_path)
raise AssertionError("File %s was not cached" % bfile_path)
elif not assert_cached and is_cached:
raise AssertionError('File %s was cached' % bfile_path)
raise AssertionError("File %s was cached" % bfile_path)
try:
bfile = _cached_bfiles[bfile_path]
except KeyError:
my_log.debug('Opening non-cached %s', path)
my_log.debug("Opening non-cached %s", path)
bfile = BlendFile(path, mode=mode)
_cached_bfiles[bfile_path] = bfile
else:
my_log.debug('Returning cached %s', path)
my_log.debug("Returning cached %s", path)
return bfile
@@ -76,13 +75,13 @@ def close_all_cached() -> None:
# Don't even log anything when there is nothing to close
return
log.debug('Closing %d cached blend files', len(_cached_bfiles))
log.debug("Closing %d cached blend files", len(_cached_bfiles))
for bfile in list(_cached_bfiles.values()):
bfile.close()
_cached_bfiles.clear()
def _cache(path: pathlib.Path, bfile: 'BlendFile'):
def _cache(path: pathlib.Path, bfile: "BlendFile"):
"""Add a BlendFile to the cache."""
bfile_path = bpathlib.make_absolute(path)
_cached_bfiles[bfile_path] = bfile
@@ -102,9 +101,16 @@ class BlendFile:
uncompressed files, but a temporary file for compressed files.
:ivar fileobj: the file object that's being accessed.
"""
log = log.getChild('BlendFile')
def __init__(self, path: pathlib.Path, mode='rb') -> None:
log = log.getChild("BlendFile")
strict_pointer_mode = True
"""Raise exceptions.SegmentationFault when dereferencing an unknown pointer.
Set to False to disable this exception, and to return None instead.
"""
def __init__(self, path: pathlib.Path, mode="rb") -> None:
"""Create a BlendFile instance for the blend file at the path.
Opens the file for reading or writing pending on the access. Compressed
@@ -121,7 +127,9 @@ class BlendFile:
self.blocks = [] # type: BFBList
"""BlendFileBlocks of this file, in disk order."""
self.code_index = collections.defaultdict(list) # type: typing.Dict[bytes, BFBList]
self.code_index = collections.defaultdict(
list
) # type: typing.Dict[bytes, BFBList]
self.structs = [] # type: typing.List[dna.Struct]
self.sdna_index_from_id = {} # type: typing.Dict[bytes, int]
self.block_from_addr = {} # type: typing.Dict[int, BlendFileBlock]
@@ -141,44 +149,13 @@ class BlendFile:
correct magic bytes.
"""
if 'b' not in mode:
raise ValueError('Only binary modes are supported, not %r' % mode)
decompressed = magic_compression.open(path, mode, FILE_BUFFER_SIZE)
self.filepath = path
self.is_compressed = decompressed.is_compressed
self.raw_filepath = decompressed.path
fileobj = path.open(mode, buffering=FILE_BUFFER_SIZE) # typing.IO[bytes]
fileobj.seek(0, os.SEEK_SET)
magic = fileobj.read(len(BLENDFILE_MAGIC))
if magic == BLENDFILE_MAGIC:
self.is_compressed = False
self.raw_filepath = path
return fileobj
if magic[:2] == GZIP_MAGIC:
self.is_compressed = True
log.debug("compressed blendfile detected: %s", path)
# Decompress to a temporary file.
tmpfile = tempfile.NamedTemporaryFile()
fileobj.seek(0, os.SEEK_SET)
with gzip.GzipFile(fileobj=fileobj, mode=mode) as gzfile:
magic = gzfile.read(len(BLENDFILE_MAGIC))
if magic != BLENDFILE_MAGIC:
raise exceptions.BlendFileError("Compressed file is not a blend file", path)
data = magic
while data:
tmpfile.write(data)
data = gzfile.read(FILE_BUFFER_SIZE)
# Further interaction should be done with the uncompressed file.
self.raw_filepath = pathlib.Path(tmpfile.name)
fileobj.close()
return tmpfile
fileobj.close()
raise exceptions.BlendFileError("File is not a blend file", path)
return decompressed.fileobj
def _load_blocks(self) -> None:
"""Read the blend file to load its DNA structure to memory."""
@@ -187,10 +164,10 @@ class BlendFile:
self.sdna_index_from_id.clear()
while True:
block = BlendFileBlock(self)
if block.code == b'ENDB':
if block.code == b"ENDB":
break
if block.code == b'DNA1':
if block.code == b"DNA1":
self.decode_structs(block)
else:
self.fileobj.seek(block.size, os.SEEK_CUR)
@@ -200,35 +177,37 @@ class BlendFile:
self.block_from_addr[block.addr_old] = block
if not self.structs:
raise exceptions.NoDNA1Block("No DNA1 block in file, not a valid .blend file",
self.filepath)
raise exceptions.NoDNA1Block(
"No DNA1 block in file, not a valid .blend file", self.filepath
)
def __repr__(self) -> str:
clsname = self.__class__.__qualname__
if self.filepath == self.raw_filepath:
return '<%s %r>' % (clsname, self.filepath)
return '<%s %r reading from %r>' % (clsname, self.filepath, self.raw_filepath)
return "<%s %r>" % (clsname, self.filepath)
return "<%s %r reading from %r>" % (clsname, self.filepath, self.raw_filepath)
def __enter__(self) -> 'BlendFile':
def __enter__(self) -> "BlendFile":
return self
def __exit__(self, exctype, excvalue, traceback) -> None:
self.close()
def copy_and_rebind(self, path: pathlib.Path, mode='rb') -> None:
def copy_and_rebind(self, path: pathlib.Path, mode="rb") -> None:
"""Change which file is bound to this BlendFile.
This allows cloning a previously opened file, and rebinding it to reuse
the already-loaded DNA structs and data blocks.
"""
log.debug('Rebinding %r to %s', self, path)
log.debug("Rebinding %r to %s", self, path)
self.close()
_uncache(self.filepath)
self.log.debug('Copying %s to %s', self.filepath, path)
self.log.debug("Copying %s to %s", self.filepath, path)
# TODO(Sybren): remove str() calls when targeting Python 3.6+
shutil.copy(str(self.filepath), str(path))
# dst needs to be a file and not a directory
shutil.copyfile(str(self.filepath), str(path))
self.fileobj = self._open_file(path, mode=mode)
_cache(path, self)
@@ -239,10 +218,10 @@ class BlendFile:
def mark_modified(self) -> None:
"""Recompess the file when it is closed."""
self.log.debug('Marking %s as modified', self.raw_filepath)
self.log.debug("Marking %s as modified", self.raw_filepath)
self._is_modified = True
def find_blocks_from_code(self, code: bytes) -> typing.List['BlendFileBlock']:
def find_blocks_from_code(self, code: bytes) -> typing.List["BlendFileBlock"]:
assert isinstance(code, bytes)
return self.code_index[code]
@@ -255,19 +234,19 @@ class BlendFile:
return
if self._is_modified:
log.debug('closing blend file %s after it was modified', self.raw_filepath)
log.debug("closing blend file %s after it was modified", self.raw_filepath)
if self._is_modified and self.is_compressed:
log.debug("recompressing modified blend file %s", self.raw_filepath)
log.debug("GZip-recompressing modified blend file %s", self.raw_filepath)
self.fileobj.seek(os.SEEK_SET, 0)
with gzip.open(str(self.filepath), 'wb') as gzfile:
with gzip.open(str(self.filepath), "wb") as gzfile:
while True:
data = self.fileobj.read(FILE_BUFFER_SIZE)
if not data:
break
gzfile.write(data)
log.debug("compressing to %s finished", self.filepath)
log.debug("GZip-compression to %s finished", self.filepath)
# Close the file object after recompressing, as it may be a temporary
# file that'll disappear as soon as we close it.
@@ -284,11 +263,15 @@ class BlendFile:
curr_struct = self.structs[sdna_index_curr]
next_struct = self.structs[sdna_index_next]
if curr_struct.size > next_struct.size:
raise RuntimeError("Can't refine to smaller type (%s -> %s)" %
(curr_struct.dna_type_id.decode('utf-8'),
next_struct.dna_type_id.decode('utf-8')))
raise RuntimeError(
"Can't refine to smaller type (%s -> %s)"
% (
curr_struct.dna_type_id.decode("utf-8"),
next_struct.dna_type_id.decode("utf-8"),
)
)
def decode_structs(self, block: 'BlendFileBlock'):
def decode_structs(self, block: "BlendFileBlock"):
"""
DNACatalog is a catalog of all information in the DNA1 file-block
"""
@@ -356,7 +339,9 @@ class BlendFile:
dna_offset = 0
for field_index in range(fields_len):
field_type_index, field_name_index = shortstruct2.unpack_from(data, offset)
field_type_index, field_name_index = shortstruct2.unpack_from(
data, offset
)
offset += 4
dna_type = types[field_type_index]
@@ -381,18 +366,33 @@ class BlendFile:
root = bpathlib.BlendPath(bfile_dir)
abspath = relpath.absolute(root)
my_log = self.log.getChild('abspath')
my_log.debug('Resolved %s relative to %s to %s', relpath, self.filepath, abspath)
my_log = self.log.getChild("abspath")
my_log.debug(
"Resolved %s relative to %s to %s", relpath, self.filepath, abspath
)
return abspath
def dereference_pointer(self, address: int) -> 'BlendFileBlock':
"""Return the pointed-to block, or raise SegmentationFault."""
def dereference_pointer(self, address: int) -> typing.Optional["BlendFileBlock"]:
"""Return the pointed-to block, or raise SegmentationFault.
When BlendFile.strict_pointer_mode is False, the exception will not be
thrown, but None will be returned.
"""
try:
return self.block_from_addr[address]
except KeyError:
raise exceptions.SegmentationFault('address does not exist', address) from None
if self.strict_pointer_mode:
raise exceptions.SegmentationFault(
"address does not exist", address
) from None
log.warning(
"Silenced SegmentationFault caused by dereferencing invalid pointer"
" (0x%x) because strict_pointer_mode is off.",
address,
)
return None
def struct(self, name: bytes) -> dna.Struct:
index = self.sdna_index_from_id[name]
@@ -410,19 +410,26 @@ class BlendFileBlock:
# dependency tracer significantly (p<0.001) faster. In my test case the
# speed improvement was 16% for a 'bam list' command.
__slots__ = (
'bfile', 'code', 'size', 'addr_old', 'sdna_index',
'count', 'file_offset', 'endian', '_id_name',
"bfile",
"code",
"size",
"addr_old",
"sdna_index",
"count",
"file_offset",
"endian",
"_id_name",
)
log = log.getChild('BlendFileBlock')
old_structure = struct.Struct(b'4sI')
log = log.getChild("BlendFileBlock")
old_structure = struct.Struct(b"4sI")
"""old blend files ENDB block structure"""
def __init__(self, bfile: BlendFile) -> None:
self.bfile = bfile
# Defaults; actual values are set by interpreting the block header.
self.code = b''
self.code = b""
self.size = 0
self.addr_old = 0
self.sdna_index = 0
@@ -438,10 +445,14 @@ class BlendFileBlock:
header_struct = bfile.block_header_struct
data = bfile.fileobj.read(header_struct.size)
if len(data) != header_struct.size:
self.log.warning("Blend file %s seems to be truncated, "
"expected %d bytes but could read only %d",
bfile.filepath, header_struct.size, len(data))
self.code = b'ENDB'
self.log.warning(
"Blend file %s seems to be truncated, "
"expected %d bytes but could read only %d",
bfile.filepath,
header_struct.size,
len(data),
)
self.code = b"ENDB"
return
# header size can be 8, 20, or 24 bytes long
@@ -449,14 +460,14 @@ class BlendFileBlock:
# 20: normal headers 32 bit platform
# 24: normal headers 64 bit platform
if len(data) <= 15:
self.log.debug('interpreting block as old-style ENB block')
self.log.debug("interpreting block as old-style ENB block")
blockheader = self.old_structure.unpack(data)
self.code = self.endian.read_data0(blockheader[0])
return
blockheader = header_struct.unpack(data)
self.code = self.endian.read_data0(blockheader[0])
if self.code != b'ENDB':
if self.code != b"ENDB":
self.size = blockheader[1]
self.addr_old = blockheader[2]
self.sdna_index = blockheader[3]
@@ -478,11 +489,13 @@ class BlendFileBlock:
def __eq__(self, other: object) -> bool:
if not isinstance(other, BlendFileBlock):
return False
return (self.code == other.code and
self.addr_old == other.addr_old and
self.bfile.filepath == other.bfile.filepath)
return (
self.code == other.code
and self.addr_old == other.addr_old
and self.bfile.filepath == other.bfile.filepath
)
def __lt__(self, other: 'BlendFileBlock') -> bool:
def __lt__(self, other: "BlendFileBlock") -> bool:
"""Order blocks by file path and offset within that file."""
if not isinstance(other, BlendFileBlock):
raise NotImplemented()
@@ -504,7 +517,7 @@ class BlendFileBlock:
@property
def dna_type_name(self) -> str:
return self.dna_type_id.decode('ascii')
return self.dna_type_id.decode("ascii")
@property
def id_name(self) -> typing.Optional[bytes]:
@@ -515,7 +528,7 @@ class BlendFileBlock:
"""
if self._id_name is ...:
try:
self._id_name = self[b'id', b'name']
self._id_name = self[b"id", b"name"]
except KeyError:
self._id_name = None
@@ -553,16 +566,19 @@ class BlendFileBlock:
:returns: tuple (offset in bytes, length of array in items)
"""
field, field_offset = self.dna_type.field_from_path(self.bfile.header.pointer_size, path)
field, field_offset = self.dna_type.field_from_path(
self.bfile.header.pointer_size, path
)
return self.file_offset + field_offset, field.name.array_size
def get(self,
path: dna.FieldPath,
default=...,
null_terminated=True,
as_str=False,
return_field=False
) -> typing.Any:
def get(
self,
path: dna.FieldPath,
default=...,
null_terminated=True,
as_str=False,
return_field=False,
) -> typing.Any:
"""Read a property and return the value.
:param path: name of the property (like `b'loc'`), tuple of names
@@ -583,21 +599,25 @@ class BlendFileBlock:
dna_struct = self.bfile.structs[self.sdna_index]
field, value = dna_struct.field_get(
self.bfile.header, self.bfile.fileobj, path,
self.bfile.header,
self.bfile.fileobj,
path,
default=default,
null_terminated=null_terminated, as_str=as_str,
null_terminated=null_terminated,
as_str=as_str,
)
if return_field:
return value, field
return value
def get_recursive_iter(self,
path: dna.FieldPath,
path_root: dna.FieldPath = b'',
default=...,
null_terminated=True,
as_str=True,
) -> typing.Iterator[typing.Tuple[dna.FieldPath, typing.Any]]:
def get_recursive_iter(
self,
path: dna.FieldPath,
path_root: dna.FieldPath = b"",
default=...,
null_terminated=True,
as_str=True,
) -> typing.Iterator[typing.Tuple[dna.FieldPath, typing.Any]]:
"""Generator, yields (path, property value) tuples.
If a property cannot be decoded, a string representing its DNA type
@@ -613,20 +633,24 @@ class BlendFileBlock:
try:
# Try accessing as simple property
yield (path_full,
self.get(path_full, default, null_terminated, as_str))
yield (path_full, self.get(path_full, default, null_terminated, as_str))
except exceptions.NoReaderImplemented as ex:
# This was not a simple property, so recurse into its DNA Struct.
dna_type = ex.dna_type
struct_index = self.bfile.sdna_index_from_id.get(dna_type.dna_type_id)
if struct_index is None:
yield (path_full, "<%s>" % dna_type.dna_type_id.decode('ascii'))
yield (path_full, "<%s>" % dna_type.dna_type_id.decode("ascii"))
return
# Recurse through the fields.
for f in dna_type.fields:
yield from self.get_recursive_iter(f.name.name_only, path_full, default=default,
null_terminated=null_terminated, as_str=as_str)
yield from self.get_recursive_iter(
f.name.name_only,
path_full,
default=default,
null_terminated=null_terminated,
as_str=as_str,
)
def hash(self) -> int:
"""Generate a pointer-independent hash for the block.
@@ -657,9 +681,10 @@ class BlendFileBlock:
return dna_struct.field_set(self.bfile.header, self.bfile.fileobj, path, value)
def get_pointer(
self, path: dna.FieldPath,
default=...,
) -> typing.Union[None, 'BlendFileBlock']:
self,
path: dna.FieldPath,
default=...,
) -> typing.Union[None, "BlendFileBlock"]:
"""Same as get() but dereferences a pointer.
:raises exceptions.SegmentationFault: when there is no datablock with
@@ -681,8 +706,9 @@ class BlendFileBlock:
ex.field_path = path
raise
def iter_array_of_pointers(self, path: dna.FieldPath, array_size: int) \
-> typing.Iterator['BlendFileBlock']:
def iter_array_of_pointers(
self, path: dna.FieldPath, array_size: int
) -> typing.Iterator["BlendFileBlock"]:
"""Dereference pointers from an array-of-pointers field.
Use this function when you have a field like Mesh materials:
@@ -698,8 +724,9 @@ class BlendFileBlock:
array = self.get_pointer(path)
assert array is not None
assert array.code == b'DATA', \
'Array data block should have code DATA, is %r' % array.code.decode()
assert array.code == b"DATA", (
"Array data block should have code DATA, is %r" % array.code.decode()
)
file_offset = array.file_offset
endian = self.bfile.header.endian
@@ -711,10 +738,15 @@ class BlendFileBlock:
address = endian.read_pointer(fileobj, ps)
if address == 0:
continue
yield self.bfile.dereference_pointer(address)
dereferenced = self.bfile.dereference_pointer(address)
if dereferenced is None:
# This can happen when strict pointer mode is disabled.
continue
yield dereferenced
def iter_fixed_array_of_pointers(self, path: dna.FieldPath) \
-> typing.Iterator['BlendFileBlock']:
def iter_fixed_array_of_pointers(
self, path: dna.FieldPath
) -> typing.Iterator["BlendFileBlock"]:
"""Yield blocks from a fixed-size array field.
Use this function when you have a field like lamp textures:
@@ -741,7 +773,12 @@ class BlendFileBlock:
if not address:
# Fixed-size arrays contain 0-pointers.
continue
yield self.bfile.dereference_pointer(address)
dereferenced = self.bfile.dereference_pointer(address)
if dereferenced is None:
# This can happen when strict pointer mode is disabled.
continue
yield dereferenced
def __getitem__(self, path: dna.FieldPath):
return self.get(path)
@@ -758,16 +795,32 @@ class BlendFileBlock:
try:
yield self[k]
except exceptions.NoReaderImplemented as ex:
yield '<%s>' % ex.dna_type.dna_type_id.decode('ascii')
yield "<%s>" % ex.dna_type.dna_type_id.decode("ascii")
def items(self) -> typing.Iterable[typing.Tuple[bytes, typing.Any]]:
for k in self.keys():
try:
yield (k, self[k])
except exceptions.NoReaderImplemented as ex:
yield (k, '<%s>' % ex.dna_type.dna_type_id.decode('ascii'))
yield (k, "<%s>" % ex.dna_type.dna_type_id.decode("ascii"))
def items_recursive(self) -> typing.Iterator[typing.Tuple[dna.FieldPath, typing.Any]]:
def items_recursive(
self,
) -> typing.Iterator[typing.Tuple[dna.FieldPath, typing.Any]]:
"""Generator, yields (property path, property value) recursively for all properties."""
for k in self.keys():
yield from self.get_recursive_iter(k, as_str=False)
def set_strict_pointer_mode(strict_pointers: bool) -> None:
"""Control behaviour when a pointer to unknown memory is dereferenced.
Strict pointer mode raise exceptions.SegmentationFault when dereferencing an
unknown pointer. This is the default.
Set to False to disable this exception, and to return None instead, i.e. to
ignore such pointers. Note that this can cause None to be returned from a
non-nil pointer.
"""
BlendFile.strict_pointer_mode = strict_pointers
blendfile/dna.py
+81 -63
View File
@@ -42,38 +42,38 @@ class Name:
self.array_size = self.calc_array_size()
def __repr__(self):
return '%s(%r)' % (type(self).__qualname__, self.name_full)
return "%s(%r)" % (type(self).__qualname__, self.name_full)
def as_reference(self, parent) -> bytes:
if not parent:
return self.name_only
return parent + b'.' + self.name_only
return parent + b"." + self.name_only
def calc_name_only(self) -> bytes:
result = self.name_full.strip(b'*()')
index = result.find(b'[')
result = self.name_full.strip(b"*()")
index = result.find(b"[")
if index == -1:
return result
return result[:index]
def calc_is_pointer(self) -> bool:
return b'*' in self.name_full
return b"*" in self.name_full
def calc_is_method_pointer(self):
return b'(*' in self.name_full
return b"(*" in self.name_full
def calc_array_size(self):
result = 1
partial_name = self.name_full
while True:
idx_start = partial_name.find(b'[')
idx_start = partial_name.find(b"[")
if idx_start < 0:
break
idx_stop = partial_name.find(b']')
result *= int(partial_name[idx_start + 1:idx_stop])
partial_name = partial_name[idx_stop + 1:]
idx_stop = partial_name.find(b"]")
result *= int(partial_name[idx_start + 1 : idx_stop])
partial_name = partial_name[idx_stop + 1 :]
return result
@@ -89,24 +89,20 @@ class Field:
:ivar offset: cached offset of the field, in bytes.
"""
def __init__(self,
dna_type: 'Struct',
name: Name,
size: int,
offset: int) -> None:
def __init__(self, dna_type: "Struct", name: Name, size: int, offset: int) -> None:
self.dna_type = dna_type
self.name = name
self.size = size
self.offset = offset
def __repr__(self):
return '<%r %r (%s)>' % (type(self).__qualname__, self.name, self.dna_type)
return "<%r %r (%s)>" % (type(self).__qualname__, self.name, self.dna_type)
class Struct:
"""dna.Struct is a C-type structure stored in the DNA."""
log = log.getChild('Struct')
log = log.getChild("Struct")
def __init__(self, dna_type_id: bytes, size: int = None) -> None:
"""
@@ -121,13 +117,13 @@ class Struct:
self._fields_by_name = {} # type: typing.Dict[bytes, Field]
def __repr__(self):
return '%s(%r)' % (type(self).__qualname__, self.dna_type_id)
return "%s(%r)" % (type(self).__qualname__, self.dna_type_id)
@property
def size(self) -> int:
if self._size is None:
if not self._fields:
raise ValueError('Unable to determine size of fieldless %r' % self)
raise ValueError("Unable to determine size of fieldless %r" % self)
last_field = max(self._fields, key=lambda f: f.offset)
self._size = last_field.offset + last_field.size
return self._size
@@ -151,10 +147,9 @@ class Struct:
def has_field(self, field_name: bytes) -> bool:
return field_name in self._fields_by_name
def field_from_path(self,
pointer_size: int,
path: FieldPath) \
-> typing.Tuple[Field, int]:
def field_from_path(
self, pointer_size: int, path: FieldPath
) -> typing.Tuple[Field, int]:
"""
Support lookups as bytes or a tuple of bytes and optional index.
@@ -181,12 +176,14 @@ class Struct:
index = 0
if not isinstance(name, bytes):
raise TypeError('name should be bytes, but is %r' % type(name))
raise TypeError("name should be bytes, but is %r" % type(name))
field = self._fields_by_name.get(name)
if not field:
raise KeyError('%r has no field %r, only %r' %
(self, name, sorted(self._fields_by_name.keys())))
raise KeyError(
"%r has no field %r, only %r"
% (self, name, sorted(self._fields_by_name.keys()))
)
offset = field.offset
if index:
@@ -195,8 +192,10 @@ class Struct:
else:
index_offset = field.dna_type.size * index
if index_offset >= field.size:
raise OverflowError('path %r is out of bounds of its DNA type %s' %
(path, field.dna_type))
raise OverflowError(
"path %r is out of bounds of its DNA type %s"
% (path, field.dna_type)
)
offset += index_offset
if name_tail:
@@ -205,14 +204,15 @@ class Struct:
return field, offset
def field_get(self,
file_header: header.BlendFileHeader,
fileobj: typing.IO[bytes],
path: FieldPath,
default=...,
null_terminated=True,
as_str=True,
) -> typing.Tuple[typing.Optional[Field], typing.Any]:
def field_get(
self,
file_header: header.BlendFileHeader,
fileobj: typing.IO[bytes],
path: FieldPath,
default=...,
null_terminated=True,
as_str=True,
) -> typing.Tuple[typing.Optional[Field], typing.Any]:
"""Read the value of the field from the blend file.
Assumes the file pointer of `fileobj` is seek()ed to the start of the
@@ -248,22 +248,26 @@ class Struct:
# Some special cases (pointers, strings/bytes)
if dna_name.is_pointer:
return field, endian.read_pointer(fileobj, file_header.pointer_size)
if dna_type.dna_type_id == b'char':
return field, self._field_get_char(file_header, fileobj, field, null_terminated, as_str)
if dna_type.dna_type_id == b"char":
return field, self._field_get_char(
file_header, fileobj, field, null_terminated, as_str
)
simple_readers = {
b'int': endian.read_int,
b'short': endian.read_short,
b'uint64_t': endian.read_ulong,
b'float': endian.read_float,
b"int": endian.read_int,
b"short": endian.read_short,
b"uint64_t": endian.read_ulong,
b"float": endian.read_float,
}
try:
simple_reader = simple_readers[dna_type.dna_type_id]
except KeyError:
raise exceptions.NoReaderImplemented(
"%r exists but not simple type (%r), can't resolve field %r" %
(path, dna_type.dna_type_id.decode(), dna_name.name_only),
dna_name, dna_type) from None
"%r exists but not simple type (%r), can't resolve field %r"
% (path, dna_type.dna_type_id.decode(), dna_name.name_only),
dna_name,
dna_type,
) from None
if isinstance(path, tuple) and len(path) > 1 and isinstance(path[-1], int):
# The caller wants to get a single item from an array. The offset we seeked to already
@@ -275,12 +279,14 @@ class Struct:
return field, [simple_reader(fileobj) for _ in range(dna_name.array_size)]
return field, simple_reader(fileobj)
def _field_get_char(self,
file_header: header.BlendFileHeader,
fileobj: typing.IO[bytes],
field: 'Field',
null_terminated: typing.Optional[bool],
as_str: bool) -> typing.Any:
def _field_get_char(
self,
file_header: header.BlendFileHeader,
fileobj: typing.IO[bytes],
field: "Field",
null_terminated: typing.Optional[bool],
as_str: bool,
) -> typing.Any:
dna_name = field.name
endian = file_header.endian
@@ -294,21 +300,23 @@ class Struct:
data = fileobj.read(dna_name.array_size)
if as_str:
return data.decode('utf8')
return data.decode("utf8")
return data
def field_set(self,
file_header: header.BlendFileHeader,
fileobj: typing.IO[bytes],
path: bytes,
value: typing.Any):
def field_set(
self,
file_header: header.BlendFileHeader,
fileobj: typing.IO[bytes],
path: bytes,
value: typing.Any,
):
"""Write a value to the blend file.
Assumes the file pointer of `fileobj` is seek()ed to the start of the
struct on disk (e.g. the start of the BlendFileBlock containing the
data).
"""
assert isinstance(path, bytes), 'path should be bytes, but is %s' % type(path)
assert isinstance(path, bytes), "path should be bytes, but is %s" % type(path)
field, offset = self.field_from_path(file_header.pointer_size, path)
@@ -316,19 +324,29 @@ class Struct:
dna_name = field.name
endian = file_header.endian
if dna_type.dna_type_id != b'char':
if dna_type.dna_type_id not in endian.accepted_types():
msg = "Setting type %r is not supported for %s.%s" % (
dna_type, self.dna_type_id.decode(), dna_name.name_full.decode())
dna_type,
self.dna_type_id.decode(),
dna_name.name_full.decode(),
)
raise exceptions.NoWriterImplemented(msg, dna_name, dna_type)
fileobj.seek(offset, os.SEEK_CUR)
if self.log.isEnabledFor(logging.DEBUG):
filepos = fileobj.tell()
thing = 'string' if isinstance(value, str) else 'bytes'
self.log.debug('writing %s %r at file offset %d / %x', thing, value, filepos, filepos)
if isinstance(value, (int, float)):
thing = dna_type.dna_type_id.decode()
else:
thing = "string" if isinstance(value, str) else "bytes"
self.log.debug(
"writing %s %r at file offset %d / %x", thing, value, filepos, filepos
)
if isinstance(value, str):
if isinstance(value, (int, float)):
return endian.accepted_types()[dna_type.dna_type_id](fileobj, value)
elif isinstance(value, str):
return endian.write_string(fileobj, value, dna_name.array_size)
else:
return endian.write_bytes(fileobj, value, dna_name.array_size)
blendfile/dna_io.py
+94 -26
View File
@@ -27,14 +27,14 @@ import typing
class EndianIO:
# TODO(Sybren): note as UCHAR: struct.Struct = None and move actual structs to LittleEndianTypes
UCHAR = struct.Struct(b'<B')
USHORT = struct.Struct(b'<H')
USHORT2 = struct.Struct(b'<HH') # two shorts in a row
SSHORT = struct.Struct(b'<h')
UINT = struct.Struct(b'<I')
SINT = struct.Struct(b'<i')
FLOAT = struct.Struct(b'<f')
ULONG = struct.Struct(b'<Q')
UCHAR = struct.Struct(b"<B")
USHORT = struct.Struct(b"<H")
USHORT2 = struct.Struct(b"<HH") # two shorts in a row
SSHORT = struct.Struct(b"<h")
UINT = struct.Struct(b"<I")
SINT = struct.Struct(b"<i")
FLOAT = struct.Struct(b"<f")
ULONG = struct.Struct(b"<Q")
@classmethod
def _read(cls, fileobj: typing.IO[bytes], typestruct: struct.Struct):
@@ -42,36 +42,74 @@ class EndianIO:
try:
return typestruct.unpack(data)[0]
except struct.error as ex:
raise struct.error('%s (read %d bytes)' % (ex, len(data))) from None
raise struct.error("%s (read %d bytes)" % (ex, len(data))) from None
@classmethod
def _write(
cls, fileobj: typing.IO[bytes], typestruct: struct.Struct, value: typing.Any
):
try:
data = typestruct.pack(value)
except struct.error as ex:
raise struct.error(f"{ex} (write '{value}')")
return fileobj.write(data)
@classmethod
def read_char(cls, fileobj: typing.IO[bytes]):
return cls._read(fileobj, cls.UCHAR)
@classmethod
def write_char(cls, fileobj: typing.IO[bytes], value: int):
return cls._write(fileobj, cls.UCHAR, value)
@classmethod
def read_ushort(cls, fileobj: typing.IO[bytes]):
return cls._read(fileobj, cls.USHORT)
@classmethod
def write_ushort(cls, fileobj: typing.IO[bytes], value: int):
return cls._write(fileobj, cls.USHORT, value)
@classmethod
def read_short(cls, fileobj: typing.IO[bytes]):
return cls._read(fileobj, cls.SSHORT)
@classmethod
def write_short(cls, fileobj: typing.IO[bytes], value: int):
return cls._write(fileobj, cls.SSHORT, value)
@classmethod
def read_uint(cls, fileobj: typing.IO[bytes]):
return cls._read(fileobj, cls.UINT)
@classmethod
def write_uint(cls, fileobj: typing.IO[bytes], value: int):
return cls._write(fileobj, cls.UINT, value)
@classmethod
def read_int(cls, fileobj: typing.IO[bytes]):
return cls._read(fileobj, cls.SINT)
@classmethod
def write_int(cls, fileobj: typing.IO[bytes], value: int):
return cls._write(fileobj, cls.SINT, value)
@classmethod
def read_float(cls, fileobj: typing.IO[bytes]):
return cls._read(fileobj, cls.FLOAT)
@classmethod
def write_float(cls, fileobj: typing.IO[bytes], value: float):
return cls._write(fileobj, cls.FLOAT, value)
@classmethod
def read_ulong(cls, fileobj: typing.IO[bytes]):
return cls._read(fileobj, cls.ULONG)
@classmethod
def write_ulong(cls, fileobj: typing.IO[bytes], value: int):
return cls._write(fileobj, cls.ULONG, value)
@classmethod
def read_pointer(cls, fileobj: typing.IO[bytes], pointer_size: int):
"""Read a pointer from a file."""
@@ -80,10 +118,22 @@ class EndianIO:
return cls.read_uint(fileobj)
if pointer_size == 8:
return cls.read_ulong(fileobj)
raise ValueError('unsupported pointer size %d' % pointer_size)
raise ValueError("unsupported pointer size %d" % pointer_size)
@classmethod
def write_string(cls, fileobj: typing.IO[bytes], astring: str, fieldlen: int) -> int:
def write_pointer(cls, fileobj: typing.IO[bytes], pointer_size: int, value: int):
"""Write a pointer to a file."""
if pointer_size == 4:
return cls.write_uint(fileobj, value)
if pointer_size == 8:
return cls.write_ulong(fileobj, value)
raise ValueError("unsupported pointer size %d" % pointer_size)
@classmethod
def write_string(
cls, fileobj: typing.IO[bytes], astring: str, fieldlen: int
) -> int:
"""Write a (truncated) string as UTF-8.
The string will always be written 0-terminated.
@@ -94,7 +144,7 @@ class EndianIO:
:returns: the number of bytes written.
"""
assert isinstance(astring, str)
encoded = astring.encode('utf-8')
encoded = astring.encode("utf-8")
# Take into account we also need space for a trailing 0-byte.
maxlen = fieldlen - 1
@@ -106,13 +156,13 @@ class EndianIO:
# is valid UTF-8 again.
while True:
try:
encoded.decode('utf8')
encoded.decode("utf8")
except UnicodeDecodeError:
encoded = encoded[:-1]
else:
break
return fileobj.write(encoded + b'\0')
return fileobj.write(encoded + b"\0")
@classmethod
def write_bytes(cls, fileobj: typing.IO[bytes], data: bytes, fieldlen: int) -> int:
@@ -126,7 +176,7 @@ class EndianIO:
if len(data) >= fieldlen:
to_write = data[0:fieldlen]
else:
to_write = data + b'\0'
to_write = data + b"\0"
return fileobj.write(to_write)
@@ -137,27 +187,45 @@ class EndianIO:
@classmethod
def read_data0_offset(cls, data, offset):
add = data.find(b'\0', offset) - offset
return data[offset:offset + add]
add = data.find(b"\0", offset) - offset
return data[offset : offset + add]
@classmethod
def read_data0(cls, data):
add = data.find(b'\0')
add = data.find(b"\0")
if add < 0:
return data
return data[:add]
@classmethod
def accepted_types(cls):
"""Return a mapping from type name to writer function.
This is mostly to make it easier to get the correct number write
function, given that Python's `int` and `float` can map to a whole range
of C types.
"""
return {
b"char": cls.write_char,
b"ushort": cls.write_ushort,
b"short": cls.write_short,
b"uint": cls.write_uint,
b"int": cls.write_int,
b"ulong": cls.write_ulong,
b"float": cls.write_float,
}
class LittleEndianTypes(EndianIO):
pass
class BigEndianTypes(LittleEndianTypes):
UCHAR = struct.Struct(b'>B')
USHORT = struct.Struct(b'>H')
USHORT2 = struct.Struct(b'>HH') # two shorts in a row
SSHORT = struct.Struct(b'>h')
UINT = struct.Struct(b'>I')
SINT = struct.Struct(b'>i')
FLOAT = struct.Struct(b'>f')
ULONG = struct.Struct(b'>Q')
UCHAR = struct.Struct(b">B")
USHORT = struct.Struct(b">H")
USHORT2 = struct.Struct(b">HH") # two shorts in a row
SSHORT = struct.Struct(b">h")
UINT = struct.Struct(b">I")
SINT = struct.Struct(b">i")
FLOAT = struct.Struct(b">f")
ULONG = struct.Struct(b">Q")
blendfile/exceptions.py
+1 -1
View File
@@ -32,7 +32,7 @@ class BlendFileError(Exception):
self.filepath = filepath
def __str__(self):
return '%s: %s' % (super().__str__(), self.filepath)
return "%s: %s" % (super().__str__(), self.filepath)
class NoDNA1Block(BlendFileError):
blendfile/header.py
+24 -15
View File
@@ -37,7 +37,8 @@ class BlendFileHeader:
It contains information about the hardware architecture, which is relevant
to the structure of the rest of the file.
"""
structure = struct.Struct(b'7s1s1s3s')
structure = struct.Struct(b"7s1s1s3s")
def __init__(self, fileobj: typing.IO[bytes], path: pathlib.Path) -> None:
log.debug("reading blend-file-header %s", path)
@@ -48,31 +49,39 @@ class BlendFileHeader:
self.magic = values[0]
pointer_size_id = values[1]
if pointer_size_id == b'-':
if pointer_size_id == b"-":
self.pointer_size = 8
elif pointer_size_id == b'_':
elif pointer_size_id == b"_":
self.pointer_size = 4
else:
raise exceptions.BlendFileError('invalid pointer size %r' % pointer_size_id, path)
raise exceptions.BlendFileError(
"invalid pointer size %r" % pointer_size_id, path
)
endian_id = values[2]
if endian_id == b'v':
if endian_id == b"v":
self.endian = dna_io.LittleEndianTypes
self.endian_str = b'<' # indication for struct.Struct()
elif endian_id == b'V':
self.endian_str = b"<" # indication for struct.Struct()
elif endian_id == b"V":
self.endian = dna_io.BigEndianTypes
self.endian_str = b'>' # indication for struct.Struct()
self.endian_str = b">" # indication for struct.Struct()
else:
raise exceptions.BlendFileError('invalid endian indicator %r' % endian_id, path)
raise exceptions.BlendFileError(
"invalid endian indicator %r" % endian_id, path
)
version_id = values[3]
self.version = int(version_id)
def create_block_header_struct(self) -> struct.Struct:
"""Create a Struct instance for parsing data block headers."""
return struct.Struct(b''.join((
self.endian_str,
b'4sI',
b'I' if self.pointer_size == 4 else b'Q',
b'II',
)))
return struct.Struct(
b"".join(
(
self.endian_str,
b"4sI",
b"I" if self.pointer_size == 4 else b"Q",
b"II",
)
)
)
blendfile/iterators.py
+12 -10
View File
@@ -26,8 +26,9 @@ from . import BlendFileBlock
from .dna import FieldPath
def listbase(block: typing.Optional[BlendFileBlock], next_path: FieldPath = b'next') \
-> typing.Iterator[BlendFileBlock]:
def listbase(
block: typing.Optional[BlendFileBlock], next_path: FieldPath = b"next"
) -> typing.Iterator[BlendFileBlock]:
"""Generator, yields all blocks in the ListBase linked list."""
while block:
yield block
@@ -37,8 +38,9 @@ def listbase(block: typing.Optional[BlendFileBlock], next_path: FieldPath = b'ne
block = block.bfile.dereference_pointer(next_ptr)
def sequencer_strips(sequence_editor: BlendFileBlock) \
-> typing.Iterator[typing.Tuple[BlendFileBlock, int]]:
def sequencer_strips(
sequence_editor: BlendFileBlock,
) -> typing.Iterator[typing.Tuple[BlendFileBlock, int]]:
"""Generator, yield all sequencer strip blocks with their type number.
Recurses into meta strips, yielding both the meta strip itself and the
@@ -49,16 +51,16 @@ def sequencer_strips(sequence_editor: BlendFileBlock) \
def iter_seqbase(seqbase) -> typing.Iterator[typing.Tuple[BlendFileBlock, int]]:
for seq in listbase(seqbase):
seq.refine_type(b'Sequence')
seq_type = seq[b'type']
seq.refine_type(b"Sequence")
seq_type = seq[b"type"]
yield seq, seq_type
if seq_type == cdefs.SEQ_TYPE_META:
# Recurse into this meta-sequence.
subseq = seq.get_pointer((b'seqbase', b'first'))
subseq = seq.get_pointer((b"seqbase", b"first"))
yield from iter_seqbase(subseq)
sbase = sequence_editor.get_pointer((b'seqbase', b'first'))
sbase = sequence_editor.get_pointer((b"seqbase", b"first"))
yield from iter_seqbase(sbase)
@@ -66,5 +68,5 @@ def modifiers(object_block: BlendFileBlock) -> typing.Iterator[BlendFileBlock]:
"""Generator, yield the object's modifiers."""
# 'ob->modifiers[...]'
mods = object_block.get_pointer((b'modifiers', b'first'))
yield from listbase(mods, next_path=(b'modifier', b'next'))
mods = object_block.get_pointer((b"modifiers", b"first"))
yield from listbase(mods, next_path=(b"modifier", b"next"))
blendfile/magic_compression.py
+169
View File
@@ -0,0 +1,169 @@
# ***** BEGIN GPL LICENSE BLOCK *****
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# ***** END GPL LICENCE BLOCK *****
#
# (c) 2021, Blender Foundation
import collections
import enum
import gzip
import logging
import os
import pathlib
import tempfile
import typing
# Blender 3.0 replaces GZip with ZStandard compression.
# Since this is not a standard library package, be careful importing it and
# treat it as optional.
try:
import zstandard
has_zstandard = True
except ImportError:
has_zstandard = False
from . import exceptions
# Magic numbers, see https://en.wikipedia.org/wiki/List_of_file_signatures
BLENDFILE_MAGIC = b"BLENDER"
GZIP_MAGIC = b"\x1f\x8b"
# ZStandard has two magic numbers, the 2nd of which doesn't use the last nibble.
# See https://tools.ietf.org/id/draft-kucherawy-dispatch-zstd-00.html#rfc.section.2.1.1
# and https://tools.ietf.org/id/draft-kucherawy-dispatch-zstd-00.html#rfc.section.2.3
ZSTD_MAGIC = b"\x28\xB5\x2F\xFD"
ZSTD_MAGIC_SKIPPABLE = b"\x50\x2A\x4D\x18"
ZSTD_MAGIC_SKIPPABLE_MASK = b"\xF0\xFF\xFF\xFF"
log = logging.getLogger(__name__)
# @dataclasses.dataclass
DecompressedFileInfo = collections.namedtuple(
"DecompressedFileInfo", "is_compressed path fileobj"
)
# is_compressed: bool
# path: pathlib.Path
# """The path of the decompressed file, or the input path if the file is not compressed."""
# fileobj: BinaryIO
class Compression(enum.Enum):
UNRECOGNISED = -1
NONE = 0
GZIP = 1
ZSTD = 2
def open(path: pathlib.Path, mode: str, buffer_size: int) -> DecompressedFileInfo:
"""Open the file, decompressing it into a temporary file if necesssary."""
fileobj = path.open(mode, buffering=buffer_size) # typing.IO[bytes]
compression = find_compression_type(fileobj)
if compression == Compression.UNRECOGNISED:
fileobj.close()
raise exceptions.BlendFileError("File is not a blend file", path)
if compression == Compression.NONE:
return DecompressedFileInfo(
is_compressed=False,
path=path,
fileobj=fileobj,
)
log.debug("%s-compressed blendfile detected: %s", compression.name, path)
# Decompress to a temporary file.
tmpfile = tempfile.NamedTemporaryFile()
fileobj.seek(0, os.SEEK_SET)
decompressor = _decompressor(fileobj, mode, compression)
with decompressor as compressed_file:
magic = compressed_file.read(len(BLENDFILE_MAGIC))
if magic != BLENDFILE_MAGIC:
raise exceptions.BlendFileError("Compressed file is not a blend file", path)
data = magic
while data:
tmpfile.write(data)
data = compressed_file.read(buffer_size)
# Further interaction should be done with the uncompressed file.
fileobj.close()
return DecompressedFileInfo(
is_compressed=True,
path=pathlib.Path(tmpfile.name),
fileobj=tmpfile,
)
def find_compression_type(fileobj: typing.IO[bytes]) -> Compression:
fileobj.seek(0, os.SEEK_SET)
# This assumes that all magics are not longer than "BLENDER".
magic = fileobj.read(len(BLENDFILE_MAGIC))
if _matches_magic(magic, BLENDFILE_MAGIC):
return Compression.NONE
if _matches_magic(magic, GZIP_MAGIC):
return Compression.GZIP
if _matches_magic(magic, ZSTD_MAGIC):
return Compression.ZSTD
if _matches_magic_masked(magic, ZSTD_MAGIC_SKIPPABLE, ZSTD_MAGIC_SKIPPABLE_MASK):
return Compression.ZSTD
return Compression.UNRECOGNISED
def _matches_magic_masked(value: bytes, magic: bytes, mask: bytes) -> bool:
"""Returns True only if value & mask == magic & mask (ignoring trailing bytes in value)."""
assert len(magic) == len(mask)
int_value = int.from_bytes(value[: len(magic)], "little")
int_magic = int.from_bytes(magic, "little")
int_mask = int.from_bytes(mask, "little")
return int_value & int_mask == int_magic & int_mask
def _matches_magic(value: bytes, magic: bytes) -> bool:
"""Returns True only if value == magic (ignoring trailing bytes in value)."""
return value[: len(magic)] == magic
def _decompressor(
fileobj: typing.IO[bytes], mode: str, compression: Compression
) -> typing.IO[bytes]:
if compression == Compression.GZIP:
decompressor = gzip.GzipFile(fileobj=fileobj, mode=mode)
return typing.cast(typing.IO[bytes], decompressor)
if compression == Compression.ZSTD:
if not has_zstandard:
# The required module was not loaded, raise an exception about this.
raise EnvironmentError(
"File is compressed with ZStandard, install the `zstandard` module to support this."
)
dctx = zstandard.ZstdDecompressor()
return dctx.stream_reader(fileobj)
raise ValueError("Unsupported compression type: %s" % compression)