Support .blend files saved with large bhead

This is mostly the same as blender/blender!140195. The header parsing code has
been updated to be able to read old and new .blend file headers.

There is a new test file which is the same as the existing `basic_file.blend`,
but saved with the new header format. A new unit test has been added to check
that this file is read correctly as well.

Pull Request: https://projects.blender.org/blender/blender-asset-tracer/pulls/92893

blender_asset_tracer/blendfile/__init__.py

@@ -135,7 +135,7 @@ class BlendFile:
         self.block_from_addr = {}  # type: typing.Dict[int, BlendFileBlock]
 
         self.header = header.BlendFileHeader(self.fileobj, self.raw_filepath)
-        self.block_header_struct = self.header.create_block_header_struct()
+        self.block_header_struct, self.block_header_fields = self.header.create_block_header_struct()
         self._load_blocks()
 
     def _open_file(self, path: pathlib.Path, mode: str) -> typing.IO[bytes]:
@@ -455,23 +455,13 @@ class BlendFileBlock:
             self.code = b"ENDB"
             return
 
-        # header size can be 8, 20, or 24 bytes long
+        blockheader = bfile.block_header_fields(*header_struct.unpack(data))
-        # 8: old blend files ENDB block (exception)
+        self.code = self.endian.read_data0(blockheader.code)
-        # 20: normal headers 32 bit platform
-        # 24: normal headers 64 bit platform
-        if len(data) <= 15:
-            self.log.debug("interpreting block as old-style ENB block")
-            blockheader = self.old_structure.unpack(data)
-            self.code = self.endian.read_data0(blockheader[0])
-            return
-
-        blockheader = header_struct.unpack(data)
-        self.code = self.endian.read_data0(blockheader[0])
         if self.code != b"ENDB":
-            self.size = blockheader[1]
+            self.size = blockheader.len
-            self.addr_old = blockheader[2]
+            self.addr_old = blockheader.old
-            self.sdna_index = blockheader[3]
+            self.sdna_index = blockheader.SDNAnr
-            self.count = blockheader[4]
+            self.count = blockheader.nr
             self.file_offset = bfile.fileobj.tell()
 
     def __repr__(self) -> str:

blender_asset_tracer/blendfile/header.py

@@ -19,6 +19,8 @@
 # (c) 2009, At Mind B.V. - Jeroen Bakker
 # (c) 2014, Blender Foundation - Campbell Barton
 # (c) 2018, Blender Foundation - Sybren A. Stüvel
+
+from dataclasses import dataclass
 import logging
 import os
 import pathlib
@@ -30,58 +32,151 @@ from . import dna_io, exceptions
 log = logging.getLogger(__name__)
 
 
+@dataclass
+class BHead4:
+    code: bytes
+    len: int
+    old: int
+    SDNAnr: int
+    nr: int
+
+@dataclass
+class SmallBHead8:
+    code: bytes
+    len: int
+    old: int
+    SDNAnr: int
+    nr: int
+
+@dataclass
+class LargeBHead8:
+    code: bytes
+    SDNAnr: int
+    old: int
+    len: int
+    nr: int
+
 class BlendFileHeader:
     """
-    BlendFileHeader represents the first 12 bytes of a blend file.
+    BlendFileHeader represents the first 12-17 bytes of a blend file.
 
     It contains information about the hardware architecture, which is relevant
     to the structure of the rest of the file.
     """
 
-    structure = struct.Struct(b"7s1s1s3s")
+    magic: bytes
+    file_format_version: int
+    pointer_size: int
+    is_little_endian: bool
+    endian: typing.Type[dna_io.EndianIO]
+    endian_str: bytes
 
     def __init__(self, fileobj: typing.IO[bytes], path: pathlib.Path) -> None:
         log.debug("reading blend-file-header %s", path)
         fileobj.seek(0, os.SEEK_SET)
-        header = fileobj.read(self.structure.size)
-        values = self.structure.unpack(header)
 
-        self.magic = values[0]
+        bytes_0_6 = fileobj.read(7)
+        if bytes_0_6 != b'BLENDER':
+            raise exceptions.BlendFileError("invalid first bytes %r" % bytes_0_6, path)
+        self.magic = bytes_0_6
 
-        pointer_size_id = values[1]
+        byte_7 = fileobj.read(1)
-        if pointer_size_id == b"-":
+        is_legacy_header = byte_7 in (b'_', b'-')
+        if is_legacy_header:
+            self.file_format_version = 0
+            if byte_7 == b'_':
+                self.pointer_size = 4
+            elif byte_7 == b'-':
+                self.pointer_size = 8
+            else:
+                raise exceptions.BlendFileError("invalid pointer size %r" % byte_7, path)
+            byte_8 = fileobj.read(1)
+            if byte_8 == b'v':
+                self.is_little_endian = True
+            elif byte_8 == b'V':
+                self.is_little_endian = False
+            else:
+                raise exceptions.BlendFileError("invalid endian indicator %r" % byte_8, path)
+            bytes_9_11 = fileobj.read(3)
+            self.version = int(bytes_9_11)
+        else:
+            byte_8 = fileobj.read(1)
+            header_size = int(byte_7 + byte_8)
+            if header_size != 17:
+                raise exceptions.BlendFileError("unknown file header size %d" % header_size, path)
+            byte_9 = fileobj.read(1)
+            if byte_9 != b'-':
+                raise exceptions.BlendFileError("invalid file header", path)
             self.pointer_size = 8
-        elif pointer_size_id == b"_":
+            byte_10_11 = fileobj.read(2)
-            self.pointer_size = 4
+            self.file_format_version = int(byte_10_11)
-        else:
+            if self.file_format_version != 1:
-            raise exceptions.BlendFileError(
+                raise exceptions.BlendFileError("unsupported file format version %r" % self.file_format_version, path)
-                "invalid pointer size %r" % pointer_size_id, path
+            byte_12 = fileobj.read(1)
-            )
+            if byte_12 != b'v':
+                raise exceptions.BlendFileError("invalid file header", path)
+            self.is_little_endian = True
+            byte_13_16 = fileobj.read(4)
+            self.version = int(byte_13_16)
 
-        endian_id = values[2]
+        if self.is_little_endian:
-        if endian_id == b"v":
+            self.endian_str = b'<'
             self.endian = dna_io.LittleEndianTypes
-            self.endian_str = b"<"  # indication for struct.Struct()
-        elif endian_id == b"V":
-            self.endian = dna_io.BigEndianTypes
-            self.endian_str = b">"  # indication for struct.Struct()
         else:
-            raise exceptions.BlendFileError(
+            self.endian_str = b'>'
-                "invalid endian indicator %r" % endian_id, path
+            self.endian = dna_io.BigEndianTypes
-            )
 
-        version_id = values[3]
+    def create_block_header_struct(self) -> typing.Tuple[struct.Struct, typing.Type[typing.Union[BHead4, SmallBHead8, LargeBHead8]]]:
-        self.version = int(version_id)
+        """
+        Returns a Struct instance for parsing data block headers and a corresponding
+        Python class for accessing the right members. Ddepending on the .blend file,
+        the order of the data members in the block header may be different.
+        """
+        assert self.file_format_version in (0, 1)
+        if self.file_format_version == 1:
+            header_struct = struct.Struct(b''.join((
+                self.endian_str,
+                # LargeBHead8.code
+                b'4s',
+                # LargeBHead8.SDNAnr
+                b'i',
+                # LargeBHead8.old
+                b'Q',
+                # LargeBHead8.len
+                b'q',
+                # LargeBHead8.nr
+                b'q',
+            )))
+            return header_struct, LargeBHead8
 
-    def create_block_header_struct(self) -> struct.Struct:
+        if self.pointer_size == 4:
-        """Create a Struct instance for parsing data block headers."""
+            header_struct = struct.Struct(b''.join((
-        return struct.Struct(
+                self.endian_str,
-            b"".join(
+                # BHead4.code
-                (
+                b'4s',
-                    self.endian_str,
+                # BHead4.len
-                    b"4sI",
+                b'i',
-                    b"I" if self.pointer_size == 4 else b"Q",
+                # BHead4.old
-                    b"II",
+                b'I',
-                )
+                # BHead4.SDNAnr
-            )
+                b'i',
-        )
+                # BHead4.nr
+                b'i',
+            )))
+            return header_struct, BHead4
+
+        assert self.pointer_size == 8
+        header_struct = struct.Struct(b''.join((
+            self.endian_str,
+            # SmallBHead8.code
+            b'4s',
+            # SmallBHead8.len
+            b'i',
+            # SmallBHead8.old
+            b'Q',
+            # SmallBHead8.SDNAnr
+            b'i',
+            # SmallBHead8.nr
+            b'i',
+        )))
+        return header_struct, SmallBHead8

tests/test_blendfile_loading.py

@@ -13,6 +13,7 @@ class BlendFileBlockTest(AbstractBlendFileTest):
 
     def test_loading(self):
         self.assertFalse(self.bf.is_compressed)
+        self.assertEqual(0, self.bf.header.file_format_version)
 
     def test_some_properties(self):
         ob = self.bf.code_index[b"OB"][0]
@@ -154,6 +155,23 @@ class BlendFileBlockTest(AbstractBlendFileTest):
         self.assertEqual("OBümlaut", ob.id_name.decode())
 
 
+class BlendFileLargeBhead8Test(AbstractBlendFileTest):
+    def setUp(self):
+        self.bf = blendfile.BlendFile(self.blendfiles / "basic_file_large_bhead8.blend")
+
+    def test_loading(self):
+        self.assertFalse(self.bf.is_compressed)
+        self.assertEqual(1, self.bf.header.file_format_version)
+
+    def test_some_properties(self):
+        ob = self.bf.code_index[b"OB"][0]
+        self.assertEqual("Object", ob.dna_type_name)
+
+        # Try high level operation to read the object location.
+        loc = ob.get(b"loc")
+        self.assertEqual([2.0, 3.0, 5.0], loc)
+
+
 class PointerTest(AbstractBlendFileTest):
     def setUp(self):
         self.bf = blendfile.BlendFile(self.blendfiles / "with_sequencer.blend")