diff --git a/CHANGELOG.md b/CHANGELOG.md index 1919581..5a8f26c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ This file logs the changes that are actually interesting to users (new features, changed functionality, fixed bugs). +# Version 1.10 (2022-02-03) + +- Avoid doubly-compressing ZStandard (Blender 3) compressed files. + # Version 1.9 (2021-11-19) - Add `bat version` command to print just the version number and exit. diff --git a/blender_asset_tracer/blendfile/magic_compression.py b/blender_asset_tracer/blendfile/magic_compression.py index 1cb06ae..51a1cc4 100644 --- a/blender_asset_tracer/blendfile/magic_compression.py +++ b/blender_asset_tracer/blendfile/magic_compression.py @@ -73,7 +73,7 @@ class Compression(enum.Enum): def open(path: pathlib.Path, mode: str, buffer_size: int) -> DecompressedFileInfo: """Open the file, decompressing it into a temporary file if necesssary.""" fileobj = path.open(mode, buffering=buffer_size) # typing.IO[bytes] - compression = _find_compression_type(fileobj) + compression = find_compression_type(fileobj) if compression == Compression.UNRECOGNISED: fileobj.close() @@ -113,7 +113,7 @@ def open(path: pathlib.Path, mode: str, buffer_size: int) -> DecompressedFileInf ) -def _find_compression_type(fileobj: typing.IO[bytes]) -> Compression: +def find_compression_type(fileobj: typing.IO[bytes]) -> Compression: fileobj.seek(0, os.SEEK_SET) # This assumes that all magics are not longer than "BLENDER". diff --git a/blender_asset_tracer/compressor.py b/blender_asset_tracer/compressor.py index 4e82d22..d9ceac3 100644 --- a/blender_asset_tracer/compressor.py +++ b/blender_asset_tracer/compressor.py @@ -5,6 +5,8 @@ import logging import pathlib import shutil +from blender_asset_tracer.blendfile import magic_compression + log = logging.getLogger(__name__) # Arbitrarily chosen block size, in bytes. @@ -55,11 +57,12 @@ def _move_or_copy( """ srcfile = src.open("rb") try: - first_bytes = srcfile.read(2) - if first_bytes == b"\x1f\x8b": - # Already a gzipped file. + comp_type = magic_compression.find_compression_type(srcfile) + if comp_type != magic_compression.Compression.NONE: + # Either already compressed or not a blend file. + # Either way we shouldn't attempt compressing this file. srcfile.close() - my_log.debug("Source file %s is GZipped already", src) + my_log.debug("Source file %s is compressed already", src) if source_must_remain: shutil.copy2(str(src), str(dest)) else: @@ -67,8 +70,8 @@ def _move_or_copy( return my_log.debug("Compressing %s on the fly while copying to %s", src, dest) + srcfile.seek(0) with gzip.open(str(dest), mode="wb") as destfile: - destfile.write(first_bytes) shutil.copyfileobj(srcfile, destfile, BLOCK_SIZE) srcfile.close() diff --git a/tests/test_blendfile_loading.py b/tests/test_blendfile_loading.py index 5e8e909..914c071 100644 --- a/tests/test_blendfile_loading.py +++ b/tests/test_blendfile_loading.py @@ -283,7 +283,7 @@ class CompressionRecognitionTest(AbstractBlendFileTest): def _find_compression_type(self, filename: str) -> magic_compression.Compression: path = self.blendfiles / filename with path.open("rb") as fileobj: - return magic_compression._find_compression_type(fileobj) + return magic_compression.find_compression_type(fileobj) def test_gzip_recognition(self): comp = self._find_compression_type("basic_file_compressed.blend") diff --git a/tests/test_compressor.py b/tests/test_compressor.py index 056df4c..2c3c7ad 100644 --- a/tests/test_compressor.py +++ b/tests/test_compressor.py @@ -68,6 +68,9 @@ class CompressorTest(AbstractBlendFileTest): def test_copy_already_compressed(self): self._test("basic_file_ñønæščii.blend", True) + def test_copy_zstandard_compressed(self): + self._test("basic_file_zstandard.blend", True) + def test_copy_compress_on_the_fly(self): self._test("basic_file.blend", True)