Avoid doubly-compressing zstandard files

This commit is contained in:
Sybren A. Stüvel
2022-02-03 17:54:03 +01:00
parent 2bf155a5f9
commit e5c7e38b44
5 changed files with 18 additions and 8 deletions
@@ -73,7 +73,7 @@ class Compression(enum.Enum):
def open(path: pathlib.Path, mode: str, buffer_size: int) -> DecompressedFileInfo:
"""Open the file, decompressing it into a temporary file if necesssary."""
fileobj = path.open(mode, buffering=buffer_size) # typing.IO[bytes]
compression = _find_compression_type(fileobj)
compression = find_compression_type(fileobj)
if compression == Compression.UNRECOGNISED:
fileobj.close()
@@ -113,7 +113,7 @@ def open(path: pathlib.Path, mode: str, buffer_size: int) -> DecompressedFileInf
)
def _find_compression_type(fileobj: typing.IO[bytes]) -> Compression:
def find_compression_type(fileobj: typing.IO[bytes]) -> Compression:
fileobj.seek(0, os.SEEK_SET)
# This assumes that all magics are not longer than "BLENDER".
+8 -5
View File
@@ -5,6 +5,8 @@ import logging
import pathlib
import shutil
from blender_asset_tracer.blendfile import magic_compression
log = logging.getLogger(__name__)
# Arbitrarily chosen block size, in bytes.
@@ -55,11 +57,12 @@ def _move_or_copy(
"""
srcfile = src.open("rb")
try:
first_bytes = srcfile.read(2)
if first_bytes == b"\x1f\x8b":
# Already a gzipped file.
comp_type = magic_compression.find_compression_type(srcfile)
if comp_type != magic_compression.Compression.NONE:
# Either already compressed or not a blend file.
# Either way we shouldn't attempt compressing this file.
srcfile.close()
my_log.debug("Source file %s is GZipped already", src)
my_log.debug("Source file %s is compressed already", src)
if source_must_remain:
shutil.copy2(str(src), str(dest))
else:
@@ -67,8 +70,8 @@ def _move_or_copy(
return
my_log.debug("Compressing %s on the fly while copying to %s", src, dest)
srcfile.seek(0)
with gzip.open(str(dest), mode="wb") as destfile:
destfile.write(first_bytes)
shutil.copyfileobj(srcfile, destfile, BLOCK_SIZE)
srcfile.close()