diff --git a/blender_asset_tracer/cli/pack.py b/blender_asset_tracer/cli/pack.py index 0c868f6..b1a3d06 100644 --- a/blender_asset_tracer/cli/pack.py +++ b/blender_asset_tracer/cli/pack.py @@ -69,6 +69,9 @@ def create_packer(args, bpath: pathlib.Path, ppath: pathlib.Path, raise ValueError('S3 uploader does not support no-op.') packer = create_s3packer(bpath, ppath, tpath) + elif tpath.suffix.lower() == '.zip': + from blender_asset_tracer.pack import zipped + packer = zipped.ZipPacker(bpath, ppath, tpath, args.noop) else: packer = pack.Packer(bpath, ppath, tpath, args.noop) @@ -109,9 +112,6 @@ def paths_from_cli(args) -> typing.Tuple[pathlib.Path, pathlib.Path, pathlib.Pat bpath = bpath.absolute().resolve() tpath = args.target - if tpath.exists() and not tpath.is_dir(): - log.critical('Target %s exists and is not a directory', tpath) - sys.exit(4) if args.project is None: ppath = bpath.absolute().parent.resolve() diff --git a/blender_asset_tracer/pack/s3.py b/blender_asset_tracer/pack/s3.py index b8141e2..bee5e09 100644 --- a/blender_asset_tracer/pack/s3.py +++ b/blender_asset_tracer/pack/s3.py @@ -18,11 +18,10 @@ # # (c) 2018, Blender Foundation - Sybren A. Stüvel """Amazon S3-compatible uploader.""" -import typing - import hashlib import logging import pathlib +import typing import urllib.parse from . import Packer, transfer @@ -157,14 +156,6 @@ class S3Transferrer(transfer.FileTransferer): raise self.AbortUpload('interrupting ongoing upload') super().report_transferred(bytes_transferred) - def delete_file(self, path: pathlib.Path): - """Deletes a file, only logging a warning if deletion fails.""" - log.debug('Deleting %s, file has been uploaded', path) - try: - path.unlink() - except IOError as ex: - log.warning('Unable to delete %s: %s', path, ex) - def get_metadata(self, bucket: str, key: str) -> typing.Tuple[str, int]: """Get MD5 sum and size on S3. diff --git a/blender_asset_tracer/pack/transfer.py b/blender_asset_tracer/pack/transfer.py index 224750d..23c9250 100644 --- a/blender_asset_tracer/pack/transfer.py +++ b/blender_asset_tracer/pack/transfer.py @@ -181,3 +181,11 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta): # Since Thread.join() neither returns anything nor raises any exception # when timing out, we don't even have to call it any more. + + def delete_file(self, path: pathlib.Path): + """Deletes a file, only logging a warning if deletion fails.""" + log.debug('Deleting %s, file has been transferred', path) + try: + path.unlink() + except IOError as ex: + log.warning('Unable to delete %s: %s', path, ex) diff --git a/blender_asset_tracer/pack/zipped.py b/blender_asset_tracer/pack/zipped.py new file mode 100644 index 0000000..d558515 --- /dev/null +++ b/blender_asset_tracer/pack/zipped.py @@ -0,0 +1,88 @@ +# ***** BEGIN GPL LICENSE BLOCK ***** +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# ***** END GPL LICENCE BLOCK ***** +# +# (c) 2018, Blender Foundation - Sybren A. Stüvel +"""ZIP file packer. + +Note: There is no official file name encoding for ZIP files. Expect trouble +when you want to use the ZIP cross-platform and you have non-ASCII names. +""" +import logging +import pathlib + +from . import Packer, transfer + +log = logging.getLogger(__name__) + +# Suffixes to store uncompressed in the zip. +STORE_ONLY = {'.jpg', '.jpeg', '.exr'} + + +class ZipPacker(Packer): + """Creates a zipped BAT Pack instead of a directory.""" + + def _create_file_transferer(self) -> transfer.FileTransferer: + return ZipTransferrer(self.target.absolute()) + + +class ZipTransferrer(transfer.FileTransferer): + """Creates a ZIP file instead of writing to a directory. + + Note: There is no official file name encoding for ZIP files. If you have + unicode file names, they will be encoded as UTF-8. WinZip interprets all + file names as encoded in CP437, also known as DOS Latin. + """ + + def __init__(self, zippath: pathlib.Path) -> None: + super().__init__() + self.zippath = zippath + + def run(self) -> None: + import zipfile + + zippath = self.zippath.absolute() + + with zipfile.ZipFile(str(zippath), 'w') as outzip: + for src, dst, act in self.iter_queue(): + assert src.is_absolute(), 'expecting only absolute paths, not %r' % src + + dst = dst.absolute() + try: + relpath = dst.relative_to(zippath) + + # Don't bother trying to compress already-compressed files. + if src.suffix.lower() in STORE_ONLY: + compression = zipfile.ZIP_STORED + log.debug('ZIP %s -> %s (uncompressed)', src, relpath) + else: + compression = zipfile.ZIP_DEFLATED + log.debug('ZIP %s -> %s', src, relpath) + outzip.write(str(src), arcname=str(relpath), compress_type=compression) + + if act == transfer.Action.MOVE: + self.delete_file(src) + except Exception: + # We have to catch exceptions in a broad way, as this is running in + # a separate thread, and exceptions won't otherwise be seen. + log.exception('Error transferring %s to %s', src, dst) + # Put the files to copy back into the queue, and abort. This allows + # the main thread to inspect the queue and see which files were not + # copied. The one we just failed (due to this exception) should also + # be reported there. + self.queue.put((src, dst, act)) + return diff --git a/tests/blendfiles/basic_file_ñønæščii.blend b/tests/blendfiles/basic_file_ñønæščii.blend new file mode 100644 index 0000000..d2fa563 Binary files /dev/null and b/tests/blendfiles/basic_file_ñønæščii.blend differ diff --git a/tests/test_pack_zipped.py b/tests/test_pack_zipped.py new file mode 100644 index 0000000..e393aa5 --- /dev/null +++ b/tests/test_pack_zipped.py @@ -0,0 +1,18 @@ +import zipfile +from test_pack import AbstractPackTest + +from blender_asset_tracer.pack import zipped + + +class ZippedPackTest(AbstractPackTest): + def test_basic_file(self): + infile = self.blendfiles / 'basic_file_ñønæščii.blend' + zippath = self.tpath / 'target.zip' + with zipped.ZipPacker(infile, infile.parent, zippath) as packer: + packer.strategise() + packer.execute() + + self.assertTrue(zippath.exists()) + with zipfile.ZipFile(str(zippath)) as inzip: + inzip.testzip() + self.assertEqual({'pack-info.txt', 'basic_file_ñønæščii.blend'}, set(inzip.namelist()))