Allow packing to ZIP files

blender_asset_tracer/cli/pack.py

@@ -69,6 +69,9 @@ def create_packer(args, bpath: pathlib.Path, ppath: pathlib.Path,
             raise ValueError('S3 uploader does not support no-op.')
 
         packer = create_s3packer(bpath, ppath, tpath)
+    elif tpath.suffix.lower() == '.zip':
+        from blender_asset_tracer.pack import zipped
+        packer = zipped.ZipPacker(bpath, ppath, tpath, args.noop)
     else:
         packer = pack.Packer(bpath, ppath, tpath, args.noop)
 
@@ -109,9 +112,6 @@ def paths_from_cli(args) -> typing.Tuple[pathlib.Path, pathlib.Path, pathlib.Pat
     bpath = bpath.absolute().resolve()
 
     tpath = args.target
-    if tpath.exists() and not tpath.is_dir():
-        log.critical('Target %s exists and is not a directory', tpath)
-        sys.exit(4)
 
     if args.project is None:
         ppath = bpath.absolute().parent.resolve()

blender_asset_tracer/pack/s3.py

@@ -18,11 +18,10 @@
 #
 # (c) 2018, Blender Foundation - Sybren A. Stüvel
 """Amazon S3-compatible uploader."""
-import typing
-
 import hashlib
 import logging
 import pathlib
+import typing
 import urllib.parse
 
 from . import Packer, transfer
@@ -157,14 +156,6 @@ class S3Transferrer(transfer.FileTransferer):
             raise self.AbortUpload('interrupting ongoing upload')
         super().report_transferred(bytes_transferred)
 
-    def delete_file(self, path: pathlib.Path):
-        """Deletes a file, only logging a warning if deletion fails."""
-        log.debug('Deleting %s, file has been uploaded', path)
-        try:
-            path.unlink()
-        except IOError as ex:
-            log.warning('Unable to delete %s: %s', path, ex)
-
     def get_metadata(self, bucket: str, key: str) -> typing.Tuple[str, int]:
         """Get MD5 sum and size on S3.
 

blender_asset_tracer/pack/transfer.py

@@ -181,3 +181,11 @@ class FileTransferer(threading.Thread, metaclass=abc.ABCMeta):
 
         # Since Thread.join() neither returns anything nor raises any exception
         # when timing out, we don't even have to call it any more.
+
+    def delete_file(self, path: pathlib.Path):
+        """Deletes a file, only logging a warning if deletion fails."""
+        log.debug('Deleting %s, file has been transferred', path)
+        try:
+            path.unlink()
+        except IOError as ex:
+            log.warning('Unable to delete %s: %s', path, ex)

blender_asset_tracer/pack/zipped.py

@@ -0,0 +1,88 @@
+# ***** BEGIN GPL LICENSE BLOCK *****
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+# ***** END GPL LICENCE BLOCK *****
+#
+# (c) 2018, Blender Foundation - Sybren A. Stüvel
+"""ZIP file packer.
+
+Note: There is no official file name encoding for ZIP files. Expect trouble
+when you want to use the ZIP cross-platform and you have non-ASCII names.
+"""
+import logging
+import pathlib
+
+from . import Packer, transfer
+
+log = logging.getLogger(__name__)
+
+# Suffixes to store uncompressed in the zip.
+STORE_ONLY = {'.jpg', '.jpeg', '.exr'}
+
+
+class ZipPacker(Packer):
+    """Creates a zipped BAT Pack instead of a directory."""
+
+    def _create_file_transferer(self) -> transfer.FileTransferer:
+        return ZipTransferrer(self.target.absolute())
+
+
+class ZipTransferrer(transfer.FileTransferer):
+    """Creates a ZIP file instead of writing to a directory.
+
+    Note: There is no official file name encoding for ZIP files. If you have
+    unicode file names, they will be encoded as UTF-8. WinZip interprets all
+    file names as encoded in CP437, also known as DOS Latin.
+    """
+
+    def __init__(self, zippath: pathlib.Path) -> None:
+        super().__init__()
+        self.zippath = zippath
+
+    def run(self) -> None:
+        import zipfile
+
+        zippath = self.zippath.absolute()
+
+        with zipfile.ZipFile(str(zippath), 'w') as outzip:
+            for src, dst, act in self.iter_queue():
+                assert src.is_absolute(), 'expecting only absolute paths, not %r' % src
+
+                dst = dst.absolute()
+                try:
+                    relpath = dst.relative_to(zippath)
+
+                    # Don't bother trying to compress already-compressed files.
+                    if src.suffix.lower() in STORE_ONLY:
+                        compression = zipfile.ZIP_STORED
+                        log.debug('ZIP %s -> %s (uncompressed)', src, relpath)
+                    else:
+                        compression = zipfile.ZIP_DEFLATED
+                        log.debug('ZIP %s -> %s', src, relpath)
+                    outzip.write(str(src), arcname=str(relpath), compress_type=compression)
+
+                    if act == transfer.Action.MOVE:
+                        self.delete_file(src)
+                except Exception:
+                    # We have to catch exceptions in a broad way, as this is running in
+                    # a separate thread, and exceptions won't otherwise be seen.
+                    log.exception('Error transferring %s to %s', src, dst)
+                    # Put the files to copy back into the queue, and abort. This allows
+                    # the main thread to inspect the queue and see which files were not
+                    # copied. The one we just failed (due to this exception) should also
+                    # be reported there.
+                    self.queue.put((src, dst, act))
+                    return

tests/test_pack_zipped.py

@@ -0,0 +1,18 @@
+import zipfile
+from test_pack import AbstractPackTest
+
+from blender_asset_tracer.pack import zipped
+
+
+class ZippedPackTest(AbstractPackTest):
+    def test_basic_file(self):
+        infile = self.blendfiles / 'basic_file_ñønæščii.blend'
+        zippath = self.tpath / 'target.zip'
+        with zipped.ZipPacker(infile, infile.parent, zippath) as packer:
+            packer.strategise()
+            packer.execute()
+
+        self.assertTrue(zippath.exists())
+        with zipfile.ZipFile(str(zippath)) as inzip:
+            inzip.testzip()
+            self.assertEqual({'pack-info.txt', 'basic_file_ñønæščii.blend'}, set(inzip.namelist()))