Rewrite blend files in temporary directory, before copying

This allows us to later support uploading to a non-local filesystem, such as Amazon S3 cloud storage, after path rewriting is complete.
2018-03-09 12:33:15 +01:00
parent 384f1d5107
commit 824ca4bcb7
4 changed files with 212 additions and 45 deletions
@@ -1,5 +1,6 @@
 import os
 import pathlib
+import tempfile

 from blender_asset_tracer import blendfile
 from blender_asset_tracer.blendfile import iterators, exceptions
@@ -284,3 +285,93 @@ class LoadNonBlendfileTest(AbstractBlendFileTest):
    def test_no_datablocks(self):
        with self.assertRaises(exceptions.NoDNA1Block):
            blendfile.BlendFile(self.blendfiles / 'corrupt_only_magic.blend')
+
+
+class BlendFileCacheTest(AbstractBlendFileTest):
+    def setUp(self):
+        super().setUp()
+        self.tdir = tempfile.TemporaryDirectory()
+        self.tpath = pathlib.Path(self.tdir.name)
+
+    def tearDown(self):
+        self.tdir.cleanup()
+
+    def test_open_cached(self):
+        infile = self.blendfiles / 'basic_file.blend'
+        bf1 = blendfile.open_cached(infile)
+        bf2 = blendfile.open_cached(infile)
+
+        # The file should only be opened & parsed once.
+        self.assertIs(bf1, bf2)
+        self.assertIs(bf1, blendfile._cached_bfiles[infile])
+
+    def test_compressed(self):
+        infile = self.blendfiles / 'linked_cube_compressed.blend'
+        bf1 = blendfile.open_cached(infile)
+        bf2 = blendfile.open_cached(infile)
+
+        # The file should only be opened & parsed once.
+        self.assertIs(bf1, bf2)
+        self.assertIs(bf1, blendfile._cached_bfiles[infile])
+
+    def test_closed(self):
+        infile = self.blendfiles / 'linked_cube_compressed.blend'
+        bf = blendfile.open_cached(infile)
+        self.assertIs(bf, blendfile._cached_bfiles[infile])
+
+        blendfile.close_all_cached()
+        self.assertTrue(bf.fileobj.closed)
+        self.assertEqual({}, blendfile._cached_bfiles)
+
+    def test_close_one_file(self):
+        path1 = self.blendfiles / 'linked_cube_compressed.blend'
+        path2 = self.blendfiles / 'basic_file.blend'
+        bf1 = blendfile.open_cached(path1)
+        bf2 = blendfile.open_cached(path2)
+        self.assertIs(bf1, blendfile._cached_bfiles[path1])
+
+        # Closing a file should remove it from the cache.
+        bf1.close()
+        self.assertTrue(bf1.fileobj.closed)
+        self.assertEqual({path2: bf2}, blendfile._cached_bfiles)
+
+    def test_open_and_rebind(self):
+        infile = self.blendfiles / 'linked_cube.blend'
+        other = self.tpath / 'copy.blend'
+        self._open_and_rebind_test(infile, other)
+
+    def test_open_and_rebind_compressed(self):
+        infile = self.blendfiles / 'linked_cube_compressed.blend'
+        other = self.tpath / 'copy.blend'
+        self._open_and_rebind_test(infile, other)
+
+    def _open_and_rebind_test(self, infile: pathlib.Path, other: pathlib.Path):
+        self.assertFalse(other.exists())
+
+        bf = blendfile.open_cached(infile)
+
+        self.assertEqual(str(bf.raw_filepath), bf.fileobj.name)
+
+        before_filepath = bf.filepath
+        before_raw_fp = bf.raw_filepath
+        before_blocks = bf.blocks
+        before_compressed = bf.is_compressed
+
+        bf.copy_and_rebind(other, mode='rb+')
+
+        self.assertTrue(other.exists())
+        self.assertEqual(before_compressed, bf.is_compressed)
+
+        if bf.is_compressed:
+            self.assertNotEqual(bf.filepath, bf.raw_filepath)
+        else:
+            self.assertEqual(bf.filepath, bf.raw_filepath)
+
+        self.assertNotEqual(before_filepath, bf.filepath)
+        self.assertNotEqual(before_raw_fp, bf.raw_filepath)
+        self.assertEqual(other, bf.filepath)
+        self.assertIs(before_blocks, bf.blocks)
+        self.assertNotIn(infile, blendfile._cached_bfiles)
+        self.assertIs(bf, blendfile._cached_bfiles[other])
+
+        self.assertEqual(str(bf.raw_filepath), bf.fileobj.name)
@@ -15,11 +15,16 @@ class AbstractPackTest(AbstractBlendFileTest):
    def setUpClass(cls):
        super().setUpClass()
        logging.getLogger('blender_asset_tracer.pack').setLevel(logging.DEBUG)
+        logging.getLogger('blender_asset_tracer.blendfile.open_cached').setLevel(logging.DEBUG)
+        logging.getLogger('blender_asset_tracer.blendfile.open_cached').setLevel(logging.DEBUG)
+        logging.getLogger('blender_asset_tracer.blendfile.BlendFile').setLevel(logging.DEBUG)

    def setUp(self):
        super().setUp()
-        self.tdir = tempfile.TemporaryDirectory()
+        self.tdir = tempfile.TemporaryDirectory(suffix='-packtest')
        self.tpath = pathlib.Path(self.tdir.name)
+        # self.tpath = pathlib.Path('/tmp/tempdir-packtest')
+        # self.tpath.mkdir(parents=True, exist_ok=True)

    def tearDown(self):
        self.tdir.cleanup()
@@ -112,7 +117,33 @@ class AbstractPackTest(AbstractBlendFileTest):
        self.assertEqual(b'LILib.002', rw_dbllink[1].block_name)
        self.assertEqual(b'//../material_textures.blend', rw_dbllink[1].asset_path)

+    def test_execute_rewrite_no_touch_origs(self):
+        infile = self._pack_with_rewrite()
+
+        # The original file shouldn't be touched.
+        bfile = blendfile.open_cached(infile, assert_cached=False)
+        libs = sorted(bfile.code_index[b'LI'])
+
+        self.assertEqual(b'LILib', libs[0].id_name)
+        self.assertEqual(b'//../linked_cube.blend', libs[0][b'name'])
+        self.assertEqual(b'LILib.002', libs[1].id_name)
+        self.assertEqual(b'//../material_textures.blend', libs[1][b'name'])
+
    def test_execute_rewrite(self):
+        infile = self._pack_with_rewrite()
+
+        extpath = pathlib.Path('//_outside_project', *self.blendfiles.parts[1:])
+        extbpath = bpathlib.BlendPath(extpath)
+
+        # Those libraries should be properly rewritten.
+        bfile = blendfile.open_cached(self.tpath / infile.name, assert_cached=False)
+        libs = sorted(bfile.code_index[b'LI'])
+        self.assertEqual(b'LILib', libs[0].id_name)
+        self.assertEqual(extbpath / b'linked_cube.blend', libs[0][b'name'])
+        self.assertEqual(b'LILib.002', libs[1].id_name)
+        self.assertEqual(extbpath / b'material_textures.blend', libs[1][b'name'])
+
+    def _pack_with_rewrite(self):
        ppath = self.blendfiles / 'subdir'
        infile = ppath / 'doubly_linked_up.blend'

@@ -120,25 +151,7 @@ class AbstractPackTest(AbstractBlendFileTest):
        packer.strategise()
        packer.execute()

-        extpath = pathlib.Path('//_outside_project', *self.blendfiles.parts[1:])
-        extbpath = bpathlib.BlendPath(extpath)
-
-        # Those libraries should be properly rewritten.
-        bfile = blendfile.open_cached(self.tpath / infile.name)
-        libs = sorted(bfile.code_index[b'LI'])
-        self.assertEqual(b'LILib', libs[0].id_name)
-        self.assertEqual(extbpath / b'linked_cube.blend', libs[0][b'name'])
-        self.assertEqual(b'LILib.002', libs[1].id_name)
-        self.assertEqual(extbpath / b'material_textures.blend', libs[1][b'name'])
-
-        # The original file shouldn't be touched, though.
-        bfile = blendfile.open_cached(infile)
-        libs = sorted(bfile.code_index[b'LI'])
-
-        self.assertEqual(b'LILib', libs[0].id_name)
-        self.assertEqual(b'//../linked_cube.blend', libs[0][b'name'])
-        self.assertEqual(b'LILib.002', libs[1].id_name)
-        self.assertEqual(b'//../material_textures.blend', libs[1][b'name'])
+        return infile

    def test_noop(self):
        ppath = self.blendfiles / 'subdir'