Fix T55007: Support fluid simulation cache files

This also adds support for entire directory paths to be assets, in
addition to globs and numbered file sequences.

blender_asset_tracer/cdefs.py

@@ -38,6 +38,7 @@ eModifierType_Wave = 7
 eModifierType_Displace = 14
 eModifierType_UVProject = 15
 eModifierType_ParticleSystem = 19
+eModifierType_Fluidsim = 26
 eModifierType_WeightVGEdit = 36
 eModifierType_WeightVGMix = 37
 eModifierType_WeightVGProximity = 38

blender_asset_tracer/pack/filesystem.py

@@ -18,8 +18,9 @@
 #
 # (c) 2018, Blender Foundation - Sybren A. Stüvel
 import logging
-import threading
+import pathlib
 import shutil
+import typing
 
 from . import transfer
 
@@ -29,13 +30,20 @@ log = logging.getLogger(__name__)
 class FileCopier(transfer.FileTransferer):
     """Copies or moves files in source directory order."""
 
-    def run(self) -> None:
+    def __init__(self):
-        files_transferred = 0
+        super().__init__()
-        files_skipped = 0
+        self.files_transferred = 0
+        self.files_skipped = 0
+        self.already_copied = set()
 
+    def run(self) -> None:
+
+        # (is_dir, action)
         transfer_funcs = {
-            transfer.Action.COPY: shutil.copy,
+            (False, transfer.Action.COPY): self.copyfile,
-            transfer.Action.MOVE: shutil.move,
+            (True, transfer.Action.COPY): self.copytree,
+            (False, transfer.Action.MOVE): self.move,
+            (True, transfer.Action.MOVE): self.move,
         }
 
         for src, dst, act in self.iter_queue():
@@ -48,18 +56,14 @@ class FileCopier(transfer.FileTransferer):
                         if act == transfer.Action.MOVE:
                             log.debug('Deleting %s', src)
                             src.unlink()
-                        files_skipped += 1
+                        self.files_skipped += 1
                         continue
 
                 log.info('%s %s → %s', act.name, src, dst)
                 dst.parent.mkdir(parents=True, exist_ok=True)
 
-                # TODO(Sybren): when we target Py 3.6+, remove the str() calls.
+                tfunc = transfer_funcs[src.is_dir(), act]
-                tfunc = transfer_funcs[act]
+                tfunc(src, dst)  # type: ignore
-                tfunc(str(src), str(dst))  # type: ignore
-                self.report_transferred(st_src.st_size)
-
-                files_transferred += 1
             except Exception:
                 # We have to catch exceptions in a broad way, as this is running in
                 # a separate thread, and exceptions won't otherwise be seen.
@@ -72,7 +76,98 @@ class FileCopier(transfer.FileTransferer):
                 self._error.set()
                 break
 
-        if files_transferred:
+        if self.files_transferred:
-            log.info('Transferred %d files', files_transferred)
+            log.info('Transferred %d files', self.files_transferred)
-        if files_skipped:
+        if self.files_skipped:
-            log.info('Skipped %d files', files_skipped)
+            log.info('Skipped %d files', self.files_skipped)
+
+    def move(self, srcpath: pathlib.Path, dstpath: pathlib.Path):
+        shutil.move(str(srcpath), str(dstpath))
+
+    def copyfile(self, srcpath: pathlib.Path, dstpath: pathlib.Path):
+        """Copy a file, skipping when it already exists."""
+
+        if self._abort.is_set() or self._error.is_set():
+            return
+
+        if (srcpath, dstpath) in self.already_copied:
+            log.debug('SKIP %s; already copied', srcpath)
+            return
+
+        s_stat = srcpath.stat()  # must exist, or it wouldn't be queued.
+        if dstpath.exists():
+            d_stat = dstpath.stat()
+            if d_stat.st_size == s_stat.st_size and d_stat.st_mtime >= s_stat.st_mtime:
+                log.info('SKIP %s; already exists', srcpath)
+                self.files_skipped += 1
+                return
+
+        log.debug('Copying %s → %s', srcpath, dstpath)
+        shutil.copy2(str(srcpath), str(dstpath))
+
+        self.already_copied.add((srcpath, dstpath))
+        self.files_transferred += 1
+
+        self.report_transferred(s_stat.st_size)
+
+    def copytree(self, src: pathlib.Path, dst: pathlib.Path,
+                 symlinks=False, ignore_dangling_symlinks=False):
+        """Recursively copy a directory tree.
+
+        Copy of shutil.copytree() with some changes:
+
+        - Using pathlib
+        - The destination directory may already exist.
+        - Existing files with the same file size are skipped.
+        - Removed ability to ignore things.
+        """
+
+        if (src, dst) in self.already_copied:
+            log.debug('SKIP %s; already copied', src)
+            return
+
+        dst.mkdir(parents=True, exist_ok=True)
+        errors = []  # type: typing.List[typing.Tuple[pathlib.Path, pathlib.Path, str]]
+        for srcpath in src.iterdir():
+            dstpath = dst / srcpath.name
+            try:
+                if srcpath.is_symlink():
+                    linkto = srcpath.resolve()
+                    if symlinks:
+                        # We can't just leave it to `copy_function` because legacy
+                        # code with a custom `copy_function` may rely on copytree
+                        # doing the right thing.
+                        linkto.symlink_to(dstpath)
+                        shutil.copystat(str(srcpath), str(dstpath), follow_symlinks=not symlinks)
+                    else:
+                        # ignore dangling symlink if the flag is on
+                        if not linkto.exists() and ignore_dangling_symlinks:
+                            continue
+                        # otherwise let the copy occurs. copy2 will raise an error
+                        if srcpath.is_dir():
+                            self.copytree(srcpath, dstpath, symlinks)
+                        else:
+                            self.copyfile(srcpath, dstpath)
+                elif srcpath.is_dir():
+                    self.copytree(srcpath, dstpath, symlinks)
+                else:
+                    # Will raise a SpecialFileError for unsupported file types
+                    self.copyfile(srcpath, dstpath)
+            # catch the Error from the recursive copytree so that we can
+            # continue with other files
+            except shutil.Error as err:
+                errors.extend(err.args[0])
+            except OSError as why:
+                errors.append((srcpath, dstpath, str(why)))
+        try:
+            shutil.copystat(str(src), str(dst))
+        except OSError as why:
+            # Copying file access times may fail on Windows
+            if getattr(why, 'winerror', None) is None:
+                errors.append((src, dst, str(why)))
+        if errors:
+            raise shutil.Error(errors)
+
+        self.already_copied.add((src, dst))
+
+        return dst

blender_asset_tracer/trace/file_sequence.py

@@ -50,7 +50,8 @@ def expand_sequence(path: pathlib.Path) -> typing.Iterator[pathlib.Path]:
         raise DoesNotExist(path)
 
     if path.is_dir():
-        raise TypeError('path is a directory: %s' % path)
+        yield path
+        return
 
     log.debug('expanding file sequence %s', path)
 

blender_asset_tracer/trace/modifier_walkers.py

@@ -22,11 +22,13 @@
 The modifier_xxx() functions all yield result.BlockUsage objects for external
 files used by the modifiers.
 """
+import logging
 import typing
 
 from blender_asset_tracer import blendfile, bpathlib, cdefs
 from . import result
 
+log = logging.getLogger(__name__)
 modifier_handlers = {}  # type: typing.Dict[int, typing.Callable]
 
 
@@ -112,6 +114,28 @@ def modifier_image(modifier: blendfile.BlendFileBlock, block_name: bytes) \
     yield from _get_image(b'image', modifier, block_name)
 
 
+def _walk_point_cache(block_name: bytes,
+                      bfile: blendfile.BlendFile,
+                      pointcache: blendfile.BlendFileBlock):
+    flag = pointcache[b'flag']
+    if flag & cdefs.PTCACHE_DISK_CACHE:
+        # See ptcache_path() in pointcache.c
+        name, field = pointcache.get(b'name', return_field=True)
+        path = b'//%b%b/%b_*%b' % (
+            cdefs.PTCACHE_PATH,
+            bfile.filepath.stem.encode(),
+            name,
+            cdefs.PTCACHE_EXT)
+        bpath = bpathlib.BlendPath(path)
+        yield result.BlockUsage(pointcache, bpath, path_full_field=field,
+                                is_sequence=True, block_name=block_name)
+    if flag & cdefs.PTCACHE_EXTERNAL:
+        path, field = pointcache.get(b'path', return_field=True)
+        bpath = bpathlib.BlendPath(path)
+        yield result.BlockUsage(pointcache, bpath, path_full_field=field,
+                                is_sequence=True, block_name=block_name)
+
+
 @mod_handler(cdefs.eModifierType_ParticleSystem)
 def modifier_particle_system(modifier: blendfile.BlendFileBlock, block_name: bytes) \
         -> typing.Iterator[result.BlockUsage]:
@@ -123,22 +147,31 @@ def modifier_particle_system(modifier: blendfile.BlendFileBlock, block_name: byt
     if pointcache is None:
         return
 
-    flag = pointcache[b'flag']
+    yield from _walk_point_cache(block_name, modifier.bfile, pointcache)
 
-    if flag & cdefs.PTCACHE_DISK_CACHE:
+
-        # See ptcache_path() in pointcache.c
+@mod_handler(cdefs.eModifierType_Fluidsim)
-        name, field = pointcache.get(b'name', return_field=True)
+def modifier_fluid_sim(modifier: blendfile.BlendFileBlock, block_name: bytes) \
-        path = b'//%b%b/%b_*%b' % (
+        -> typing.Iterator[result.BlockUsage]:
-            cdefs.PTCACHE_PATH,
+    my_log = log.getChild('modifier_fluid_sim')
-            modifier.bfile.filepath.stem.encode(),
+
-            name,
+    fss = modifier.get_pointer(b'fss')
-            cdefs.PTCACHE_EXT)
+    if fss is None:
+        my_log.debug('Modifier %r (%r) has no fss',
+                     modifier[b'modifier', b'name'], block_name)
+        return
+
+    path, field = fss.get(b'surfdataPath', return_field=True)
+
+    # This may match more than is used by Blender, but at least it shouldn't
+    # miss any files.
+    # The 'fluidsurface' prefix is defined in source/blender/makesdna/DNA_object_fluidsim_types.h
     bpath = bpathlib.BlendPath(path)
-        yield result.BlockUsage(pointcache, bpath, path_full_field=field,
+    yield result.BlockUsage(fss, bpath, path_full_field=field,
                             is_sequence=True, block_name=block_name)
 
-    if flag & cdefs.PTCACHE_EXTERNAL:
+    # TODO(Sybren): check whether this is actually used
-        path, field = pointcache.get(b'path', return_field=True)
+    # (in Blender's source there is a point_cache pointer, but it's NULL in my test)
-        bpath = bpathlib.BlendPath(path)
+    pointcache = modifier.get_pointer(b'point_cache')
-        yield result.BlockUsage(pointcache, bpath, path_full_field=field,
+    if pointcache:
-                                is_sequence=True, block_name=block_name)
+        yield from _walk_point_cache(block_name, modifier.bfile, pointcache)

tests/test_pack_progress.py

@@ -15,7 +15,7 @@ class ThreadedProgressTest(unittest.TestCase):
 
         # Flushing an empty queue should be fast.
         before = time.time()
-        tscb.flush(timeout=1)
+        tscb.flush()
         duration = time.time() - before
         self.assertLess(duration, 1)