From dfa07e19cc3522ed7ff743220c686cbb40c06ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Tue, 5 Jun 2018 10:33:00 +0200 Subject: [PATCH] Fix T55007: Support fluid simulation cache files This also adds support for entire directory paths to be assets, in addition to globs and numbered file sequences. --- blender_asset_tracer/cdefs.py | 1 + blender_asset_tracer/pack/filesystem.py | 129 +++++++++++++++--- blender_asset_tracer/trace/file_sequence.py | 3 +- .../trace/modifier_walkers.py | 67 ++++++--- tests/test_pack_progress.py | 2 +- 5 files changed, 166 insertions(+), 36 deletions(-) diff --git a/blender_asset_tracer/cdefs.py b/blender_asset_tracer/cdefs.py index 16e1243..3fc9948 100644 --- a/blender_asset_tracer/cdefs.py +++ b/blender_asset_tracer/cdefs.py @@ -38,6 +38,7 @@ eModifierType_Wave = 7 eModifierType_Displace = 14 eModifierType_UVProject = 15 eModifierType_ParticleSystem = 19 +eModifierType_Fluidsim = 26 eModifierType_WeightVGEdit = 36 eModifierType_WeightVGMix = 37 eModifierType_WeightVGProximity = 38 diff --git a/blender_asset_tracer/pack/filesystem.py b/blender_asset_tracer/pack/filesystem.py index b059ec2..c27528e 100644 --- a/blender_asset_tracer/pack/filesystem.py +++ b/blender_asset_tracer/pack/filesystem.py @@ -18,8 +18,9 @@ # # (c) 2018, Blender Foundation - Sybren A. Stüvel import logging -import threading +import pathlib import shutil +import typing from . import transfer @@ -29,13 +30,20 @@ log = logging.getLogger(__name__) class FileCopier(transfer.FileTransferer): """Copies or moves files in source directory order.""" - def run(self) -> None: - files_transferred = 0 - files_skipped = 0 + def __init__(self): + super().__init__() + self.files_transferred = 0 + self.files_skipped = 0 + self.already_copied = set() + def run(self) -> None: + + # (is_dir, action) transfer_funcs = { - transfer.Action.COPY: shutil.copy, - transfer.Action.MOVE: shutil.move, + (False, transfer.Action.COPY): self.copyfile, + (True, transfer.Action.COPY): self.copytree, + (False, transfer.Action.MOVE): self.move, + (True, transfer.Action.MOVE): self.move, } for src, dst, act in self.iter_queue(): @@ -48,18 +56,14 @@ class FileCopier(transfer.FileTransferer): if act == transfer.Action.MOVE: log.debug('Deleting %s', src) src.unlink() - files_skipped += 1 + self.files_skipped += 1 continue log.info('%s %s → %s', act.name, src, dst) dst.parent.mkdir(parents=True, exist_ok=True) - # TODO(Sybren): when we target Py 3.6+, remove the str() calls. - tfunc = transfer_funcs[act] - tfunc(str(src), str(dst)) # type: ignore - self.report_transferred(st_src.st_size) - - files_transferred += 1 + tfunc = transfer_funcs[src.is_dir(), act] + tfunc(src, dst) # type: ignore except Exception: # We have to catch exceptions in a broad way, as this is running in # a separate thread, and exceptions won't otherwise be seen. @@ -72,7 +76,98 @@ class FileCopier(transfer.FileTransferer): self._error.set() break - if files_transferred: - log.info('Transferred %d files', files_transferred) - if files_skipped: - log.info('Skipped %d files', files_skipped) + if self.files_transferred: + log.info('Transferred %d files', self.files_transferred) + if self.files_skipped: + log.info('Skipped %d files', self.files_skipped) + + def move(self, srcpath: pathlib.Path, dstpath: pathlib.Path): + shutil.move(str(srcpath), str(dstpath)) + + def copyfile(self, srcpath: pathlib.Path, dstpath: pathlib.Path): + """Copy a file, skipping when it already exists.""" + + if self._abort.is_set() or self._error.is_set(): + return + + if (srcpath, dstpath) in self.already_copied: + log.debug('SKIP %s; already copied', srcpath) + return + + s_stat = srcpath.stat() # must exist, or it wouldn't be queued. + if dstpath.exists(): + d_stat = dstpath.stat() + if d_stat.st_size == s_stat.st_size and d_stat.st_mtime >= s_stat.st_mtime: + log.info('SKIP %s; already exists', srcpath) + self.files_skipped += 1 + return + + log.debug('Copying %s → %s', srcpath, dstpath) + shutil.copy2(str(srcpath), str(dstpath)) + + self.already_copied.add((srcpath, dstpath)) + self.files_transferred += 1 + + self.report_transferred(s_stat.st_size) + + def copytree(self, src: pathlib.Path, dst: pathlib.Path, + symlinks=False, ignore_dangling_symlinks=False): + """Recursively copy a directory tree. + + Copy of shutil.copytree() with some changes: + + - Using pathlib + - The destination directory may already exist. + - Existing files with the same file size are skipped. + - Removed ability to ignore things. + """ + + if (src, dst) in self.already_copied: + log.debug('SKIP %s; already copied', src) + return + + dst.mkdir(parents=True, exist_ok=True) + errors = [] # type: typing.List[typing.Tuple[pathlib.Path, pathlib.Path, str]] + for srcpath in src.iterdir(): + dstpath = dst / srcpath.name + try: + if srcpath.is_symlink(): + linkto = srcpath.resolve() + if symlinks: + # We can't just leave it to `copy_function` because legacy + # code with a custom `copy_function` may rely on copytree + # doing the right thing. + linkto.symlink_to(dstpath) + shutil.copystat(str(srcpath), str(dstpath), follow_symlinks=not symlinks) + else: + # ignore dangling symlink if the flag is on + if not linkto.exists() and ignore_dangling_symlinks: + continue + # otherwise let the copy occurs. copy2 will raise an error + if srcpath.is_dir(): + self.copytree(srcpath, dstpath, symlinks) + else: + self.copyfile(srcpath, dstpath) + elif srcpath.is_dir(): + self.copytree(srcpath, dstpath, symlinks) + else: + # Will raise a SpecialFileError for unsupported file types + self.copyfile(srcpath, dstpath) + # catch the Error from the recursive copytree so that we can + # continue with other files + except shutil.Error as err: + errors.extend(err.args[0]) + except OSError as why: + errors.append((srcpath, dstpath, str(why))) + try: + shutil.copystat(str(src), str(dst)) + except OSError as why: + # Copying file access times may fail on Windows + if getattr(why, 'winerror', None) is None: + errors.append((src, dst, str(why))) + if errors: + raise shutil.Error(errors) + + self.already_copied.add((src, dst)) + + return dst diff --git a/blender_asset_tracer/trace/file_sequence.py b/blender_asset_tracer/trace/file_sequence.py index 26dfd5c..2c61ba3 100644 --- a/blender_asset_tracer/trace/file_sequence.py +++ b/blender_asset_tracer/trace/file_sequence.py @@ -50,7 +50,8 @@ def expand_sequence(path: pathlib.Path) -> typing.Iterator[pathlib.Path]: raise DoesNotExist(path) if path.is_dir(): - raise TypeError('path is a directory: %s' % path) + yield path + return log.debug('expanding file sequence %s', path) diff --git a/blender_asset_tracer/trace/modifier_walkers.py b/blender_asset_tracer/trace/modifier_walkers.py index 075ff5a..9273ff8 100644 --- a/blender_asset_tracer/trace/modifier_walkers.py +++ b/blender_asset_tracer/trace/modifier_walkers.py @@ -22,11 +22,13 @@ The modifier_xxx() functions all yield result.BlockUsage objects for external files used by the modifiers. """ +import logging import typing from blender_asset_tracer import blendfile, bpathlib, cdefs from . import result +log = logging.getLogger(__name__) modifier_handlers = {} # type: typing.Dict[int, typing.Callable] @@ -112,6 +114,28 @@ def modifier_image(modifier: blendfile.BlendFileBlock, block_name: bytes) \ yield from _get_image(b'image', modifier, block_name) +def _walk_point_cache(block_name: bytes, + bfile: blendfile.BlendFile, + pointcache: blendfile.BlendFileBlock): + flag = pointcache[b'flag'] + if flag & cdefs.PTCACHE_DISK_CACHE: + # See ptcache_path() in pointcache.c + name, field = pointcache.get(b'name', return_field=True) + path = b'//%b%b/%b_*%b' % ( + cdefs.PTCACHE_PATH, + bfile.filepath.stem.encode(), + name, + cdefs.PTCACHE_EXT) + bpath = bpathlib.BlendPath(path) + yield result.BlockUsage(pointcache, bpath, path_full_field=field, + is_sequence=True, block_name=block_name) + if flag & cdefs.PTCACHE_EXTERNAL: + path, field = pointcache.get(b'path', return_field=True) + bpath = bpathlib.BlendPath(path) + yield result.BlockUsage(pointcache, bpath, path_full_field=field, + is_sequence=True, block_name=block_name) + + @mod_handler(cdefs.eModifierType_ParticleSystem) def modifier_particle_system(modifier: blendfile.BlendFileBlock, block_name: bytes) \ -> typing.Iterator[result.BlockUsage]: @@ -123,22 +147,31 @@ def modifier_particle_system(modifier: blendfile.BlendFileBlock, block_name: byt if pointcache is None: return - flag = pointcache[b'flag'] + yield from _walk_point_cache(block_name, modifier.bfile, pointcache) - if flag & cdefs.PTCACHE_DISK_CACHE: - # See ptcache_path() in pointcache.c - name, field = pointcache.get(b'name', return_field=True) - path = b'//%b%b/%b_*%b' % ( - cdefs.PTCACHE_PATH, - modifier.bfile.filepath.stem.encode(), - name, - cdefs.PTCACHE_EXT) - bpath = bpathlib.BlendPath(path) - yield result.BlockUsage(pointcache, bpath, path_full_field=field, - is_sequence=True, block_name=block_name) - if flag & cdefs.PTCACHE_EXTERNAL: - path, field = pointcache.get(b'path', return_field=True) - bpath = bpathlib.BlendPath(path) - yield result.BlockUsage(pointcache, bpath, path_full_field=field, - is_sequence=True, block_name=block_name) +@mod_handler(cdefs.eModifierType_Fluidsim) +def modifier_fluid_sim(modifier: blendfile.BlendFileBlock, block_name: bytes) \ + -> typing.Iterator[result.BlockUsage]: + my_log = log.getChild('modifier_fluid_sim') + + fss = modifier.get_pointer(b'fss') + if fss is None: + my_log.debug('Modifier %r (%r) has no fss', + modifier[b'modifier', b'name'], block_name) + return + + path, field = fss.get(b'surfdataPath', return_field=True) + + # This may match more than is used by Blender, but at least it shouldn't + # miss any files. + # The 'fluidsurface' prefix is defined in source/blender/makesdna/DNA_object_fluidsim_types.h + bpath = bpathlib.BlendPath(path) + yield result.BlockUsage(fss, bpath, path_full_field=field, + is_sequence=True, block_name=block_name) + + # TODO(Sybren): check whether this is actually used + # (in Blender's source there is a point_cache pointer, but it's NULL in my test) + pointcache = modifier.get_pointer(b'point_cache') + if pointcache: + yield from _walk_point_cache(block_name, modifier.bfile, pointcache) diff --git a/tests/test_pack_progress.py b/tests/test_pack_progress.py index 21e889a..9d9e41e 100644 --- a/tests/test_pack_progress.py +++ b/tests/test_pack_progress.py @@ -15,7 +15,7 @@ class ThreadedProgressTest(unittest.TestCase): # Flushing an empty queue should be fast. before = time.time() - tscb.flush(timeout=1) + tscb.flush() duration = time.time() - before self.assertLess(duration, 1)