autour_de_minuit/blender-asset-tracer
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Fix T55007: Support fluid simulation cache files

Browse Source 
This also adds support for entire directory paths to be assets, in
addition to globs and numbered file sequences.
This commit is contained in:
Sybren A. Stüvel 2018-06-05 10:33:00 +02:00
parent fc144138d0
commit dfa07e19cc
5 changed files with 166 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
blender_asset_tracer/cdefs.py
View File

@ -38,6 +38,7 @@ eModifierType_Wave = 7
eModifierType_Displace = 14
eModifierType_UVProject = 15
eModifierType_ParticleSystem = 19
eModifierType_Fluidsim = 26
eModifierType_WeightVGEdit = 36
eModifierType_WeightVGMix = 37
eModifierType_WeightVGProximity = 38

129
blender_asset_tracer/pack/filesystem.py
View File

@ -18,8 +18,9 @@
#
# (c) 2018, Blender Foundation - Sybren A. Stüvel
import logging
import threading
import pathlib
import shutil
import typing
from . import transfer
@ -29,13 +30,20 @@ log = logging.getLogger(__name__)
class FileCopier(transfer.FileTransferer):
"""Copies or moves files in source directory order."""
def run(self) -> None:
files_transferred = 0
files_skipped = 0
def __init__(self):
super().__init__()
self.files_transferred = 0
self.files_skipped = 0
self.already_copied = set()
def run(self) -> None:
# (is_dir, action)
transfer_funcs = {
transfer.Action.COPY: shutil.copy,
transfer.Action.MOVE: shutil.move,
(False, transfer.Action.COPY): self.copyfile,
(True, transfer.Action.COPY): self.copytree,
(False, transfer.Action.MOVE): self.move,
(True, transfer.Action.MOVE): self.move,
}
for src, dst, act in self.iter_queue():
@ -48,18 +56,14 @@ class FileCopier(transfer.FileTransferer):
if act == transfer.Action.MOVE:
log.debug('Deleting %s', src)
src.unlink()
files_skipped += 1
self.files_skipped += 1
continue
log.info('%s %s → %s', act.name, src, dst)
dst.parent.mkdir(parents=True, exist_ok=True)
# TODO(Sybren): when we target Py 3.6+, remove the str() calls.
tfunc = transfer_funcs[act]
tfunc(str(src), str(dst)) # type: ignore
self.report_transferred(st_src.st_size)
files_transferred += 1
tfunc = transfer_funcs[src.is_dir(), act]
tfunc(src, dst) # type: ignore
except Exception:
# We have to catch exceptions in a broad way, as this is running in
# a separate thread, and exceptions won't otherwise be seen.
@ -72,7 +76,98 @@ class FileCopier(transfer.FileTransferer):
self._error.set()
break
if files_transferred:
log.info('Transferred %d files', files_transferred)
if files_skipped:
log.info('Skipped %d files', files_skipped)
if self.files_transferred:
log.info('Transferred %d files', self.files_transferred)
if self.files_skipped:
log.info('Skipped %d files', self.files_skipped)
def move(self, srcpath: pathlib.Path, dstpath: pathlib.Path):
shutil.move(str(srcpath), str(dstpath))
def copyfile(self, srcpath: pathlib.Path, dstpath: pathlib.Path):
"""Copy a file, skipping when it already exists."""
if self._abort.is_set() or self._error.is_set():
return
if (srcpath, dstpath) in self.already_copied:
log.debug('SKIP %s; already copied', srcpath)
return
s_stat = srcpath.stat() # must exist, or it wouldn't be queued.
if dstpath.exists():
d_stat = dstpath.stat()
if d_stat.st_size == s_stat.st_size and d_stat.st_mtime >= s_stat.st_mtime:
log.info('SKIP %s; already exists', srcpath)
self.files_skipped += 1
return
log.debug('Copying %s → %s', srcpath, dstpath)
shutil.copy2(str(srcpath), str(dstpath))
self.already_copied.add((srcpath, dstpath))
self.files_transferred += 1
self.report_transferred(s_stat.st_size)
def copytree(self, src: pathlib.Path, dst: pathlib.Path,
symlinks=False, ignore_dangling_symlinks=False):
"""Recursively copy a directory tree.
Copy of shutil.copytree() with some changes:
- Using pathlib
- The destination directory may already exist.
- Existing files with the same file size are skipped.
- Removed ability to ignore things.
"""
if (src, dst) in self.already_copied:
log.debug('SKIP %s; already copied', src)
return
dst.mkdir(parents=True, exist_ok=True)
errors = [] # type: typing.List[typing.Tuple[pathlib.Path, pathlib.Path, str]]
for srcpath in src.iterdir():
dstpath = dst / srcpath.name
try:
if srcpath.is_symlink():
linkto = srcpath.resolve()
if symlinks:
# We can't just leave it to `copy_function` because legacy
# code with a custom `copy_function` may rely on copytree
# doing the right thing.
linkto.symlink_to(dstpath)
shutil.copystat(str(srcpath), str(dstpath), follow_symlinks=not symlinks)
else:
# ignore dangling symlink if the flag is on
if not linkto.exists() and ignore_dangling_symlinks:
continue
# otherwise let the copy occurs. copy2 will raise an error
if srcpath.is_dir():
self.copytree(srcpath, dstpath, symlinks)
else:
self.copyfile(srcpath, dstpath)
elif srcpath.is_dir():
self.copytree(srcpath, dstpath, symlinks)
else:
# Will raise a SpecialFileError for unsupported file types
self.copyfile(srcpath, dstpath)
# catch the Error from the recursive copytree so that we can
# continue with other files
except shutil.Error as err:
errors.extend(err.args[0])
except OSError as why:
errors.append((srcpath, dstpath, str(why)))
try:
shutil.copystat(str(src), str(dst))
except OSError as why:
# Copying file access times may fail on Windows
if getattr(why, 'winerror', None) is None:
errors.append((src, dst, str(why)))
if errors:
raise shutil.Error(errors)
self.already_copied.add((src, dst))
return dst

3
blender_asset_tracer/trace/file_sequence.py
View File

@ -50,7 +50,8 @@ def expand_sequence(path: pathlib.Path) -> typing.Iterator[pathlib.Path]:
raise DoesNotExist(path)
if path.is_dir():
raise TypeError('path is a directory: %s' % path)
yield path
return
log.debug('expanding file sequence %s', path)

63
blender_asset_tracer/trace/modifier_walkers.py
View File

@ -22,11 +22,13 @@
The modifier_xxx() functions all yield result.BlockUsage objects for external
files used by the modifiers.
"""
import logging
import typing
from blender_asset_tracer import blendfile, bpathlib, cdefs
from . import result
log = logging.getLogger(__name__)
modifier_handlers = {} # type: typing.Dict[int, typing.Callable]
@ -112,6 +114,28 @@ def modifier_image(modifier: blendfile.BlendFileBlock, block_name: bytes) \
yield from _get_image(b'image', modifier, block_name)
def _walk_point_cache(block_name: bytes,
bfile: blendfile.BlendFile,
pointcache: blendfile.BlendFileBlock):
flag = pointcache[b'flag']
if flag & cdefs.PTCACHE_DISK_CACHE:
# See ptcache_path() in pointcache.c
name, field = pointcache.get(b'name', return_field=True)
path = b'//%b%b/%b_*%b' % (
cdefs.PTCACHE_PATH,
bfile.filepath.stem.encode(),
name,
cdefs.PTCACHE_EXT)
bpath = bpathlib.BlendPath(path)
yield result.BlockUsage(pointcache, bpath, path_full_field=field,
is_sequence=True, block_name=block_name)
if flag & cdefs.PTCACHE_EXTERNAL:
path, field = pointcache.get(b'path', return_field=True)
bpath = bpathlib.BlendPath(path)
yield result.BlockUsage(pointcache, bpath, path_full_field=field,
is_sequence=True, block_name=block_name)
@mod_handler(cdefs.eModifierType_ParticleSystem)
def modifier_particle_system(modifier: blendfile.BlendFileBlock, block_name: bytes) \
-> typing.Iterator[result.BlockUsage]:
@ -123,22 +147,31 @@ def modifier_particle_system(modifier: blendfile.BlendFileBlock, block_name: byt
if pointcache is None:
return
flag = pointcache[b'flag']
yield from _walk_point_cache(block_name, modifier.bfile, pointcache)
if flag & cdefs.PTCACHE_DISK_CACHE:
# See ptcache_path() in pointcache.c
name, field = pointcache.get(b'name', return_field=True)
path = b'//%b%b/%b_*%b' % (
cdefs.PTCACHE_PATH,
modifier.bfile.filepath.stem.encode(),
name,
cdefs.PTCACHE_EXT)
@mod_handler(cdefs.eModifierType_Fluidsim)
def modifier_fluid_sim(modifier: blendfile.BlendFileBlock, block_name: bytes) \
-> typing.Iterator[result.BlockUsage]:
my_log = log.getChild('modifier_fluid_sim')
fss = modifier.get_pointer(b'fss')
if fss is None:
my_log.debug('Modifier %r (%r) has no fss',
modifier[b'modifier', b'name'], block_name)
return
path, field = fss.get(b'surfdataPath', return_field=True)
# This may match more than is used by Blender, but at least it shouldn't
# miss any files.
# The 'fluidsurface' prefix is defined in source/blender/makesdna/DNA_object_fluidsim_types.h
bpath = bpathlib.BlendPath(path)
yield result.BlockUsage(pointcache, bpath, path_full_field=field,
yield result.BlockUsage(fss, bpath, path_full_field=field,
is_sequence=True, block_name=block_name)
if flag & cdefs.PTCACHE_EXTERNAL:
path, field = pointcache.get(b'path', return_field=True)
bpath = bpathlib.BlendPath(path)
yield result.BlockUsage(pointcache, bpath, path_full_field=field,
is_sequence=True, block_name=block_name)
# TODO(Sybren): check whether this is actually used
# (in Blender's source there is a point_cache pointer, but it's NULL in my test)
pointcache = modifier.get_pointer(b'point_cache')
if pointcache:
yield from _walk_point_cache(block_name, modifier.bfile, pointcache)

2
tests/test_pack_progress.py
View File

@ -15,7 +15,7 @@ class ThreadedProgressTest(unittest.TestCase):
# Flushing an empty queue should be fast.
before = time.time()
tscb.flush(timeout=1)
tscb.flush()
duration = time.time() - before
self.assertLess(duration, 1)