finish make library conformation working

This commit is contained in:
“christopheseux”
2022-12-30 23:36:09 +01:00
parent ab72a15a2c
commit 501cb460c8
19 changed files with 319 additions and 303 deletions
+120 -91
View File
@@ -3,7 +3,7 @@
from asset_library.common.bl_utils import get_addon_prefs, load_datablocks
from asset_library.common.file_utils import read_file, write_file
from asset_library.common.template import Template
from asset_library.constants import (PREVIEW_ASSETS_SCRIPT, MODULE_DIR)
from asset_library.constants import (MODULE_DIR, RESOURCES_DIR)
from asset_library import (action, collection, file)
@@ -36,48 +36,26 @@ class AssetLibraryAdapter(PropertyGroup):
for lib in prefs.libraries:
if lib.adapter == self:
return lib
if lib.conform.adapter == self:
return lib
#@property
#def library_path(self):
# return self.library.library_path
@property
def is_conform(self):
prefs = self.addon_prefs
for lib in prefs.libraries:
if lib.adapter == self:
return False
if lib.conform.adapter == self:
return True
@property
def target_directory(self):
if self.is_conform:
return self.library.conform.directory
return self.library.bundle_directory
def bundle_directory(self):
return self.library.library_path
@property
def blend_depth(self):
if self.is_conform:
return self.library.conform.blend_depth
return self.library.blend_depth
# @property
# def blend_depth(self):
# return self.library.blend_depth
@property
def template_image(self):
return Template(self.library.conform.template_image)
# @property
# def template_image(self):
# return Template(self.library.template_image)
@property
def template_video(self):
return Template(self.library.conform.template_video)
# @property
# def template_video(self):
# return Template(self.library.template_video)
@property
def template_description(self):
return Template(self.library.conform.template_description)
# @property
# def template_description(self):
# return Template(self.library.template_description)
@property
def data_type(self):
@@ -87,21 +65,18 @@ class AssetLibraryAdapter(PropertyGroup):
def data_types(self):
return self.library.data_types
#@property
#def externalize_data(self):
# return self.library.externalize_data
#@property
#def catalog_path(self):
# return self.library.catalog_path
def get_catalog_path(self, directory=None):
directory = directory or self.target_directory
directory = directory or self.bundle_directory
return Path(directory, 'blender_assets.cats.txt')
@property
def cache_file(self):
return Path(self.target_directory) / f"blender_assets.{self.library.id}.json"
return Path(self.bundle_directory) / f"blender_assets.{self.library.id}.json"
#return get_asset_datas_file(self.library_path)
@property
def tmp_cache_file(self):
return Path(bpy.app.tempdir) / f"blender_assets.{self.library.id}.json"
#return get_asset_datas_file(self.library_path)
@property
@@ -149,22 +124,26 @@ class AssetLibraryAdapter(PropertyGroup):
src = Path(source)
dst = Path(destination)
if not source.exists():
print(f'Cannot copy file {source}: file not exist')
if not src.exists():
print(f'Cannot copy file {src}: file not exist')
return
dst.parent.mkdir(exist_ok=True, parents=True)
if src == dst:
print(f'Cannot copy file {source}: source and destination are the same')
print(f'Cannot copy file {src}: source and destination are the same')
return
print(f'Copy file from {source} to {destination}')
shutil.copy2(str(source), str(destination))
print(f'Copy file from {src} to {dst}')
shutil.copy2(str(src), str(dst))
def load_datablocks(self, src, names=None, type='objects', link=True, expr=None):
def load_datablocks(self, src, names=None, type='objects', link=True, expr=None, assets_only=False):
"""Link or append a datablock from a blendfile"""
return load_datablocks(src, names=names, type=type, link=link, expr=expr)
if type.isupper():
type = f'{type.lower()}s'
return load_datablocks(src, names=names, type=type, link=link, expr=expr, assets_only=assets_only)
def get_asset_relative_path(self, name, catalog):
'''Get a relative path for the asset'''
@@ -220,7 +199,7 @@ class AssetLibraryAdapter(PropertyGroup):
template = Path(asset_path, template).as_posix()
params = {
'name': name,
'asset_name': name,
'asset_path': Path(asset_path),
'catalog': catalog,
'catalog_name': catalog.replace('/', '_'),
@@ -230,13 +209,13 @@ class AssetLibraryAdapter(PropertyGroup):
def get_description_path(self, name, asset_path, catalog) -> Path:
""""Get the path of the json or yaml describing all assets data in one file"""
return self.get_template_path(self.library.conform.template_description, name, asset_path, catalog)
return self.get_template_path(self.library.template_description, name, asset_path, catalog)
def get_image_path(self, name, asset_path, catalog) -> Path:
return self.get_template_path(self.library.conform.template_image, name, asset_path, catalog)
return self.get_template_path(self.library.template_image, name, asset_path, catalog)
def get_video_path(self, name, asset_path, catalog) -> Path:
return self.get_template_path(self.library.conform.template_video, name, asset_path, catalog)
return self.get_template_path(self.library.template_video, name, asset_path, catalog)
'''
def get_path(self, type, name, asset_path, template=None) -> Path:
@@ -355,12 +334,13 @@ class AssetLibraryAdapter(PropertyGroup):
print(f'Catalog writen at: {catalog_path}')
catalog_path.write_text('\n'.join(lines), encoding="utf-8")
def read_cache(self):
print(f'Read cache from {self.cache_file}')
return self.read_file(self.cache_file)
def read_cache(self, cache_path=None):
cache_path = cache_path or self.cache_file
print(f'Read cache from {cache_path}')
return self.read_file(cache_path)
def write_cache(self, asset_descriptions):
cache_path = self.cache_file
def write_cache(self, asset_descriptions, cache_path=None):
cache_path = cache_path or self.cache_file
print(f'cache file writen to {cache_path}')
return write_file(cache_path, list(asset_descriptions))
@@ -408,7 +388,7 @@ class AssetLibraryAdapter(PropertyGroup):
catalog_parts = asset_data['catalog'].split('/') + [asset_data['name']]
return catalog_parts[:self.blend_depth]
return catalog_parts[:self.library.blend_depth]
#def transfert_preview(self, )
@@ -463,10 +443,44 @@ class AssetLibraryAdapter(PropertyGroup):
)
)
'''
def generate_preview(self, asset_description):
def generate_blend_preview(self, asset_description):
asset_name = asset_description['name']
catalog = asset_description['catalog']
asset_path = self.format_path(asset_description['filepath'])
dst_image_path = self.get_image_path(asset_name, asset_path, catalog)
if dst_image_path.exists():
return
# Check if a source image exists and if so copying it in the new directory
src_image_path = asset_description.get('image')
if src_image_path:
src_image_path = self.get_template_path(src_image_path, asset_name, asset_path, catalog)
if src_image_path and src_image_path.exists():
self.copy_file(src_image_path, dst_image_path)
return
print(f'Thumbnailing {asset_path} to {dst_image_path}')
blender_thumbnailer = Path(bpy.app.binary_path).parent / 'blender-thumbnailer'
dst_image_path.parent.mkdir(exist_ok=True, parents=True)
subprocess.call([blender_thumbnailer, str(asset_path), str(dst_image_path)])
success = dst_image_path.exists()
if not success:
empty_preview = RESOURCES_DIR / 'empty_preview.png'
self.copy_file(str(empty_preview), str(dst_image_path))
return success
def generate_asset_preview(self, asset_description):
"""Only generate preview when conforming a library"""
print('\ngenerate_preview', asset_description)
print('\ngenerate_preview', asset_description['filepath'])
scn = bpy.context.scene
#Creating the preview for collection, object or material
@@ -474,19 +488,28 @@ class AssetLibraryAdapter(PropertyGroup):
vl = bpy.context.view_layer
data_type = self.data_type #asset_description['data_type']
asset_path = asset_description['filepath']
asset_path = self.format_path(asset_description['filepath'])
asset_data_names = {}
for asset_data in asset_description['assets']:
name = asset_data['name']
catalog = asset_data['catalog']
image_path = self.get_image_path(name, asset_path, catalog)
if image_path.exists():
dst_image_path = self.get_image_path(name, asset_path, catalog)
if dst_image_path.exists():
continue
# Check if a source image exists and if so copying it in the new directory
src_image_path = asset_data.get('image')
if src_image_path:
src_image_path = self.get_template_path(src_image_path, name, asset_path, catalog)
if src_image_path and src_image_path.exists():
self.copy_file(src_image_path, dst_image_path)
return
#Store in a dict all asset_data that does not have preview
asset_data_names[name] = dict(asset_data, image_path=image_path)
asset_data_names[name] = dict(asset_data, image_path=dst_image_path)
if not asset_data_names:
# No preview to generate
@@ -512,8 +535,6 @@ class AssetLibraryAdapter(PropertyGroup):
scn.render.filepath = str(image_path)
print(f'Render asset {asset.name} to {image_path}')
bpy.ops.render.render(write_still=True)
@@ -522,25 +543,27 @@ class AssetLibraryAdapter(PropertyGroup):
bpy.data.objects.remove(instance)
#bpy.ops.object.delete(use_global=False)
#scn.collection.children.unlink(asset)
bpy.ops.outliner.orphans_purge(do_local_ids=True, do_linked_ids=True, do_recursive=True)
def generate_previews(self, asset_descriptions=None):
def generate_previews(self, cache=None):
print('Generate previews')
asset_descriptions = asset_descriptions or self.fetch()
if cache in (None, ''):
cache = self.fetch()
elif isinstance(cache, (Path, str)):
cache = self.read_cache(cache)
#cache_diff.sort(key=lambda x :x['filepath'])
#blend_groups = groupby(cache_diff, key=lambda x :x['filepath'])
#TODO Support all multiple data_type
for asset_description in asset_descriptions:
self.generate_preview(asset_description)
for asset_description in cache:
if asset_description['type'] == 'FILE':
self.generate_blend_preview(asset_description)
else:
self.generate_asset_preview(asset_description)
# filepath = asset_description['filepath']
@@ -561,7 +584,11 @@ class AssetLibraryAdapter(PropertyGroup):
asset_path = asset_data['filepath']
catalog = asset_data['catalog']
image_path = self.get_image_path(name, asset_path, catalog)
image_path = asset_data.get('image')
if self.library.template_image:
image_path = self.get_image_path(name, asset_path, catalog)
elif image_path:
image_path = self.get_template_path(image_path, name, asset_path, catalog)
if image_path and image_path.exists():
with bpy.context.temp_override(id=asset):
@@ -663,9 +690,6 @@ class AssetLibraryAdapter(PropertyGroup):
print(f'{self.data_type} is not supported yet')
return
target_dir = self.target_directory
catalog_data = self.read_catalog() #TODO remove unused catalog
write_cache = False
@@ -673,12 +697,15 @@ class AssetLibraryAdapter(PropertyGroup):
# Get list of all modifications
asset_descriptions = self.fetch()
cache, cache_diff = self.diff(asset_descriptions)
# Only write complete cache at the end
write_cache = True
self.generate_previews(asset_descriptions)
#self.generate_previews(asset_descriptions)
self.write_cache(asset_descriptions, self.tmp_cache_file)
bpy.ops.assetlib.generate_previews(name=self.library.name, cache=str(self.tmp_cache_file))
#print()
#print(cache)
@@ -688,7 +715,7 @@ class AssetLibraryAdapter(PropertyGroup):
cache_diff = json.loads(Path(cache_diff).read_text(encoding='utf-8'))
if self.blend_depth == 0:
if self.library.blend_depth == 0:
raise Exception('Blender depth must be 1 at min')
#groups = [(cache_diff)]
else:
@@ -702,11 +729,14 @@ class AssetLibraryAdapter(PropertyGroup):
print('No assets found')
return
#data_types = self.data_types
#if self.data_types == 'FILE'
i = 0
#assets_to_preview = []
for sub_path, asset_datas in groups:
blend_name = sub_path[-1].replace(' ', '_').lower()
blend_path = Path(target_dir, *sub_path, blend_name).with_suffix('.blend')
blend_path = Path(self.bundle_directory, *sub_path, blend_name).with_suffix('.blend')
if blend_path.exists():
print(f'Opening existing bundle blend: {blend_path}')
@@ -768,7 +798,6 @@ class AssetLibraryAdapter(PropertyGroup):
print(f'Saving Blend to {blend_path}')
blend_path.parent.mkdir(exist_ok=True, parents=True)
bpy.ops.wm.save_as_mainfile(filepath=str(blend_path), compress=True)
if write_cache:
@@ -847,8 +876,8 @@ class AssetLibraryAdapter(PropertyGroup):
def format_path(self, template, **kargs):
params = dict(
bundle_dir=Path(self.library.bundle_directory),
conform_dir=Path(self.library.conform.directory),
bundle_dir=Path(self.bundle_directory),
#conform_dir=Path(self.library.conform.directory),
**kargs,
**self.to_dict(),
)
+14 -2
View File
@@ -8,6 +8,7 @@ from asset_library.adapters.adapter import AssetLibraryAdapter
from asset_library.common.file_utils import copy_dir
from bpy.props import StringProperty
from os.path import expandvars
import bpy
class CopyFolderLibrary(AssetLibraryAdapter):
@@ -21,7 +22,7 @@ class CopyFolderLibrary(AssetLibraryAdapter):
def bundle(self, cache_diff=None):
src = expandvars(self.source_directory)
dst = expandvars(self.target_directory)
dst = expandvars(self.bundle_directory)
includes = [inc.strip() for inc in self.includes.split(',')]
excludes = [ex.strip() for ex in self.excludes.split(',')]
@@ -31,4 +32,15 @@ class CopyFolderLibrary(AssetLibraryAdapter):
src, dst, only_recent=True,
excludes=excludes, includes=includes
)
def filter_prop(self, prop):
if prop in ('template_description', 'template_video', 'template_image', 'blend_depth'):
return False
return True
# def draw_prop(self, layout, prop):
# if prop in ('template_description', 'template_video', 'template_image', 'blend_depth'):
# return
# super().draw_prop(layout)
+7 -15
View File
@@ -28,6 +28,7 @@ class KitsuLibrary(AssetLibraryAdapter):
template_name : StringProperty()
template_file : StringProperty()
source_directory : StringProperty(subtype='DIR_PATH')
#blend_depth: IntProperty(default=1)
url: StringProperty()
login: StringProperty()
@@ -83,8 +84,8 @@ class KitsuLibrary(AssetLibraryAdapter):
description=data['description'],
tags=[],
type=self.data_type,
image=self.template_image.raw,
video=self.template_video.raw,
image=self.library.template_image,
video=self.library.template_video,
name=data['name'])
]
)
@@ -95,16 +96,6 @@ class KitsuLibrary(AssetLibraryAdapter):
# """Group all asset in one or multiple blends for the asset browser"""
# return super().bundle(cache_diff=cache_diff)
def get_preview(self, asset_data):
name = asset_data['name']
preview = (f / template_image.format(name=name)).resolve()
if not preview.exists():
preview_blend_file(f, preview)
return preview
def fetch(self):
"""Gather in a list all assets found in the folder"""
@@ -122,11 +113,11 @@ class KitsuLibrary(AssetLibraryAdapter):
entity_types_ids = {e['id']: e['name'] for e in entity_types}
asset_descriptions = []
for asset_data in gazu.asset.all_assets_for_project(project)[:10]:
for asset_data in gazu.asset.all_assets_for_project(project):
asset_data['entity_type_name'] = entity_types_ids[asset_data.pop('entity_type_id')]
asset_name = asset_data['name']
asset_field_data = dict(name=asset_name, type=asset_data['entity_type_name'], source_directory=self.source_directory)
asset_field_data = dict(asset_name=asset_name, type=asset_data['entity_type_name'], source_directory=self.source_directory)
try:
asset_field_data.update(template_name.parse(asset_name))
@@ -139,7 +130,8 @@ class KitsuLibrary(AssetLibraryAdapter):
continue
#print(asset_path)
# TODO group when multiple asset are store in the same blend
asset_descriptions.append(self.get_asset_description(asset_data, asset_path))
#asset = load_datablocks(asset_path, data_type='collections', names=asset_data['name'], link=True)
+94 -96
View File
@@ -23,8 +23,11 @@ class ScanFolderLibrary(AssetLibraryAdapter):
name = "Scan Folder"
source_directory : StringProperty(subtype='DIR_PATH')
template : StringProperty()
blend_depth : IntProperty()
template_file : StringProperty()
template_image : StringProperty()
template_video : StringProperty()
template_description : StringProperty()
#blend_depth : IntProperty()
#externalize_preview : BoolProperty(default=True)
#def draw_header(self, layout):
@@ -38,6 +41,7 @@ class ScanFolderLibrary(AssetLibraryAdapter):
directory = directory or self.source_directory
return Path(directory, self.get_asset_relative_path(name, catalog))
'''
def get_asset_description(self, asset, catalog, modified):
asset_path = self.get_asset_relative_path(name=asset.name, catalog=catalog)
@@ -61,6 +65,38 @@ class ScanFolderLibrary(AssetLibraryAdapter):
)
return asset_description
'''
def get_asset_description(self, data, asset_path):
asset_path = self.prop_rel_path(asset_path, 'source_directory')
if self.data_type == 'FILE':
return dict(
filepath=asset_path,
modified=data['modified'],
catalog=data['catalog'],
tags=[],
type=self.data_type,
image=self.template_image,
name=data['name']
)
return dict(
filepath=asset_path,
modified=data['modified'],
library_id=self.library.id,
assets=[dict(
catalog=asset_data['catalog'],
metadata=asset_data.get('metadata', {}),
description=asset_data.get('description'),
tags=asset_data.get('tags', []),
type=self.data_type,
image=self.template_image,
video=self.template_video,
name=asset_data['name']) for asset_data in data['assets']
]
)
def _find_blend_files(self):
'''Get a sorted list of all blender files found matching the template'''
@@ -76,13 +112,14 @@ class ScanFolderLibrary(AssetLibraryAdapter):
return blend_files
'''
def _group_key(self, asset_data):
"""Group assets inside one blend"""
catalog_parts = asset_data['catalog'].split('/') + [asset_data['name']]
return catalog_parts[:self.blend_depth]
def bundle(self, cache_diff=None):
"""Group all asset in one or multiple blends for the asset browser"""
@@ -238,17 +275,9 @@ class ScanFolderLibrary(AssetLibraryAdapter):
self.write_catalog(catalog_data)
bpy.ops.wm.quit_blender()
'''
def get_preview(self, asset_data):
name = asset_data['name']
preview = (f / template_image.format(name=name)).resolve()
if not preview.exists():
preview_blend_file(f, preview)
return preview
'''
def conform(self, directory, templates):
"""Split each assets per blend and externalize preview"""
@@ -317,20 +346,20 @@ class ScanFolderLibrary(AssetLibraryAdapter):
self.copy_file(src_video_path, dst_video_path)
self.write_catalog(catalog_data, filepath=directory)
'''
def fetch(self):
"""Gather in a list all assets found in the folder"""
print(f'Fetch Assets for {self.library.name}')
source_directory = Path(os.path.expandvars(self.source_directory))
template = Template(self.template)
catalog_data = self.read_catalog(filepath=source_directory)
catalog_ids = {v['id']: {'path': k, 'name': v['name']} for k,v in catalog_data.items()}
source_directory = Path(self.source_directory)
template_file = Template(self.template_file)
catalog_data = self.read_catalog(directory=source_directory)
catalog_ids = {v['id']: k for k, v in catalog_data.items()}
cache = self.read_cache() or []
print(f'Search for blend using glob template: {template.glob_pattern}')
print(f'Search for blend using glob template: {template_file.glob_pattern}')
print(f'Scanning Folder {source_directory}...')
#blend_files = list(source_directory.glob(template.glob_pattern))
@@ -344,102 +373,71 @@ class ScanFolderLibrary(AssetLibraryAdapter):
#blend_paths = []
new_cache = []
for blend_file in template.glob(source_directory):#sorted(blend_files):
for asset_path in template_file.glob(source_directory):#sorted(blend_files):
source_rel_path = self.prop_rel_path(blend_file, 'source_directory')
modified = blend_file.stat().st_mtime_ns
source_rel_path = self.prop_rel_path(asset_path, 'source_directory')
modified = asset_path.stat().st_mtime_ns
# Check if the asset description as already been cached
asset_description = next((a for a in cache if a['filepath'] == source_rel_path), None)
if asset_description and asset_description['modified'] >= modified:
print(blend_file, 'is skipped because not modified')
print(asset_path, 'is skipped because not modified')
new_cache.append(asset_description)
continue
rel_path = blend_file.relative_to(source_directory).as_posix()
#field_values = re.findall(re_pattern, rel_path)[0]
#field_data = {k:v for k,v in zip(field_names, field_values)}
field_data = template.parse(rel_path)
if not field_data:
raise Exception()
#asset_data = (blend_file / prefs.template_description.format(name=name)).resolve()
rel_path = asset_path.relative_to(source_directory).as_posix()
field_data = template_file.parse(rel_path)
catalogs = [v for k,v in sorted(field_data.items()) if k.isdigit()]
catalogs = [c.replace('_', ' ').title() for c in catalogs]
asset_datas = {
"name": field_data['asset_name'],
"catalog": '/'.join(catalogs),
"assets": [],
'modified': modified
}
if self.data_type == 'FILE':
name = field_data.get('name', blend_file.stem)
image = self.get_path('image', name=name, asset_path=blend_file)
asset_description = dict(
filepath=source_rel_path,
modified=modified,
catalog='/'.join(catalogs),
tags=[],
type=self.data_type,
image=self.prop_rel_path(image, 'source_directory'),
name=name
)
asset_description = self.get_asset_description(asset_datas, asset_path)
new_cache.append(asset_description)
continue
#First Check if there is a asset_data .json
asset_description = self.read_asset_description_file(blend_file)
# Now check if there is a asset description file
asset_description_path = self.get_template_path(
self.template_description,
asset_datas['asset_name'],
asset_path,
asset_datas['catalog'])
if not asset_description:
# Scan the blend file for assets inside and write a custom asset description for info found
if asset_description_path and asset_description_path.exists():
new_cache.append(self.read_file(asset_description_path))
continue
print(f'Scanning blendfile {blend_file}...')
with bpy.data.libraries.load(str(blend_file), link=True, assets_only=True) as (data_from, data_to):
asset_names = getattr(data_from, self.data_types)
print(f'Found {len(asset_names)} {self.data_types} inside')
# Scan the blend file for assets inside and write a custom asset description for info found
print(f'Scanning blendfile {asset_path}...')
assets = self.load_datablocks(asset_path, type=self.data_types, link=True, assets_only=True)
print(f'Found {len(assets)} {self.data_types} inside')
setattr(data_to, self.data_types, asset_names)
assets = getattr(data_to, self.data_types)
for asset in assets:
catalog_path = catalog_ids.get(asset.asset_data.catalog_id)
asset_description = dict(
filepath=source_rel_path,
modified=modified,
assets=[]
)
if not catalog_path:
print(f'No catalog found for asset {asset.name}')
catalog_path = asset_path.relative_to(self.source_directory).as_posix()
asset_datas['assets'] += [dict(
catalog=catalog_path,
tags=asset.asset_data.tags.keys(),
metadata=dict(asset.asset_data),
type=self.data_type,
name=asset.name
)]
getattr(bpy.data, self.data_types).remove(asset)
for asset in assets:
asset_catalog_data = catalog_ids.get(asset.asset_data.catalog_id)
if not asset_catalog_data:
print(f'No catalog found for asset {asset.name}')
asset_catalog_data = {"path": blend_file.relative_to(self.source_directory).as_posix()}
catalog_path = asset_catalog_data['path']
image_path = self.get_path('image', asset.name, catalog_path)
image = self.prop_rel_path(image_path, 'source_directory')
# Write image only if no image was found
if not image_path.exists():
image_path = self.get_cache_image_path(asset.name, catalog_path)
image = self.prop_rel_path(image_path, 'library_path')
self.write_preview(asset.preview, image_path)
video_path = self.get_path('video', asset.name, catalog_path)
video = self.prop_rel_path(video_path, 'source_directory')
asset_data = dict(
filepath=self.prop_rel_path(blend_file, 'source_directory'),
modified=modified,
catalog=catalog_path,
tags=asset.asset_data.tags.keys(),
type=self.data_type,
image=image,
video=video,
name=asset.name
)
asset_description['assets'].append(asset_data)
getattr(bpy.data, self.data_types).remove(asset)
asset_description = self.get_asset_description(asset_datas, asset_path)
new_cache.append(asset_description)