Archived
initial commit
This commit is contained in:
Pullusb
@@ -0,0 +1,130 @@
|
||||
# ***** BEGIN GPL LICENSE BLOCK *****
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
#
|
||||
# ***** END GPL LICENCE BLOCK *****
|
||||
#
|
||||
# (c) 2019, Blender Foundation - Sybren A. Stüvel
|
||||
"""Shaman Client interface."""
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import typing
|
||||
import urllib.parse
|
||||
|
||||
import requests
|
||||
|
||||
import blender_asset_tracer.pack as bat_pack
|
||||
import blender_asset_tracer.pack.transfer as bat_transfer
|
||||
|
||||
from .transfer import ShamanTransferrer
|
||||
from .client import ShamanClient
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ShamanPacker(bat_pack.Packer):
|
||||
"""Creates BAT Packs on a Shaman server."""
|
||||
|
||||
def __init__(self,
|
||||
bfile: pathlib.Path,
|
||||
project: pathlib.Path,
|
||||
target: str,
|
||||
endpoint: str,
|
||||
checkout_id: str,
|
||||
**kwargs) -> None:
|
||||
"""Constructor
|
||||
|
||||
:param target: mock target '/' to construct project-relative paths.
|
||||
:param endpoint: URL of the Shaman endpoint.
|
||||
"""
|
||||
super().__init__(bfile, project, target, **kwargs)
|
||||
self.checkout_id = checkout_id
|
||||
self.shaman_endpoint = endpoint
|
||||
self._checkout_location = ''
|
||||
|
||||
def _get_auth_token(self) -> str:
|
||||
# TODO: get a token from the Flamenco Server.
|
||||
token_from_env = os.environ.get('SHAMAN_JWT_TOKEN')
|
||||
if token_from_env:
|
||||
return token_from_env
|
||||
|
||||
log.warning('Using temporary hack to get auth token from Shaman, '
|
||||
'set SHAMAN_JTW_TOKEN to prevent')
|
||||
unauth_shaman = ShamanClient('', self.shaman_endpoint)
|
||||
resp = unauth_shaman.get('get-token', timeout=10)
|
||||
resp.raise_for_status()
|
||||
return resp.text
|
||||
|
||||
def _create_file_transferer(self) -> bat_transfer.FileTransferer:
|
||||
# TODO: pass self._get_auth_token itself, so that the Transferer will be able to
|
||||
# decide when to get this token (and how many times).
|
||||
auth_token = self._get_auth_token()
|
||||
return ShamanTransferrer(auth_token, self.project, self.shaman_endpoint, self.checkout_id)
|
||||
|
||||
def _make_target_path(self, target: str) -> pathlib.PurePath:
|
||||
return pathlib.PurePosixPath('/')
|
||||
|
||||
def _on_file_transfer_finished(self, *, file_transfer_completed: bool):
|
||||
super()._on_file_transfer_finished(file_transfer_completed=file_transfer_completed)
|
||||
|
||||
assert isinstance(self._file_transferer, ShamanTransferrer)
|
||||
self._checkout_location = self._file_transferer.checkout_location
|
||||
|
||||
@property
|
||||
def checkout_location(self) -> str:
|
||||
"""Return the checkout location of the packed blend file.
|
||||
|
||||
:return: the checkout location, or '' if no checkout was made.
|
||||
"""
|
||||
return self._checkout_location
|
||||
|
||||
@property
|
||||
def output_path(self) -> pathlib.PurePath:
|
||||
"""The path of the packed blend file in the target directory."""
|
||||
assert self._output_path is not None
|
||||
|
||||
checkout_location = pathlib.PurePosixPath(self._checkout_location)
|
||||
rel_output = self._output_path.relative_to(self._target_path)
|
||||
return checkout_location / rel_output
|
||||
|
||||
def execute(self):
|
||||
try:
|
||||
super().execute()
|
||||
except requests.exceptions.ConnectionError as ex:
|
||||
log.exception('Error communicating with Shaman')
|
||||
self.abort(str(ex))
|
||||
self._check_aborted()
|
||||
|
||||
|
||||
def parse_endpoint(shaman_url: str) -> typing.Tuple[str, str]:
|
||||
"""Convert shaman://hostname/path#checkoutID into endpoint URL + checkout ID."""
|
||||
|
||||
urlparts = urllib.parse.urlparse(str(shaman_url))
|
||||
|
||||
if urlparts.scheme in {'shaman', 'shaman+https'}:
|
||||
scheme = 'https'
|
||||
elif urlparts.scheme == 'shaman+http':
|
||||
scheme = 'http'
|
||||
else:
|
||||
raise ValueError('Invalid scheme %r, choose shaman:// or shaman+http://', urlparts.scheme)
|
||||
|
||||
checkout_id = urllib.parse.unquote(urlparts.fragment)
|
||||
|
||||
path = urlparts.path or '/'
|
||||
new_urlparts = (scheme, urlparts.netloc, path, *urlparts[3:-1], '')
|
||||
endpoint = urllib.parse.urlunparse(new_urlparts)
|
||||
|
||||
return endpoint, checkout_id
|
||||
@@ -0,0 +1,197 @@
|
||||
# ***** BEGIN GPL LICENSE BLOCK *****
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
#
|
||||
# ***** END GPL LICENCE BLOCK *****
|
||||
#
|
||||
# (c) 2019, Blender Foundation - Sybren A. Stüvel
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
import typing
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
|
||||
from . import time_tracker
|
||||
|
||||
CACHE_ROOT = Path().home() / '.cache/shaman-client/shasums'
|
||||
MAX_CACHE_FILES_AGE_SECS = 3600 * 24 * 60 # 60 days
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TimeInfo:
|
||||
computing_checksums = 0.0
|
||||
checksum_cache_handling = 0.0
|
||||
|
||||
|
||||
def find_files(root: Path) -> typing.Iterable[Path]:
|
||||
"""Recursively finds files in the given root path.
|
||||
|
||||
Directories are recursed into, and file paths are yielded.
|
||||
Symlinks are yielded if they refer to a regular file.
|
||||
"""
|
||||
queue = deque([root])
|
||||
while queue:
|
||||
path = queue.popleft()
|
||||
|
||||
# Ignore hidden files/dirs; these can be things like '.svn' or '.git',
|
||||
# which shouldn't be sent to Shaman.
|
||||
if path.name.startswith('.'):
|
||||
continue
|
||||
|
||||
if path.is_dir():
|
||||
for child in path.iterdir():
|
||||
queue.append(child)
|
||||
continue
|
||||
|
||||
# Only yield symlinks if they link to (a link to) a normal file.
|
||||
if path.is_symlink():
|
||||
symlinked = path.resolve()
|
||||
if symlinked.is_file():
|
||||
yield path
|
||||
continue
|
||||
|
||||
if path.is_file():
|
||||
yield path
|
||||
|
||||
|
||||
def compute_checksum(filepath: Path) -> str:
|
||||
"""Compute the SHA256 checksum for the given file."""
|
||||
blocksize = 32 * 1024
|
||||
|
||||
log.debug('Computing checksum of %s', filepath)
|
||||
with time_tracker.track_time(TimeInfo, 'computing_checksums'):
|
||||
hasher = hashlib.sha256()
|
||||
with filepath.open('rb') as infile:
|
||||
while True:
|
||||
block = infile.read(blocksize)
|
||||
if not block:
|
||||
break
|
||||
hasher.update(block)
|
||||
checksum = hasher.hexdigest()
|
||||
return checksum
|
||||
|
||||
|
||||
def _cache_path(filepath: Path) -> Path:
|
||||
"""Compute the cache file for the given file path."""
|
||||
|
||||
fs_encoding = sys.getfilesystemencoding()
|
||||
filepath = filepath.absolute()
|
||||
|
||||
# Reverse the directory, because most variation is in the last bytes.
|
||||
rev_dir = str(filepath.parent)[::-1]
|
||||
encoded_path = filepath.stem + rev_dir + filepath.suffix
|
||||
cache_key = base64.urlsafe_b64encode(encoded_path.encode(fs_encoding)).decode().rstrip('=')
|
||||
|
||||
cache_path = CACHE_ROOT / cache_key[:10] / cache_key[10:]
|
||||
return cache_path
|
||||
|
||||
|
||||
def compute_cached_checksum(filepath: Path) -> str:
|
||||
"""Computes the SHA256 checksum.
|
||||
|
||||
The checksum is cached to disk. If the cache is still valid, it is used to
|
||||
skip the actual SHA256 computation.
|
||||
"""
|
||||
|
||||
with time_tracker.track_time(TimeInfo, 'checksum_cache_handling'):
|
||||
current_stat = filepath.stat()
|
||||
cache_path = _cache_path(filepath)
|
||||
|
||||
try:
|
||||
with cache_path.open('r') as cache_file:
|
||||
payload = json.load(cache_file)
|
||||
except (OSError, ValueError):
|
||||
# File may not exist, or have invalid contents.
|
||||
pass
|
||||
else:
|
||||
checksum = payload.get('checksum', '')
|
||||
cached_mtime = payload.get('file_mtime', 0.0)
|
||||
cached_size = payload.get('file_size', -1)
|
||||
|
||||
if (checksum
|
||||
and current_stat.st_size == cached_size
|
||||
and abs(cached_mtime - current_stat.st_mtime) < 0.01):
|
||||
cache_path.touch()
|
||||
return checksum
|
||||
|
||||
checksum = compute_checksum(filepath)
|
||||
|
||||
with time_tracker.track_time(TimeInfo, 'checksum_cache_handling'):
|
||||
payload = {
|
||||
'checksum': checksum,
|
||||
'file_mtime': current_stat.st_mtime,
|
||||
'file_size': current_stat.st_size,
|
||||
}
|
||||
|
||||
try:
|
||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with cache_path.open('w') as cache_file:
|
||||
json.dump(payload, cache_file)
|
||||
except IOError as ex:
|
||||
log.warning('Unable to write checksum cache file %s: %s', cache_path, ex)
|
||||
|
||||
return checksum
|
||||
|
||||
|
||||
def cleanup_cache() -> None:
|
||||
"""Remove all cache files that are older than MAX_CACHE_FILES_AGE_SECS."""
|
||||
|
||||
if not CACHE_ROOT.exists():
|
||||
return
|
||||
|
||||
with time_tracker.track_time(TimeInfo, 'checksum_cache_handling'):
|
||||
queue = deque([CACHE_ROOT])
|
||||
rmdir_queue = []
|
||||
|
||||
now = time.time()
|
||||
num_removed_files = 0
|
||||
num_removed_dirs = 0
|
||||
while queue:
|
||||
path = queue.popleft()
|
||||
|
||||
if path.is_dir():
|
||||
queue.extend(path.iterdir())
|
||||
rmdir_queue.append(path)
|
||||
continue
|
||||
|
||||
assert path.is_file()
|
||||
path.relative_to(CACHE_ROOT)
|
||||
|
||||
age = now - path.stat().st_mtime
|
||||
# Don't trust files from the future either.
|
||||
if 0 <= age <= MAX_CACHE_FILES_AGE_SECS:
|
||||
continue
|
||||
|
||||
path.unlink()
|
||||
num_removed_files += 1
|
||||
|
||||
for dirpath in reversed(rmdir_queue):
|
||||
assert dirpath.is_dir()
|
||||
dirpath.relative_to(CACHE_ROOT)
|
||||
|
||||
try:
|
||||
dirpath.rmdir()
|
||||
num_removed_dirs += 1
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
if num_removed_dirs or num_removed_files:
|
||||
log.info('Cache Cleanup: removed %d dirs and %d files', num_removed_dirs, num_removed_files)
|
||||
@@ -0,0 +1,129 @@
|
||||
# ***** BEGIN GPL LICENSE BLOCK *****
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
#
|
||||
# ***** END GPL LICENCE BLOCK *****
|
||||
#
|
||||
# (c) 2019, Blender Foundation - Sybren A. Stüvel
|
||||
|
||||
import urllib.parse
|
||||
|
||||
import requests.packages.urllib3.util.retry
|
||||
import requests.adapters
|
||||
|
||||
|
||||
class ShamanClient:
|
||||
"""Thin wrapper around a Requests session to perform Shaman requests."""
|
||||
|
||||
def __init__(self, auth_token: str, base_url: str):
|
||||
self._auth_token = auth_token
|
||||
self._base_url = base_url
|
||||
|
||||
retries = requests.packages.urllib3.util.retry.Retry(
|
||||
total=10,
|
||||
backoff_factor=0.05,
|
||||
)
|
||||
http_adapter = requests.adapters.HTTPAdapter(max_retries=retries)
|
||||
self._session = requests.session()
|
||||
self._session.mount('https://', http_adapter)
|
||||
self._session.mount('http://', http_adapter)
|
||||
|
||||
if auth_token:
|
||||
self._session.headers['Authorization'] = 'Bearer ' + auth_token
|
||||
|
||||
def request(self, method: str, url: str, **kwargs) -> requests.Response:
|
||||
kwargs.setdefault('timeout', 300)
|
||||
full_url = urllib.parse.urljoin(self._base_url, url)
|
||||
return self._session.request(method, full_url, **kwargs)
|
||||
|
||||
def get(self, url, **kwargs):
|
||||
r"""Sends a GET request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param kwargs: Optional arguments that ``request`` takes.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
kwargs.setdefault('allow_redirects', True)
|
||||
return self.request('GET', url, **kwargs)
|
||||
|
||||
def options(self, url, **kwargs):
|
||||
r"""Sends a OPTIONS request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param kwargs: Optional arguments that ``request`` takes.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
kwargs.setdefault('allow_redirects', True)
|
||||
return self.request('OPTIONS', url, **kwargs)
|
||||
|
||||
def head(self, url, **kwargs):
|
||||
r"""Sends a HEAD request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param kwargs: Optional arguments that ``request`` takes.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
kwargs.setdefault('allow_redirects', False)
|
||||
return self.request('HEAD', url, **kwargs)
|
||||
|
||||
def post(self, url, data=None, json=None, **kwargs):
|
||||
r"""Sends a POST request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
|
||||
object to send in the body of the :class:`Request`.
|
||||
:param json: (optional) json to send in the body of the :class:`Request`.
|
||||
:param kwargs: Optional arguments that ``request`` takes.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
return self.request('POST', url, data=data, json=json, **kwargs)
|
||||
|
||||
def put(self, url, data=None, **kwargs):
|
||||
r"""Sends a PUT request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
|
||||
object to send in the body of the :class:`Request`.
|
||||
:param kwargs: Optional arguments that ``request`` takes.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
return self.request('PUT', url, data=data, **kwargs)
|
||||
|
||||
def patch(self, url, data=None, **kwargs):
|
||||
r"""Sends a PATCH request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary, list of tuples, bytes, or file-like
|
||||
object to send in the body of the :class:`Request`.
|
||||
:param kwargs: Optional arguments that ``request`` takes.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
return self.request('PATCH', url, data=data, **kwargs)
|
||||
|
||||
def delete(self, url, **kwargs):
|
||||
r"""Sends a DELETE request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param kwargs: Optional arguments that ``request`` takes.
|
||||
:rtype: requests.Response
|
||||
"""
|
||||
|
||||
return self.request('DELETE', url, **kwargs)
|
||||
@@ -0,0 +1,32 @@
|
||||
# ***** BEGIN GPL LICENSE BLOCK *****
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
#
|
||||
# ***** END GPL LICENCE BLOCK *****
|
||||
#
|
||||
# (c) 2019, Blender Foundation - Sybren A. Stüvel
|
||||
import contextlib
|
||||
import time
|
||||
import typing
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def track_time(tracker_object: typing.Any, attribute: str):
|
||||
"""Context manager, tracks how long the context took to run."""
|
||||
start_time = time.monotonic()
|
||||
yield
|
||||
duration = time.monotonic() - start_time
|
||||
tracked_so_far = getattr(tracker_object, attribute, 0.0)
|
||||
setattr(tracker_object, attribute, tracked_so_far + duration)
|
||||
@@ -0,0 +1,359 @@
|
||||
# ***** BEGIN GPL LICENSE BLOCK *****
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software Foundation,
|
||||
# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
||||
#
|
||||
# ***** END GPL LICENCE BLOCK *****
|
||||
#
|
||||
# (c) 2019, Blender Foundation - Sybren A. Stüvel
|
||||
import collections
|
||||
import logging
|
||||
import pathlib
|
||||
import random
|
||||
import typing
|
||||
|
||||
import requests
|
||||
|
||||
import blender_asset_tracer.pack.transfer as bat_transfer
|
||||
from blender_asset_tracer import bpathlib
|
||||
|
||||
MAX_DEFERRED_PATHS = 8
|
||||
MAX_FAILED_PATHS = 8
|
||||
|
||||
response_file_unknown = "file-unknown"
|
||||
response_already_uploading = "already-uploading"
|
||||
|
||||
|
||||
class FileInfo:
|
||||
def __init__(self, checksum: str, filesize: int, abspath: pathlib.Path):
|
||||
self.checksum = checksum
|
||||
self.filesize = filesize
|
||||
self.abspath = abspath
|
||||
|
||||
|
||||
class ShamanTransferrer(bat_transfer.FileTransferer):
|
||||
"""Sends files to a Shaman server."""
|
||||
|
||||
class AbortUpload(Exception):
|
||||
"""Raised from the upload callback to abort an upload."""
|
||||
|
||||
def __init__(self, auth_token: str, project_root: pathlib.Path,
|
||||
shaman_endpoint: str, checkout_id: str) -> None:
|
||||
from . import client
|
||||
super().__init__()
|
||||
self.client = client.ShamanClient(auth_token, shaman_endpoint)
|
||||
self.project_root = project_root
|
||||
self.checkout_id = checkout_id
|
||||
self.log = logging.getLogger(__name__)
|
||||
|
||||
self._file_info = {} # type: typing.Dict[str, FileInfo]
|
||||
|
||||
# When the Shaman creates a checkout, it'll return the location of that
|
||||
# checkout. This can then be combined with the project-relative path
|
||||
# of the to-be-rendered blend file (e.g. the one 'bat pack' was pointed
|
||||
# at).
|
||||
self._checkout_location = ''
|
||||
|
||||
self.uploaded_files = 0
|
||||
self.uploaded_bytes = 0
|
||||
|
||||
# noinspection PyBroadException
|
||||
def run(self) -> None:
|
||||
try:
|
||||
self.uploaded_files = 0
|
||||
self.uploaded_bytes = 0
|
||||
|
||||
# Construct the Shaman Checkout Definition file.
|
||||
# This blocks until we know the entire list of files to transfer.
|
||||
definition_file, allowed_relpaths, delete_when_done = self._create_checkout_definition()
|
||||
if not definition_file:
|
||||
# An error has already been logged.
|
||||
return
|
||||
|
||||
self.log.info('Created checkout definition file of %d KiB',
|
||||
len(definition_file) // 1024)
|
||||
self.log.info('Feeding %d files to the Shaman', len(self._file_info))
|
||||
if self.log.isEnabledFor(logging.INFO):
|
||||
for path in self._file_info:
|
||||
self.log.info(' - %s', path)
|
||||
|
||||
# Try to upload all the files.
|
||||
failed_paths = set() # type: typing.Set[str]
|
||||
max_tries = 50
|
||||
for try_index in range(max_tries):
|
||||
# Send the file to the Shaman and see what we still need to send there.
|
||||
to_upload = self._send_checkout_def_to_shaman(definition_file, allowed_relpaths)
|
||||
if to_upload is None:
|
||||
# An error has already been logged.
|
||||
return
|
||||
|
||||
if not to_upload:
|
||||
break
|
||||
|
||||
# Send the files that still need to be sent.
|
||||
self.log.info('Upload attempt %d', try_index + 1)
|
||||
failed_paths = self._upload_files(to_upload)
|
||||
if not failed_paths:
|
||||
break
|
||||
|
||||
# Having failed paths at this point is expected when multiple
|
||||
# clients are sending the same files. Instead of retrying on a
|
||||
# file-by-file basis, we just re-send the checkout definition
|
||||
# file to the Shaman and obtain a new list of files to upload.
|
||||
|
||||
if failed_paths:
|
||||
self.log.error('Aborting upload due to too many failures')
|
||||
self.error_set('Giving up after %d attempts to upload the files' % max_tries)
|
||||
return
|
||||
|
||||
self.log.info('All files uploaded succesfully')
|
||||
self._request_checkout(definition_file)
|
||||
|
||||
# Delete the files that were supposed to be moved.
|
||||
for src in delete_when_done:
|
||||
self.delete_file(src)
|
||||
|
||||
except Exception as ex:
|
||||
# We have to catch exceptions in a broad way, as this is running in
|
||||
# a separate thread, and exceptions won't otherwise be seen.
|
||||
self.log.exception('Error transferring files to Shaman')
|
||||
self.error_set('Unexpected exception transferring files to Shaman: %s' % ex)
|
||||
|
||||
# noinspection PyBroadException
|
||||
def _create_checkout_definition(self) \
|
||||
-> typing.Tuple[bytes, typing.Set[str], typing.List[pathlib.Path]]:
|
||||
"""Create the checkout definition file for this BAT pack.
|
||||
|
||||
:returns: the checkout definition (as bytes), a set of paths in that file,
|
||||
and list of paths to delete.
|
||||
|
||||
If there was an error and file transfer was aborted, the checkout
|
||||
definition file will be empty.
|
||||
"""
|
||||
from . import cache
|
||||
|
||||
definition_lines = [] # type: typing.List[bytes]
|
||||
delete_when_done = [] # type: typing.List[pathlib.Path]
|
||||
|
||||
# We keep track of the relative paths we want to send to the Shaman,
|
||||
# so that the Shaman cannot ask us to upload files we didn't want to.
|
||||
relpaths = set() # type: typing.Set[str]
|
||||
|
||||
for src, dst, act in self.iter_queue():
|
||||
try:
|
||||
checksum = cache.compute_cached_checksum(src)
|
||||
filesize = src.stat().st_size
|
||||
# relpath = dst.relative_to(self.project_root)
|
||||
relpath = bpathlib.strip_root(dst).as_posix()
|
||||
|
||||
self._file_info[relpath] = FileInfo(
|
||||
checksum=checksum,
|
||||
filesize=filesize,
|
||||
abspath=src,
|
||||
)
|
||||
line = '%s %s %s' % (checksum, filesize, relpath)
|
||||
definition_lines.append(line.encode('utf8'))
|
||||
relpaths.add(relpath)
|
||||
|
||||
if act == bat_transfer.Action.MOVE:
|
||||
delete_when_done.append(src)
|
||||
except Exception:
|
||||
# We have to catch exceptions in a broad way, as this is running in
|
||||
# a separate thread, and exceptions won't otherwise be seen.
|
||||
msg = 'Error transferring %s to %s' % (src, dst)
|
||||
self.log.exception(msg)
|
||||
# Put the files to copy back into the queue, and abort. This allows
|
||||
# the main thread to inspect the queue and see which files were not
|
||||
# copied. The one we just failed (due to this exception) should also
|
||||
# be reported there.
|
||||
self.queue.put((src, dst, act))
|
||||
self.error_set(msg)
|
||||
return b'', set(), delete_when_done
|
||||
|
||||
cache.cleanup_cache()
|
||||
return b'\n'.join(definition_lines), relpaths, delete_when_done
|
||||
|
||||
def _send_checkout_def_to_shaman(self, definition_file: bytes,
|
||||
allowed_relpaths: typing.Set[str]) \
|
||||
-> typing.Optional[collections.deque]:
|
||||
"""Send the checkout definition file to the Shaman.
|
||||
|
||||
:return: An iterable of paths (relative to the project root) that still
|
||||
need to be uploaded, or None if there was an error.
|
||||
"""
|
||||
resp = self.client.post('checkout/requirements', data=definition_file, stream=True,
|
||||
headers={'Content-Type': 'text/plain'},
|
||||
timeout=15)
|
||||
if resp.status_code >= 300:
|
||||
msg = 'Error from Shaman, code %d: %s' % (resp.status_code, resp.text)
|
||||
self.log.error(msg)
|
||||
self.error_set(msg)
|
||||
return None
|
||||
|
||||
to_upload = collections.deque() # type: collections.deque
|
||||
for line in resp.iter_lines():
|
||||
response, path = line.decode().split(' ', 1)
|
||||
self.log.debug(' %s: %s', response, path)
|
||||
|
||||
if path not in allowed_relpaths:
|
||||
msg = 'Shaman requested path we did not intend to upload: %r' % path
|
||||
self.log.error(msg)
|
||||
self.error_set(msg)
|
||||
return None
|
||||
|
||||
if response == response_file_unknown:
|
||||
to_upload.appendleft(path)
|
||||
elif response == response_already_uploading:
|
||||
to_upload.append(path)
|
||||
elif response == 'ERROR':
|
||||
msg = 'Error from Shaman: %s' % path
|
||||
self.log.error(msg)
|
||||
self.error_set(msg)
|
||||
return None
|
||||
else:
|
||||
msg = 'Unknown response from Shaman for path %r: %r' % (path, response)
|
||||
self.log.error(msg)
|
||||
self.error_set(msg)
|
||||
return None
|
||||
|
||||
return to_upload
|
||||
|
||||
def _upload_files(self, to_upload: collections.deque) -> typing.Set[str]:
|
||||
"""Actually upload the files to Shaman.
|
||||
|
||||
Returns the set of files that we did not upload.
|
||||
"""
|
||||
failed_paths = set() # type: typing.Set[str]
|
||||
deferred_paths = set()
|
||||
|
||||
def defer(some_path: str):
|
||||
nonlocal to_upload
|
||||
|
||||
self.log.info(' %s deferred (already being uploaded by someone else)', some_path)
|
||||
deferred_paths.add(some_path)
|
||||
|
||||
# Instead of deferring this one file, randomize the files to upload.
|
||||
# This prevents multiple deferrals when someone else is uploading
|
||||
# files from the same project (because it probably happens alphabetically).
|
||||
all_files = list(to_upload)
|
||||
random.shuffle(all_files)
|
||||
to_upload = collections.deque(all_files)
|
||||
|
||||
if not to_upload:
|
||||
self.log.info('All %d files are at the Shaman already', len(self._file_info))
|
||||
self.report_transferred(0)
|
||||
return failed_paths
|
||||
|
||||
self.log.info('Going to upload %d of %d files', len(to_upload), len(self._file_info))
|
||||
while to_upload:
|
||||
# After too many failures, just retry to get a fresh set of files to upload.
|
||||
if len(failed_paths) > MAX_FAILED_PATHS:
|
||||
self.log.info('Too many failures, going to abort this iteration')
|
||||
failed_paths.update(to_upload)
|
||||
return failed_paths
|
||||
|
||||
path = to_upload.popleft()
|
||||
fileinfo = self._file_info[path]
|
||||
self.log.info(' %s', path)
|
||||
|
||||
headers = {
|
||||
'X-Shaman-Original-Filename': path,
|
||||
}
|
||||
# Let the Shaman know whether we can defer uploading this file or not.
|
||||
can_defer = (len(deferred_paths) < MAX_DEFERRED_PATHS
|
||||
and path not in deferred_paths
|
||||
and len(to_upload))
|
||||
if can_defer:
|
||||
headers['X-Shaman-Can-Defer-Upload'] = 'true'
|
||||
|
||||
url = 'files/%s/%d' % (fileinfo.checksum, fileinfo.filesize)
|
||||
try:
|
||||
with fileinfo.abspath.open('rb') as infile:
|
||||
resp = self.client.post(url, data=infile, headers=headers)
|
||||
|
||||
except requests.ConnectionError as ex:
|
||||
if can_defer:
|
||||
# Closing the connection with an 'X-Shaman-Can-Defer-Upload: true' header
|
||||
# indicates that we should defer the upload. Requests doesn't give us the
|
||||
# reply, even though it was written by the Shaman before it closed the
|
||||
# connection.
|
||||
defer(path)
|
||||
else:
|
||||
self.log.info(' %s could not be uploaded, might retry later: %s', path, ex)
|
||||
failed_paths.add(path)
|
||||
continue
|
||||
|
||||
if resp.status_code == 208:
|
||||
# For small files we get the 208 response, because the server closes the
|
||||
# connection after we sent the entire request. For bigger files the server
|
||||
# responds sooner, and Requests gives us the above ConnectionError.
|
||||
if can_defer:
|
||||
defer(path)
|
||||
else:
|
||||
self.log.info(' %s skipped (already existed on the server)', path)
|
||||
continue
|
||||
|
||||
if resp.status_code >= 300:
|
||||
msg = 'Error from Shaman uploading %s, code %d: %s' % (
|
||||
fileinfo.abspath, resp.status_code, resp.text)
|
||||
self.log.error(msg)
|
||||
self.error_set(msg)
|
||||
return failed_paths
|
||||
|
||||
failed_paths.discard(path)
|
||||
self.uploaded_files += 1
|
||||
file_size = fileinfo.abspath.stat().st_size
|
||||
self.uploaded_bytes += file_size
|
||||
self.report_transferred(file_size)
|
||||
|
||||
if not failed_paths:
|
||||
self.log.info('Done uploading %d bytes in %d files',
|
||||
self.uploaded_bytes, self.uploaded_files)
|
||||
else:
|
||||
self.log.info('Uploaded %d bytes in %d files so far',
|
||||
self.uploaded_bytes, self.uploaded_files)
|
||||
|
||||
return failed_paths
|
||||
|
||||
def report_transferred(self, bytes_transferred: int):
|
||||
if self._abort.is_set():
|
||||
self.log.warning('Interrupting ongoing upload')
|
||||
raise self.AbortUpload('interrupting ongoing upload')
|
||||
super().report_transferred(bytes_transferred)
|
||||
|
||||
def _request_checkout(self, definition_file: bytes):
|
||||
"""Ask the Shaman to create a checkout of this BAT pack."""
|
||||
|
||||
if not self.checkout_id:
|
||||
self.log.warning('NOT requesting checkout at Shaman')
|
||||
return
|
||||
|
||||
self.log.info('Requesting checkout at Shaman for checkout_id=%r', self.checkout_id)
|
||||
resp = self.client.post('checkout/create/%s' % self.checkout_id, data=definition_file,
|
||||
headers={'Content-Type': 'text/plain'})
|
||||
if resp.status_code >= 300:
|
||||
msg = 'Error from Shaman, code %d: %s' % (resp.status_code, resp.text)
|
||||
self.log.error(msg)
|
||||
self.error_set(msg)
|
||||
return
|
||||
|
||||
self._checkout_location = resp.text.strip()
|
||||
self.log.info('Response from Shaman, code %d: %s', resp.status_code, resp.text)
|
||||
|
||||
@property
|
||||
def checkout_location(self) -> str:
|
||||
"""Returns the checkout location, or '' if no checkout was made."""
|
||||
if not self._checkout_location:
|
||||
return ''
|
||||
return self._checkout_location
|
||||
Reference in New Issue
Block a user