autour_de_minuit/blender-asset-tracer
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

S3Packer now takes a URL as endpoint, not a hostname

Browse Source
This commit is contained in:
Sybren A. Stüvel 2018-03-20 16:36:29 +01:00
parent c4a57039f1
commit 56fb89da3d
3 changed files with 20 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -21,7 +21,7 @@ BAT Pack supports uploading to S3-compatible storage. This requires a credential
You can then send a BAT Pack to the storage using a target `s3:/ENDPOINT/bucketname/path-in-bucket`, You can then send a BAT Pack to the storage using a target `s3:/ENDPOINT/bucketname/path-in-bucket`,
for example: for example:
bat pack my_blendfile.blend s3:/storage.qarnot.com/jobs/awesome_work bat pack my_blendfile.blend s3:/storage.service.cloud/jobs/awesome_work
This will upload the blend file and its dependencies to `awesome_work/my_blendfile.blend` in This will upload the blend file and its dependencies to `awesome_work/my_blendfile.blend` in
the `jobs` bucket. the `jobs` bucket.

2
blender_asset_tracer/cli/pack.py
View File

@ -87,7 +87,7 @@ def create_s3packer(bpath, ppath, tpath) -> pack.Packer:
# Split the target path into 's3:/', hostname, and actual target path # Split the target path into 's3:/', hostname, and actual target path
parts = tpath.parts parts = tpath.parts
endpoint = parts[1] endpoint = 'https://%s/' % parts[1]
tpath = pathlib.Path(*tpath.parts[2:]) tpath = pathlib.Path(*tpath.parts[2:])
log.info('Uploading to S3-compatible storage %s at %s', endpoint, tpath) log.info('Uploading to S3-compatible storage %s at %s', endpoint, tpath)

21
blender_asset_tracer/pack/s3.py
View File

@ -23,7 +23,7 @@ import typing
import hashlib import hashlib
import logging import logging
import pathlib import pathlib
import threading import urllib.parse
from . import Packer, transfer from . import Packer, transfer
@ -31,6 +31,7 @@ log = logging.getLogger(__name__)
def compute_md5(filepath: pathlib.Path) -> str: def compute_md5(filepath: pathlib.Path) -> str:
log.debug('Computing MD5sum of %s', filepath)
hasher = hashlib.md5() hasher = hashlib.md5()
with filepath.open('rb') as infile: with filepath.open('rb') as infile:
while True: while True:
@ -39,6 +40,7 @@ def compute_md5(filepath: pathlib.Path) -> str:
break break
hasher.update(block) hasher.update(block)
md5 = hasher.hexdigest() md5 = hasher.hexdigest()
log.debug('MD5sum of %s is %s', filepath, md5)
return md5 return md5
@ -46,23 +48,30 @@ class S3Packer(Packer):
"""Creates BAT Packs on S3-compatible storage.""" """Creates BAT Packs on S3-compatible storage."""
def __init__(self, *args, endpoint, **kwargs) -> None: def __init__(self, *args, endpoint, **kwargs) -> None:
"""Constructor
:param endpoint: URL of the S3 storage endpoint
"""
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
import boto3 import boto3
# Create a session so that credentials can be read from the [endpoint] # Create a session so that credentials can be read from the [endpoint]
# section in ~/.aws/credentials. # section in ~/.aws/credentials.
# See https://boto3.readthedocs.io/en/latest/guide/configuration.html#guide-configuration # See https://boto3.readthedocs.io/en/latest/guide/configuration.html#guide-configuration
self.session = boto3.Session(profile_name=endpoint) components = urllib.parse.urlparse(endpoint)
self.client = self.session.client('s3', endpoint_url='https://%s' % endpoint) profile_name = components.netloc
endpoint = urllib.parse.urlunparse(components)
log.debug('Using Boto3 profile name %r for url %r', profile_name, endpoint)
self.session = boto3.Session(profile_name=profile_name)
self.client = self.session.client('s3', endpoint_url=endpoint)
def set_credentials(self, def set_credentials(self,
endpoint: str, endpoint: str,
access_key_id: str, access_key_id: str,
secret_access_key: str): secret_access_key: str):
"""Set S3 credentials.""" """Set S3 credentials."""
import boto3 self.client = self.session.client('s3',
self.client = boto3.client('s3',
endpoint_url=endpoint, endpoint_url=endpoint,
aws_access_key_id=access_key_id, aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key) aws_secret_access_key=secret_access_key)