From 56fb89da3d5fc64faa14eaf85b81246a70ac52b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Tue, 20 Mar 2018 16:36:29 +0100 Subject: [PATCH] S3Packer now takes a URL as endpoint, not a hostname --- README.md | 2 +- blender_asset_tracer/cli/pack.py | 2 +- blender_asset_tracer/pack/s3.py | 27 ++++++++++++++++++--------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index ef0585a..c01380f 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ BAT Pack supports uploading to S3-compatible storage. This requires a credential You can then send a BAT Pack to the storage using a target `s3:/ENDPOINT/bucketname/path-in-bucket`, for example: - bat pack my_blendfile.blend s3:/storage.qarnot.com/jobs/awesome_work + bat pack my_blendfile.blend s3:/storage.service.cloud/jobs/awesome_work This will upload the blend file and its dependencies to `awesome_work/my_blendfile.blend` in the `jobs` bucket. diff --git a/blender_asset_tracer/cli/pack.py b/blender_asset_tracer/cli/pack.py index e80eaa9..0bb3d1a 100644 --- a/blender_asset_tracer/cli/pack.py +++ b/blender_asset_tracer/cli/pack.py @@ -87,7 +87,7 @@ def create_s3packer(bpath, ppath, tpath) -> pack.Packer: # Split the target path into 's3:/', hostname, and actual target path parts = tpath.parts - endpoint = parts[1] + endpoint = 'https://%s/' % parts[1] tpath = pathlib.Path(*tpath.parts[2:]) log.info('Uploading to S3-compatible storage %s at %s', endpoint, tpath) diff --git a/blender_asset_tracer/pack/s3.py b/blender_asset_tracer/pack/s3.py index 5e9c180..47c350a 100644 --- a/blender_asset_tracer/pack/s3.py +++ b/blender_asset_tracer/pack/s3.py @@ -23,7 +23,7 @@ import typing import hashlib import logging import pathlib -import threading +import urllib.parse from . import Packer, transfer @@ -31,6 +31,7 @@ log = logging.getLogger(__name__) def compute_md5(filepath: pathlib.Path) -> str: + log.debug('Computing MD5sum of %s', filepath) hasher = hashlib.md5() with filepath.open('rb') as infile: while True: @@ -39,6 +40,7 @@ def compute_md5(filepath: pathlib.Path) -> str: break hasher.update(block) md5 = hasher.hexdigest() + log.debug('MD5sum of %s is %s', filepath, md5) return md5 @@ -46,26 +48,33 @@ class S3Packer(Packer): """Creates BAT Packs on S3-compatible storage.""" def __init__(self, *args, endpoint, **kwargs) -> None: + """Constructor + + :param endpoint: URL of the S3 storage endpoint + """ super().__init__(*args, **kwargs) import boto3 # Create a session so that credentials can be read from the [endpoint] # section in ~/.aws/credentials. # See https://boto3.readthedocs.io/en/latest/guide/configuration.html#guide-configuration - self.session = boto3.Session(profile_name=endpoint) - self.client = self.session.client('s3', endpoint_url='https://%s' % endpoint) + components = urllib.parse.urlparse(endpoint) + profile_name = components.netloc + endpoint = urllib.parse.urlunparse(components) + log.debug('Using Boto3 profile name %r for url %r', profile_name, endpoint) + self.session = boto3.Session(profile_name=profile_name) + + self.client = self.session.client('s3', endpoint_url=endpoint) def set_credentials(self, endpoint: str, access_key_id: str, secret_access_key: str): """Set S3 credentials.""" - import boto3 - - self.client = boto3.client('s3', - endpoint_url=endpoint, - aws_access_key_id=access_key_id, - aws_secret_access_key=secret_access_key) + self.client = self.session.client('s3', + endpoint_url=endpoint, + aws_access_key_id=access_key_id, + aws_secret_access_key=secret_access_key) def _create_file_transferer(self) -> transfer.FileTransferer: return S3Transferrer(self.client)