From 825767a6d6970d357b627de4c5b5e88a14dc0e75 Mon Sep 17 00:00:00 2001 From: Alexey Kinev Date: Fri, 12 Nov 2021 17:45:13 +0300 Subject: [PATCH] Updated S3 provider to be more generic --- .gitignore | 2 +- CHANGELOG.md | 6 ++++++ signa/__init__.py | 2 +- signa/core.py | 17 ++++++----------- signa/providers/aws.py | 1 - signa/providers/b2.py | 6 ++---- signa/providers/dospaces.py | 32 +++++++++++++------------------- signa/providers/onesignal.py | 19 ------------------- signa/providers/oss.py | 32 +++++++++++++------------------- signa/providers/s3.py | 35 +++++++++++++++++++++++++++-------- tests/__init__.py | 8 +++----- 11 files changed, 72 insertions(+), 88 deletions(-) delete mode 100644 signa/providers/onesignal.py diff --git a/.gitignore b/.gitignore index bbc8041..8daf399 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ __pycache__/ # Distribution / packaging .Python -env/ +.venv/ build/ develop-eggs/ dist/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a069de..652bef7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +0.3.0 +----- + +- Breaking change: removed Onesignal module +- Extended S3 `s3.new()` interface with `endpoint` argument + 0.2.8 ----- diff --git a/signa/__init__.py b/signa/__init__.py index 9a75910..add75fc 100644 --- a/signa/__init__.py +++ b/signa/__init__.py @@ -1,3 +1,3 @@ from .core import new, Factory -__version__ = '0.2.8' +__version__ = '0.3.0' diff --git a/signa/core.py b/signa/core.py index 91ead5d..005686c 100644 --- a/signa/core.py +++ b/signa/core.py @@ -4,9 +4,6 @@ dospaces, yaobject, oss, - - # TODO: why is it here, haha? - onesignal ) PROVIDERS = { @@ -15,9 +12,6 @@ 'dospaces': dospaces.new, 'yaobject': yaobject.new, 'oss': oss.new, - - # TODO: remove after deprecation cycle - 'onesignal': onesignal.new, } @@ -42,15 +36,15 @@ def new(self, **kwargs): if __name__ == '__main__': - import yaml + import os import json import requests + from dotenv import load_dotenv, find_dotenv - with open('config.yml', 'r') as f: - config = yaml.load(f) + load_dotenv(find_dotenv()) - access_key = config['s3']['access_key'] - secret_key = config['s3']['secret_key'] + access_key = os.environ['AWS_ACCESS_KEY_ID'] + secret_key = os.environ['AWS_SECRET_ACCESS_KEY'] signed = new( 's3', method='PUT', @@ -70,6 +64,7 @@ def new(self, **kwargs): print('\n') r = requests.put(signed['url'], headers=signed['headers'], data=b'xxxxxxxx') + r.raise_for_status() print(r.text) print('\n') diff --git a/signa/providers/aws.py b/signa/providers/aws.py index 8af5ee4..7720c67 100644 --- a/signa/providers/aws.py +++ b/signa/providers/aws.py @@ -1,7 +1,6 @@ import datetime import hashlib import hmac -import json import urllib.parse from signa.logger import get_logger diff --git a/signa/providers/b2.py b/signa/providers/b2.py index af16d8c..2fef7d5 100644 --- a/signa/providers/b2.py +++ b/signa/providers/b2.py @@ -1,16 +1,14 @@ from .aws import aws_headers REGIONS = { - 'us-west-000', 'us-west-001', 'us-west-002', - # TODO: add European region + 'us-west-000', 'us-west-001', 'us-west-002', 'eu-central-003' } def new(method=None, region=None, bucket=None, key=None, auth=None, headers=None, payload=None): headers = headers.copy() if headers else {} - assert region in REGIONS - + # assert region in REGIONS # headers['host'] = '%s.s3.%s.backblazeb2.com' % (bucket, region) # https://s3..backblazeb2.com/ diff --git a/signa/providers/dospaces.py b/signa/providers/dospaces.py index d7c2af3..f08d1b8 100644 --- a/signa/providers/dospaces.py +++ b/signa/providers/dospaces.py @@ -1,25 +1,19 @@ -from .aws import aws_headers +from typing import Any, Dict, Optional +from . import s3 -def new(method=None, region=None, bucket=None, key=None, - auth=None, headers=None, payload=None): - headers = headers.copy() if headers else {} - - headers['host'] = '%s.%s.digitaloceanspaces.com' % (bucket, region) - - rel_uri = ('/%s' % key) if key else '/' - - headers.update(aws_headers( +def new(method: str, region: str = '', bucket: str = '', + key: str = '', auth: Optional[Dict[str, str]] = None, + headers: Optional[Dict[str, str]] = None, + payload: Any = None, +) -> Dict[str, str]: + return s3.new( method=method, region=region, - service='s3', - uri=rel_uri, + bucket=bucket, + key=key, auth=auth, headers=headers, - payload=payload - )) - - return { - 'url': 'https://%s%s' % (headers['host'], rel_uri), - 'headers': headers, - } + payload=payload, + endpoint='{region}.digitaloceanspaces.com' + ) diff --git a/signa/providers/onesignal.py b/signa/providers/onesignal.py deleted file mode 100644 index ad0895e..0000000 --- a/signa/providers/onesignal.py +++ /dev/null @@ -1,19 +0,0 @@ -import json - - -def new(app_id=None, api_key=None, api_method=None, **payload): - if not all((app_id, api_key, api_method)): - raise RuntimeError("Missing one or more of required arguments: " - "app_id, api_key, api_method") - - url = 'https://onesignal.com/api/v1/%s' % api_method - headers = { - 'Content-Type': 'application/json; charset=utf-8', - 'Authorization': 'Basic %s' % api_key - } - payload['app_id'] = app_id - return { - 'url': url, - 'headers': headers, - 'data': json.dumps(payload) - } diff --git a/signa/providers/oss.py b/signa/providers/oss.py index 15f6071..e64ce9f 100644 --- a/signa/providers/oss.py +++ b/signa/providers/oss.py @@ -1,25 +1,19 @@ -from .aws import aws_headers +from typing import Any, Dict, Optional +from . import s3 -def new(method=None, region=None, bucket=None, key=None, - auth=None, headers=None, payload=None): - headers = headers.copy() if headers else {} - - headers['host'] = '%s.%s.aliyuncs.com' % (bucket, region) - - rel_uri = ('/%s' % key) if key else '/' - - headers.update(aws_headers( +def new(method: str, region: str = '', bucket: str = '', + key: str = '', auth: Optional[Dict[str, str]] = None, + headers: Optional[Dict[str, str]] = None, + payload: Any = None, +) -> Dict[str, str]: + return s3.new( method=method, region=region, - service='s3', - uri=rel_uri, + bucket=bucket, + key=key, auth=auth, headers=headers, - payload=payload - )) - - return { - 'url': 'https://%s%s' % (headers['host'], rel_uri), - 'headers': headers, - } + payload=payload, + endpoint='{region}.aliyuncs.com' + ) diff --git a/signa/providers/s3.py b/signa/providers/s3.py index ed6697a..cc2a894 100644 --- a/signa/providers/s3.py +++ b/signa/providers/s3.py @@ -1,15 +1,33 @@ +from typing import Any, Dict, Optional from .aws import aws_headers -def new(method=None, region=None, bucket=None, key=None, - auth=None, headers=None, payload=None): +def new(method: str, region: str = '', bucket: str = '', + key: str = '', auth: Optional[Dict[str, str]] = None, + headers: Optional[Dict[str, str]] = None, + payload: Any = None, + endpoint:str = 's3.{region}.amazonaws.com' + ) -> Dict[str, str]: + """ + Create new signature for S3 request. + + Returns + ------- + Dictionary with `url` and `headers` keys. + """ headers = headers.copy() if headers else {} + if endpoint and region: + endpoint = endpoint.format(region=region) + # Details on buckets URLs: # https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html - headers['host'] = '%s.s3.%s.amazonaws.com' % (bucket, region) + # https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/ + headers['host'] = '{bucket}.{endpoint}'.format( + bucket=bucket, endpoint=endpoint + ) - rel_uri = ('/%s' % key) if key else '/' + rel_uri = '/{key}'.format(key=(key or '')).strip() headers.update(aws_headers( method=method, @@ -21,7 +39,8 @@ def new(method=None, region=None, bucket=None, key=None, payload=payload )) - return { - 'url': 'https://%s%s' % (headers['host'], rel_uri), - 'headers': headers, - } + url = 'https://{host}{rel_uri}'.format( + host=headers['host'], rel_uri=rel_uri + ) + + return {'url': url, 'headers': headers} diff --git a/tests/__init__.py b/tests/__init__.py index 5aae240..76b050b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -24,7 +24,7 @@ def tearDown(self): def test_new_s3(self): key = 'test.txt' - _ = signa.new( + result = signa.new( aws_s3_provider, method='PUT', region=aws_region, @@ -38,7 +38,5 @@ def test_new_s3(self): 'access_key': aws_access_key, 'secret_key': aws_secret_key, }) - self.assertTrue(True) - - def test_new_onesignal(self): - self.assertTrue(True) + self.assertTrue('url' in result) + self.assertTrue('headers' in result)