From 4c8d14478c3fdd25c7b6bdcfdee8b65fb2c7c862 Mon Sep 17 00:00:00 2001 From: Juho Hong Date: Wed, 14 Feb 2024 23:15:49 +0900 Subject: [PATCH] Add S3 Support --- .env.dist | 5 +- apps/shared/utils/storage.py | 113 ++++++++++++++++++++++++++++++++++ apps/shared/utils/string.py | 4 ++ project_name/settings/base.py | 10 +-- requirements.txt | 1 + 5 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 apps/shared/utils/storage.py diff --git a/.env.dist b/.env.dist index 761d6ef..e99b26a 100644 --- a/.env.dist +++ b/.env.dist @@ -19,5 +19,6 @@ SERVER_EMAIL=webmaster@example.com # CACHE CACHE_URL=rediscache://127.0.0.1:6379/1 -# FILES -MEDIA_ROOT=/path/to/media +# STORAGES +STORAGE_DEFAULT=s3://key:secret@region.hostname/bucket?querystring_expire=86400&location=media +STORAGE_STATICFILES=s3://key:secret@region.hostname/bucket?querystring_expire=86400&location=static diff --git a/apps/shared/utils/storage.py b/apps/shared/utils/storage.py new file mode 100644 index 0000000..e39a5c4 --- /dev/null +++ b/apps/shared/utils/storage.py @@ -0,0 +1,113 @@ +import json +from urllib.parse import parse_qs, urlparse + +from .string import str_to_bool + + +class BaseStorageParser: + BACKEND = None + OPTION_CASTS = {} + + def __init__(self, url): + self.url = urlparse(url) + self.query_params = self.url_querystring_to_dict() + + def __call__(self): + storage = {'BACKEND': self.BACKEND} + options = self.get_options() + if options: + storage['OPTIONS'] = options + return storage + + def url_querystring_to_dict(self): + query_string = self.url.query + + query_dict = parse_qs(query_string) + + for key, value in query_dict.items(): + if len(value) == 1: + query_dict[key] = value[0] + + return { + key: self.OPTION_CASTS[key](value) if key in self.OPTION_CASTS else value + for key, value in query_dict.items() + } + + def get_options(self): + return None + + +class FileSystemStorageParser(BaseStorageParser): + BACKEND = 'django.core.files.storage.FileSystemStorage' + + def get_options(self): + options = {**self.query_params} + + location = '' + if self.url.hostname: + location = self.url.hostname + if self.url.path: + location += self.url.path + + if location: + options['location'] = location + return options + + +class StaticFilesStorageParser(FileSystemStorageParser): + BACKEND = 'django.contrib.staticfiles.storage.StaticFilesStorage' + + +class S3StorageParser(BaseStorageParser): + BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' + OPTION_CASTS = { + 'object_parameters': json.loads, + 'querystring_auth': str_to_bool, + 'max_memory_size': int, + 'querystring_expire': int, + 'file_overwrite': str_to_bool, + 'gzip': str_to_bool, + 'use_ssl': str_to_bool, + 'verify': str_to_bool, + 'proxies': json.loads, + 'transfer_config': json.loads, + } + + def get_options(self): + endpoint_url = '.'.join(self.url.hostname.split('.')[1:]) + return { + 'endpoint_url': f'https://{endpoint_url}', + 'bucket_name': self.url.path[1:], + 'access_key': self.url.username, + 'secret_key': self.url.password, + 'region_name': self.url.hostname.split('.')[0], + **self.query_params, + } + + +STORAGE_PARSERS = { + 'filesystem': FileSystemStorageParser, + 'staticfiles': StaticFilesStorageParser, + 's3': S3StorageParser, +} + + +def get_storages(env, storages=None): + if storages is None: + storages = ['default', ('staticfiles', 'staticfiles://')] + + storage_settings = {} + + for storage in storages: + if isinstance(storage, tuple): + storage_url = env(f'STORAGE_{storage[0].upper()}', default=storage[1]) + storage_name = storage[0] + else: + storage_url = env(f'STORAGE_{storage.upper()}') + storage_name = storage + + storage_scheme = urlparse(storage_url).scheme + parser_class = STORAGE_PARSERS[storage_scheme] + storage_settings[storage_name] = parser_class(storage_url)() + + return storage_settings diff --git a/apps/shared/utils/string.py b/apps/shared/utils/string.py index 6c3eb95..99a42bb 100644 --- a/apps/shared/utils/string.py +++ b/apps/shared/utils/string.py @@ -5,3 +5,7 @@ def generate_random(type='char', length=6): chars = string.ascii_uppercase + string.digits return ''.join(random.choice(chars) for _ in range(length)) + + +def str_to_bool(value): + return value.lower() in ['true', '1', 'yes'] diff --git a/project_name/settings/base.py b/project_name/settings/base.py index 9db87f4..04de807 100644 --- a/project_name/settings/base.py +++ b/project_name/settings/base.py @@ -2,6 +2,8 @@ import environ +from apps.shared.utils.storage import get_storages + env = environ.Env() # Build paths inside the project like this: BASE_DIR / 'subdir'. @@ -106,20 +108,20 @@ USE_TZ = True +# Storages for Static and Media Files +# https://docs.djangoproject.com/en/5.0/ref/settings/#std-setting-STORAGES + +STORAGES = get_storages(env) # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.0/howto/static-files/ STATIC_URL = '/static/' -STATIC_ROOT = BASE_DIR / 'resources/static' - STATICFILES_DIRS = [BASE_DIR / 'static'] MEDIA_URL = '/media/' -MEDIA_ROOT = env('MEDIA_ROOT') - # Default primary key field type # https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field diff --git a/requirements.txt b/requirements.txt index 6a0ca24..2062a31 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ django==5.0 django-cleanup django-environ +django-storages[s3]==1.14.2 easy-thumbnails psycopg[binary]