From 571495cb65d897c7f5477f1442563fe8a68997e9 Mon Sep 17 00:00:00 2001 From: GareArc Date: Wed, 18 Dec 2024 23:56:54 -0500 Subject: [PATCH 01/34] feat: account delete --- api/configs/feature/__init__.py | 24 ++-- api/controllers/console/workspace/account.py | 66 ++++++++-- api/libs/helper.py | 12 +- api/models/account.py | 14 ++ api/services/account_deletion_log_service.py | 34 +++++ api/services/account_service.py | 99 ++++++++++----- api/services/billing_service.py | 13 +- api/tasks/delete_account_task.py | 67 ++++++++++ api/tasks/mail_account_deletion_task.py | 127 +++++++++++++++++++ 9 files changed, 395 insertions(+), 61 deletions(-) create mode 100644 api/services/account_deletion_log_service.py create mode 100644 api/tasks/delete_account_task.py create mode 100644 api/tasks/mail_account_deletion_task.py diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index 73f8a95989baaf..c9fd24e9c197dc 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -1,18 +1,10 @@ from typing import Annotated, Literal, Optional -from pydantic import ( - AliasChoices, - Field, - HttpUrl, - NegativeInt, - NonNegativeInt, - PositiveFloat, - PositiveInt, - computed_field, -) -from pydantic_settings import BaseSettings - from configs.feature.hosted_service import HostedServiceConfig +from pydantic import (AliasChoices, Field, HttpUrl, NegativeInt, + NonNegativeInt, PositiveFloat, PositiveInt, + computed_field) +from pydantic_settings import BaseSettings class SecurityConfig(BaseSettings): @@ -767,6 +759,13 @@ class LoginConfig(BaseSettings): ) +class RegisterConfig(BaseSettings): + EMAIL_FREEZE_PERIOD_IN_DAYS: PositiveInt = Field( + description="Freeze period in days for re-registering with the same email", + default=30, + ) + + class FeatureConfig( # place the configs in alphabet order AppExecutionConfig, @@ -794,6 +793,7 @@ class FeatureConfig( WorkflowNodeExecutionConfig, WorkspaceConfig, LoginConfig, + RegisterConfig, # hosted services config HostedServiceConfig, CeleryBeatConfig, diff --git a/api/controllers/console/workspace/account.py b/api/controllers/console/workspace/account.py index f704783cfff56b..091b3342bffd6e 100644 --- a/api/controllers/console/workspace/account.py +++ b/api/controllers/console/workspace/account.py @@ -1,27 +1,28 @@ import datetime import pytz -from flask import request -from flask_login import current_user -from flask_restful import Resource, fields, marshal_with, reqparse - from configs import dify_config from constants.languages import supported_language from controllers.console import api -from controllers.console.workspace.error import ( - AccountAlreadyInitedError, - CurrentPasswordIncorrectError, - InvalidInvitationCodeError, - RepeatPasswordNotMatchError, -) -from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required +from controllers.console.workspace.error import (AccountAlreadyInitedError, + CurrentPasswordIncorrectError, + InvalidInvitationCodeError, + RepeatPasswordNotMatchError) +from controllers.console.wraps import (account_initialization_required, + enterprise_license_required, + setup_required) from extensions.ext_database import db from fields.member_fields import account_fields +from flask import request +from flask_login import current_user +from flask_restful import Resource, fields, marshal_with, reqparse from libs.helper import TimestampField, timezone from libs.login import login_required from models import AccountIntegrate, InvitationCode from services.account_service import AccountService -from services.errors.account import CurrentPasswordIncorrectError as ServiceCurrentPasswordIncorrectError +from services.errors.account import \ + CurrentPasswordIncorrectError as ServiceCurrentPasswordIncorrectError +from services.errors.account import RateLimitExceededError class AccountInitApi(Resource): @@ -242,6 +243,45 @@ def get(self): return {"data": integrate_data} +class AccountDeleteVerifyApi(Resource): + + @setup_required + @login_required + @account_initialization_required + def post(self): + account = current_user + + try: + token, code = AccountService.generate_account_deletion_verification_code(account) + AccountService.send_account_delete_verification_email(account, code) + except RateLimitExceededError: + return {"result": "fail", "error": "Rate limit exceeded."}, 429 + + return {"result": "success", "data": token} + + +class AccountDeleteApi(Resource): + + @setup_required + @login_required + @account_initialization_required + def post(self): + account = current_user + + parser = reqparse.RequestParser() + parser.add_argument("token", type=str, required=True, location="json") + parser.add_argument("code", type=str, required=True, location="json") + parser.add_argument("reason", type=str, required=True, location="json") + args = parser.parse_args() + + if not AccountService.verify_account_deletion_code(args["token"], args["code"]): + return {"result": "fail", "error": "Verification code is invalid."}, 400 + + AccountService.delete_account(account, args["reason"]) + + return {"result": "success"} + + # Register API resources api.add_resource(AccountInitApi, "/account/init") api.add_resource(AccountProfileApi, "/account/profile") @@ -252,5 +292,7 @@ def get(self): api.add_resource(AccountTimezoneApi, "/account/timezone") api.add_resource(AccountPasswordApi, "/account/password") api.add_resource(AccountIntegrateApi, "/account/integrates") +api.add_resource(AccountDeleteVerifyApi, "/account/delete/verify") +api.add_resource(AccountDeleteApi, "/account/delete") # api.add_resource(AccountEmailApi, '/account/email') # api.add_resource(AccountEmailVerifyApi, '/account/email-verify') diff --git a/api/libs/helper.py b/api/libs/helper.py index 91b1d1fe173d6f..62bc9fa98ded10 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -8,19 +8,21 @@ import uuid from collections.abc import Generator, Mapping from datetime import datetime +from datetime import timezone as tz from hashlib import sha256 from typing import Any, Optional, Union, cast from zoneinfo import available_timezones -from flask import Response, stream_with_context -from flask_restful import fields - from configs import dify_config from core.app.features.rate_limiting.rate_limit import RateLimitGenerator from core.file import helpers as file_helpers from extensions.ext_redis import redis_client +from flask import Response, stream_with_context +from flask_restful import fields from models.account import Account +from api.configs import dify_config + def run(script): return subprocess.getstatusoutput("source /root/.bashrc && " + script) @@ -297,3 +299,7 @@ def increment_rate_limit(self, email: str): redis_client.zadd(key, {current_time: current_time}) redis_client.expire(key, self.time_window * 2) + + +def get_current_datetime(): + return datetime.now(tz.utc).replace(tzinfo=None) diff --git a/api/models/account.py b/api/models/account.py index 932ba1da578b8a..d5ed206e959eca 100644 --- a/api/models/account.py +++ b/api/models/account.py @@ -267,3 +267,17 @@ class InvitationCode(db.Model): used_by_account_id = db.Column(StringUUID) deprecated_at = db.Column(db.DateTime) created_at = db.Column(db.DateTime, nullable=False, server_default=func.current_timestamp()) + + +class AccountDeletionLog(db.Model): + __tablename__ = "account_deletion_logs" + __table_args__ = ( + db.PrimaryKeyConstraint("id", name="account_deletion_log_pkey"), + ) + + id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()")) + email = db.Column(db.String(255), nullable=False) + reason = db.Column(db.Text, nullable=True) + + created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")) + updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")) diff --git a/api/services/account_deletion_log_service.py b/api/services/account_deletion_log_service.py new file mode 100644 index 00000000000000..f149bc12261422 --- /dev/null +++ b/api/services/account_deletion_log_service.py @@ -0,0 +1,34 @@ + +from datetime import timedelta + +from extensions.ext_database import db + +from api.configs import dify_config +from api.libs.helper import get_current_datetime +from api.models.account import AccountDeletionLog + + +class AccountDeletionLogService: + @staticmethod + def create_account_deletion_log(email, reason): + account_deletion_log = AccountDeletionLog() + account_deletion_log.email = email + account_deletion_log.reason = reason + account_deletion_log.updated_at = get_current_datetime() + + return account_deletion_log + + @staticmethod + def email_in_freeze(email): + log = db.session.query(AccountDeletionLog) \ + .filter(AccountDeletionLog.email == email) \ + .order_by(AccountDeletionLog.created_at.desc()) \ + .first() + + if not log: + return False + + # check if email is in freeze + if log.created_at + timedelta(days=dify_config.EMAIL_FREEZE_PERIOD_IN_DAYS) > get_current_datetime(): + return True + return False diff --git a/api/services/account_service.py b/api/services/account_service.py index 22b54a3ab87473..8cd8dbd08fded6 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -8,51 +8,44 @@ from hashlib import sha256 from typing import Any, Optional -from pydantic import BaseModel -from sqlalchemy import func -from werkzeug.exceptions import Unauthorized - from configs import dify_config from constants.languages import language_timezone_mapping, languages from events.tenant_event import tenant_was_created from extensions.ext_database import db from extensions.ext_redis import redis_client -from libs.helper import RateLimiter, TokenManager +from libs.helper import RateLimiter, TokenManager, generate_string from libs.passport import PassportService from libs.password import compare_password, hash_password, valid_password from libs.rsa import generate_key_pair -from models.account import ( - Account, - AccountIntegrate, - AccountStatus, - Tenant, - TenantAccountJoin, - TenantAccountJoinRole, - TenantAccountRole, - TenantStatus, -) +from models.account import (Account, AccountIntegrate, AccountStatus, Tenant, + TenantAccountJoin, TenantAccountJoinRole, + TenantAccountRole, TenantStatus) from models.model import DifySetup -from services.errors.account import ( - AccountAlreadyInTenantError, - AccountLoginError, - AccountNotFoundError, - AccountNotLinkTenantError, - AccountPasswordError, - AccountRegisterError, - CannotOperateSelfError, - CurrentPasswordIncorrectError, - InvalidActionError, - LinkAccountIntegrateError, - MemberNotInTenantError, - NoPermissionError, - RoleAlreadyAssignedError, - TenantNotFoundError, -) +from pydantic import BaseModel +from services.errors.account import (AccountAlreadyInTenantError, + AccountLoginError, AccountNotFoundError, + AccountNotLinkTenantError, + AccountPasswordError, + AccountRegisterError, + CannotOperateSelfError, + CurrentPasswordIncorrectError, + InvalidActionError, + LinkAccountIntegrateError, + MemberNotInTenantError, NoPermissionError, + RoleAlreadyAssignedError, + TenantNotFoundError) from services.errors.workspace import WorkSpaceNotAllowedCreateError from services.feature_service import FeatureService +from sqlalchemy import func from tasks.mail_email_code_login import send_email_code_login_mail_task from tasks.mail_invite_member_task import send_invite_member_mail_task from tasks.mail_reset_password_task import send_reset_password_mail_task +from werkzeug.exceptions import Unauthorized + +from api.services.account_deletion_log_service import AccountDeletionLogService +from api.tasks.delete_account_task import delete_account_task +from api.tasks.mail_account_deletion_task import \ + send_account_deletion_verification_code class TokenPair(BaseModel): @@ -201,6 +194,10 @@ def create_account( from controllers.console.error import AccountNotFound raise AccountNotFound() + + if AccountDeletionLogService.email_in_freeze(email): + raise AccountRegisterError("Email is in freeze.") + account = Account() account.email = email account.name = name @@ -240,6 +237,35 @@ def create_account_and_tenant( return account + @staticmethod + def generate_account_deletion_verification_code(account: Account) -> tuple[str, str]: + code = generate_string(6) + token = TokenManager.generate_token( + account=account, token_type="account_deletion", additional_data={"code": code} + ) + return code, token + + @staticmethod + def send_account_deletion_verification_email(account: Account, code: str): + language, email = account.interface_language, account.email + send_account_deletion_verification_code.delay(language=language, to=email, code=code) + + @staticmethod + def verify_account_deletion_code(token: str, code: str) -> bool: + token_data = TokenManager.get_token_data(token, "account_deletion") + if token_data is None: + return False + + if token_data["code"] != code: + return False + + return True + + @staticmethod + def delete_account(account: Account, reason="") -> None: + """Delete account. This method only adds a task to the queue for deletion.""" + delete_account_task.delay(account, reason) + @staticmethod def link_account_integrate(provider: str, open_id: str, account: Account) -> None: """Link account integrate""" @@ -349,7 +375,8 @@ def send_reset_password_email( account_email = account.email if account else email if cls.reset_password_rate_limiter.is_rate_limited(account_email): - from controllers.console.auth.error import PasswordResetRateLimitExceededError + from controllers.console.auth.error import \ + PasswordResetRateLimitExceededError raise PasswordResetRateLimitExceededError() @@ -373,12 +400,18 @@ def revoke_reset_password_token(cls, token: str): def get_reset_password_data(cls, token: str) -> Optional[dict[str, Any]]: return TokenManager.get_token_data(token, "reset_password") + @classmethod + def send_account_delete_verification_email(cls, account: Account, code: str): + language, email = account.interface_language, account.email + send_account_deletion_verification_code.delay(language=language, to=email, code=code) + @classmethod def send_email_code_login_email( cls, account: Optional[Account] = None, email: Optional[str] = None, language: Optional[str] = "en-US" ): if cls.email_code_login_rate_limiter.is_rate_limited(email): - from controllers.console.auth.error import EmailCodeLoginRateLimitExceededError + from controllers.console.auth.error import \ + EmailCodeLoginRateLimitExceededError raise EmailCodeLoginRateLimitExceededError() diff --git a/api/services/billing_service.py b/api/services/billing_service.py index 911d2346415ce5..8c836284a8bd19 100644 --- a/api/services/billing_service.py +++ b/api/services/billing_service.py @@ -1,7 +1,6 @@ import os import requests - from extensions.ext_database import db from models.account import TenantAccountJoin, TenantAccountRole @@ -59,3 +58,15 @@ def is_tenant_owner_or_admin(current_user): if not TenantAccountRole.is_privileged_role(join.role): raise ValueError("Only team owner or team admin can perform this action") + + @staticmethod + def delete_tenant_customer(cls, tenant_id: str): + """ Delete related customer in billing service. Used when tenant is deleted.""" + params = {"tenant_id": tenant_id} + headers = {"Content-Type": "application/json", "Billing-Api-Secret-Key": cls.secret_key} + + url = f"{cls.base_url}/customer" + response = requests.request("DELETE", url, params=params, headers=headers) + if response.status_code != 200: + raise Exception(f"Failed to delete customer for tenant {tenant_id}.") + return response.json() diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py new file mode 100644 index 00000000000000..ec14aab9d98b83 --- /dev/null +++ b/api/tasks/delete_account_task.py @@ -0,0 +1,67 @@ +import logging +import time + +import click +from celery import shared_task +from extensions.ext_database import db + +from api.models.account import (Account, Tenant, TenantAccountJoin, + TenantAccountJoinRole) +from api.services.account_deletion_log_service import AccountDeletionLogService +from api.services.billing_service import BillingService +from api.tasks.mail_account_deletion_task import (send_deletion_fail_task, + send_deletion_success_task) + +logger = logging.getLogger(__name__) + + +@shared_task(queue="dataset") +def delete_account_task(account: Account, reason: str): + logger.info(click.style("Start delete account task.", fg="green")) + start_at = time.perf_counter() + + logger.info(f"Start deletion of account {account.email}.") + try: + tenant_account_joins = db.session.query(TenantAccountJoin).filter(TenantAccountJoin.account_id == account.id).all() + with db.session.begin(): + # find all tenants this account belongs to + for ta in tenant_account_joins: + if ta.role == TenantAccountJoinRole.OWNER: + # dismiss all members of the tenant + members = db.session.query(TenantAccountJoin).filter(TenantAccountJoin.tenant_id == ta.tenant_id).delete() + logging.info(f"Dismissed {members} members from tenant {ta.tenant_id}.") + + # delete the tenant + db.session.query(Tenant).filter(Tenant.id == ta.tenant_id).delete() + logging.info(f"Deleted tenant {ta.tenant_id}.") + + # delete subscription + try: + BillingService.delete_tenant_customer(ta.tenant_id) + except Exception as e: + logging.error(f"Failed to delete subscription for tenant {ta.tenant_id}: {e}.") + raise + else: + # remove the account from tenant + db.session.delete(ta) + logging.info(f"Removed account {account.email} from tenant {ta.tenant_id}.") + + # delete the account + db.session.delete(account) + + # prepare account deletion log + account_deletion_log = AccountDeletionLogService.create_account_deletion_log(account.email, reason) + db.session.add(account_deletion_log) + + except Exception as e: + logging.error(f"Failed to delete account {account.email}.") + send_deletion_fail_task.delay(account.interface_language, account.email) + return + + send_deletion_success_task.delay(account.interface_language, account.email) + end_at = time.perf_counter() + logging.info( + click.style( + "Account deletion task completed: latency: {}".format(end_at - start_at), fg="green" + ) + ) diff --git a/api/tasks/mail_account_deletion_task.py b/api/tasks/mail_account_deletion_task.py new file mode 100644 index 00000000000000..561cb5046cbae6 --- /dev/null +++ b/api/tasks/mail_account_deletion_task.py @@ -0,0 +1,127 @@ +import logging +import time + +import click +from celery import shared_task +from extensions.ext_mail import mail +from flask import render_template + + +@shared_task(queue="mail") +def send_deletion_success_task(language, to): + """Send email to user regarding account deletion. + + Args: + log (AccountDeletionLog): Account deletion log object + """ + if not mail.is_inited(): + return + + logging.info( + click.style(f"Start send account deletion success email to {to}", fg="green") + ) + start_at = time.perf_counter() + + try: + if language == "zh-Hans": + html_content = render_template( + "delete_account_mail_template_zh-CN.html", + to=to, + # TODO: Add more template variables + ) + mail.send(to=to, subject="Dify 账户删除成功", html=html_content) + else: + html_content = render_template( + "delete_account_mail_template_en-US.html", + to=to, + # TODO: Add more template variables + ) + mail.send(to=to, subject="Dify Account Deleted", html=html_content) + + end_at = time.perf_counter() + logging.info( + click.style( + "Send account deletion success email to {}: latency: {}".format(to, end_at - start_at), fg="green" + ) + ) + except Exception: + logging.exception("Send account deletion success email to {} failed".format(to)) + + +@shared_task(queue="mail") +def send_deletion_fail_task(language, to): + """Send email to user regarding account deletion.""" + if not mail.is_inited(): + return + + logging.info( + click.style(f"Start send account deletion success email to {to}", fg="green") + ) + start_at = time.perf_counter() + + try: + if language == "zh-Hans": + html_content = render_template( + "delete_account_fail_mail_template_zh-CN.html", + to=to, + # TODO: Add more template variables + ) + mail.send(to=to, subject="Dify 账户删除失败", html=html_content) + else: + html_content = render_template( + "delete_account_fail_mail_template_en-US.html", + to=to, + # TODO: Add more template variables + ) + mail.send(to=to, subject="Dify Account Deletion Failed", html=html_content) + + end_at = time.perf_counter() + logging.info( + click.style( + "Send account deletion failed email to {}: latency: {}".format(to, end_at - start_at), fg="green" + ) + ) + except Exception: + logging.exception("Send account deletion success email to {} failed".format(to)) + + +@shared_task(queue="mail") +def send_account_deletion_verification_code(language, to, code): + """Send email to user regarding account deletion verification code. + + Args: + to (str): Recipient email address + code (str): Verification code + """ + if not mail.is_inited(): + return + + logging.info( + click.style(f"Start send account deletion verification code email to {to}", fg="green") + ) + start_at = time.perf_counter() + + try: + if language == "zh-Hans": + html_content = render_template( + "delete_account_verification_code_mail_template_zh-CN.html", + to=to, + code=code + ) + mail.send(to=to, subject="Dify 删除账户验证码", html=html_content) + else: + html_content = render_template( + "delete_account_verification_code_mail_template_en-US.html", + to=to, + code=code + ) + mail.send(to=to, subject="Dify Account Deletion Verification Code", html=html_content) + + end_at = time.perf_counter() + logging.info( + click.style( + "Send account deletion verification code email to {} succeeded: latency: {}".format(to, end_at - start_at), fg="green" + ) + ) + except Exception: + logging.exception("Send account deletion verification code email to {} failed".format(to)) From db66860a60b7ff7dacb0249eb49e47c872ae5b89 Mon Sep 17 00:00:00 2001 From: GareArc Date: Wed, 18 Dec 2024 23:59:26 -0500 Subject: [PATCH 02/34] fix: use get method for verification code --- api/controllers/console/workspace/account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/console/workspace/account.py b/api/controllers/console/workspace/account.py index 091b3342bffd6e..15b4a5efe4ec99 100644 --- a/api/controllers/console/workspace/account.py +++ b/api/controllers/console/workspace/account.py @@ -248,7 +248,7 @@ class AccountDeleteVerifyApi(Resource): @setup_required @login_required @account_initialization_required - def post(self): + def get(self): account = current_user try: From cffcca3f5be8de1a145bfebefd9b380b04c228f4 Mon Sep 17 00:00:00 2001 From: GareArc Date: Fri, 20 Dec 2024 20:55:08 -0500 Subject: [PATCH 03/34] feat: add rate limiter --- api/controllers/console/auth/error.py | 6 ++++++ api/services/account_service.py | 15 +++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/api/controllers/console/auth/error.py b/api/controllers/console/auth/error.py index e6e30c3c0b015f..8ef10c7bbb11cd 100644 --- a/api/controllers/console/auth/error.py +++ b/api/controllers/console/auth/error.py @@ -53,3 +53,9 @@ class EmailCodeLoginRateLimitExceededError(BaseHTTPException): error_code = "email_code_login_rate_limit_exceeded" description = "Too many login emails have been sent. Please try again in 5 minutes." code = 429 + + +class EmailCodeAccountDeletionRateLimitExceededError(BaseHTTPException): + error_code = "email_code_account_deletion_rate_limit_exceeded" + description = "Too many account deletion emails have been sent. Please try again in 5 minutes." + code = 429 diff --git a/api/services/account_service.py b/api/services/account_service.py index 8cd8dbd08fded6..22be504b3e979e 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -63,6 +63,9 @@ class AccountService: email_code_login_rate_limiter = RateLimiter( prefix="email_code_login_rate_limit", max_attempts=1, time_window=60 * 1 ) + email_code_account_deletion_rate_limiter = RateLimiter( + prefix="email_code_account_deletion_rate_limit", max_attempts=1, time_window=60 * 1 + ) LOGIN_MAX_ERROR_LIMITS = 5 @staticmethod @@ -245,11 +248,19 @@ def generate_account_deletion_verification_code(account: Account) -> tuple[str, ) return code, token - @staticmethod - def send_account_deletion_verification_email(account: Account, code: str): + @classmethod + def send_account_deletion_verification_email(cls, account: Account, code: str): + if cls.email_code_account_deletion_rate_limiter.is_rate_limited(email): + from controllers.console.auth.error import \ + EmailCodeAccountDeletionRateLimitExceededError + + raise EmailCodeAccountDeletionRateLimitExceededError() + language, email = account.interface_language, account.email send_account_deletion_verification_code.delay(language=language, to=email, code=code) + cls.email_code_account_deletion_rate_limiter.increment_rate_limit(email) + @staticmethod def verify_account_deletion_code(token: str, code: str) -> bool: token_data = TokenManager.get_token_data(token, "account_deletion") From 56fe09ee2247b0cc24e58715a21626ba5a4b808a Mon Sep 17 00:00:00 2001 From: GareArc Date: Fri, 20 Dec 2024 21:37:51 -0500 Subject: [PATCH 04/34] feat: add migration --- api/libs/helper.py | 2 -- ...82e52119c70_added_account_deletion_logs.py | 36 +++++++++++++++++++ api/services/account_deletion_log_service.py | 7 ++-- api/services/account_service.py | 9 +++-- api/tasks/delete_account_task.py | 13 ++++--- 5 files changed, 49 insertions(+), 18 deletions(-) create mode 100644 api/migrations/versions/2024_12_20_2136-582e52119c70_added_account_deletion_logs.py diff --git a/api/libs/helper.py b/api/libs/helper.py index 62bc9fa98ded10..349a75d73e6737 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -21,8 +21,6 @@ from flask_restful import fields from models.account import Account -from api.configs import dify_config - def run(script): return subprocess.getstatusoutput("source /root/.bashrc && " + script) diff --git a/api/migrations/versions/2024_12_20_2136-582e52119c70_added_account_deletion_logs.py b/api/migrations/versions/2024_12_20_2136-582e52119c70_added_account_deletion_logs.py new file mode 100644 index 00000000000000..17fd5dbe3a8c0f --- /dev/null +++ b/api/migrations/versions/2024_12_20_2136-582e52119c70_added_account_deletion_logs.py @@ -0,0 +1,36 @@ +"""added account_deletion_logs + +Revision ID: 582e52119c70 +Revises: e1944c35e15e +Create Date: 2024-12-20 21:36:46.856033 + +""" +from alembic import op +import models as models +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '582e52119c70' +down_revision = 'e1944c35e15e' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('account_deletion_logs', + sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('email', sa.String(length=255), nullable=False), + sa.Column('reason', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.PrimaryKeyConstraint('id', name='account_deletion_log_pkey') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('account_deletion_logs') + # ### end Alembic commands ### diff --git a/api/services/account_deletion_log_service.py b/api/services/account_deletion_log_service.py index f149bc12261422..205b16a6b0afe8 100644 --- a/api/services/account_deletion_log_service.py +++ b/api/services/account_deletion_log_service.py @@ -1,11 +1,10 @@ from datetime import timedelta +from configs import dify_config from extensions.ext_database import db - -from api.configs import dify_config -from api.libs.helper import get_current_datetime -from api.models.account import AccountDeletionLog +from libs.helper import get_current_datetime +from models.account import AccountDeletionLog class AccountDeletionLogService: diff --git a/api/services/account_service.py b/api/services/account_service.py index 22be504b3e979e..6fe2860505923a 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -22,6 +22,7 @@ TenantAccountRole, TenantStatus) from models.model import DifySetup from pydantic import BaseModel +from services.account_deletion_log_service import AccountDeletionLogService from services.errors.account import (AccountAlreadyInTenantError, AccountLoginError, AccountNotFoundError, AccountNotLinkTenantError, @@ -37,16 +38,14 @@ from services.errors.workspace import WorkSpaceNotAllowedCreateError from services.feature_service import FeatureService from sqlalchemy import func +from tasks.delete_account_task import delete_account_task +from tasks.mail_account_deletion_task import \ + send_account_deletion_verification_code from tasks.mail_email_code_login import send_email_code_login_mail_task from tasks.mail_invite_member_task import send_invite_member_mail_task from tasks.mail_reset_password_task import send_reset_password_mail_task from werkzeug.exceptions import Unauthorized -from api.services.account_deletion_log_service import AccountDeletionLogService -from api.tasks.delete_account_task import delete_account_task -from api.tasks.mail_account_deletion_task import \ - send_account_deletion_verification_code - class TokenPair(BaseModel): access_token: str diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index ec14aab9d98b83..484460fb01e51e 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -4,13 +4,12 @@ import click from celery import shared_task from extensions.ext_database import db - -from api.models.account import (Account, Tenant, TenantAccountJoin, - TenantAccountJoinRole) -from api.services.account_deletion_log_service import AccountDeletionLogService -from api.services.billing_service import BillingService -from api.tasks.mail_account_deletion_task import (send_deletion_fail_task, - send_deletion_success_task) +from models.account import (Account, Tenant, TenantAccountJoin, + TenantAccountJoinRole) +from services.account_deletion_log_service import AccountDeletionLogService +from services.billing_service import BillingService +from tasks.mail_account_deletion_task import (send_deletion_fail_task, + send_deletion_success_task) logger = logging.getLogger(__name__) From 5647fe7260c0455f98854a4a6992e22e93a0794f Mon Sep 17 00:00:00 2001 From: GareArc Date: Sat, 21 Dec 2024 14:05:33 -0500 Subject: [PATCH 05/34] update --- api/configs/feature/__init__.py | 8 ++++++++ api/tasks/delete_account_task.py | 28 ++++++++++++++-------------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index c9fd24e9c197dc..0730c8a0b569a9 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -766,6 +766,13 @@ class RegisterConfig(BaseSettings): ) +class AccountConfig(BaseSettings): + ACCOUNT_DELETION_TOKEN_EXPIRY_MINUTES: PositiveInt = Field( + description="Duration in minutes for which a account deletion token remains valid", + default=5, + ) + + class FeatureConfig( # place the configs in alphabet order AppExecutionConfig, @@ -794,6 +801,7 @@ class FeatureConfig( WorkspaceConfig, LoginConfig, RegisterConfig, + AccountConfig, # hosted services config HostedServiceConfig, CeleryBeatConfig, diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index 484460fb01e51e..c72d88fd404d2d 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -20,11 +20,11 @@ def delete_account_task(account: Account, reason: str): start_at = time.perf_counter() logger.info(f"Start deletion of account {account.email}.") - try: - tenant_account_joins = db.session.query(TenantAccountJoin).filter(TenantAccountJoin.account_id == account.id).all() - with db.session.begin(): - # find all tenants this account belongs to - for ta in tenant_account_joins: + # find all tenants this account belongs to + tenant_account_joins = db.session.query(TenantAccountJoin).filter(TenantAccountJoin.account_id == account.id).all() + for ta in tenant_account_joins: + try: + with db.session.begin(): if ta.role == TenantAccountJoinRole.OWNER: # dismiss all members of the tenant members = db.session.query(TenantAccountJoin).filter(TenantAccountJoin.tenant_id == ta.tenant_id).delete() @@ -45,17 +45,17 @@ def delete_account_task(account: Account, reason: str): db.session.delete(ta) logging.info(f"Removed account {account.email} from tenant {ta.tenant_id}.") - # delete the account - db.session.delete(account) + # delete the account + db.session.delete(account) - # prepare account deletion log - account_deletion_log = AccountDeletionLogService.create_account_deletion_log(account.email, reason) - db.session.add(account_deletion_log) + # prepare account deletion log + account_deletion_log = AccountDeletionLogService.create_account_deletion_log(account.email, reason) + db.session.add(account_deletion_log) - except Exception as e: - logging.error(f"Failed to delete account {account.email}.") - send_deletion_fail_task.delay(account.interface_language, account.email) - return + except Exception as e: + logging.error(f"Failed to delete account {account.email}.") + send_deletion_fail_task.delay(account.interface_language, account.email) + return send_deletion_success_task.delay(account.interface_language, account.email) end_at = time.perf_counter() From f7ca0a2ec0a74986308a255b2287207f4d09e0d7 Mon Sep 17 00:00:00 2001 From: GareArc Date: Sat, 21 Dec 2024 14:13:19 -0500 Subject: [PATCH 06/34] fix: token wrong position --- api/services/account_service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/services/account_service.py b/api/services/account_service.py index 6fe2860505923a..f07e88599f15ca 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -245,7 +245,8 @@ def generate_account_deletion_verification_code(account: Account) -> tuple[str, token = TokenManager.generate_token( account=account, token_type="account_deletion", additional_data={"code": code} ) - return code, token + logging.info(f"Account {account.id} generated account deletion verification code {code} with token {token}") + return token, code @classmethod def send_account_deletion_verification_email(cls, account: Account, code: str): From 4f393222971ed8f712c8f293df490028b55cfaf3 Mon Sep 17 00:00:00 2001 From: GareArc Date: Sat, 21 Dec 2024 14:20:40 -0500 Subject: [PATCH 07/34] fix: params of celery function should be serializable --- api/services/account_service.py | 2 +- api/tasks/delete_account_task.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api/services/account_service.py b/api/services/account_service.py index f07e88599f15ca..69b88dc4eb5073 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -275,7 +275,7 @@ def verify_account_deletion_code(token: str, code: str) -> bool: @staticmethod def delete_account(account: Account, reason="") -> None: """Delete account. This method only adds a task to the queue for deletion.""" - delete_account_task.delay(account, reason) + delete_account_task.delay(account.id, reason) @staticmethod def link_account_integrate(provider: str, open_id: str, account: Account) -> None: diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index c72d88fd404d2d..ffea9d971e6491 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -15,7 +15,8 @@ @shared_task(queue="dataset") -def delete_account_task(account: Account, reason: str): +def delete_account_task(account_id, reason: str): + account = db.session.query(Account).filter(Account.id == account_id).first() logger.info(click.style("Start delete account task.", fg="green")) start_at = time.perf_counter() From 4be59e120566960e2824400652e88e9e55a651e8 Mon Sep 17 00:00:00 2001 From: GareArc Date: Sat, 21 Dec 2024 15:03:16 -0500 Subject: [PATCH 08/34] fix: db session error --- api/controllers/console/workspace/account.py | 5 +- api/services/account_service.py | 1 - api/tasks/delete_account_task.py | 50 ++++++++++---------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/api/controllers/console/workspace/account.py b/api/controllers/console/workspace/account.py index 15b4a5efe4ec99..412b8a8b212f54 100644 --- a/api/controllers/console/workspace/account.py +++ b/api/controllers/console/workspace/account.py @@ -22,7 +22,6 @@ from services.account_service import AccountService from services.errors.account import \ CurrentPasswordIncorrectError as ServiceCurrentPasswordIncorrectError -from services.errors.account import RateLimitExceededError class AccountInitApi(Resource): @@ -254,8 +253,8 @@ def get(self): try: token, code = AccountService.generate_account_deletion_verification_code(account) AccountService.send_account_delete_verification_email(account, code) - except RateLimitExceededError: - return {"result": "fail", "error": "Rate limit exceeded."}, 429 + except Exception as e: + return {"result": "fail", "error": str(e)}, 429 return {"result": "success", "data": token} diff --git a/api/services/account_service.py b/api/services/account_service.py index 69b88dc4eb5073..fe1ea9bc1b3de1 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -245,7 +245,6 @@ def generate_account_deletion_verification_code(account: Account) -> tuple[str, token = TokenManager.generate_token( account=account, token_type="account_deletion", additional_data={"code": code} ) - logging.info(f"Account {account.id} generated account deletion verification code {code} with token {token}") return token, code @classmethod diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index ffea9d971e6491..bc13830f3a86ab 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -25,36 +25,38 @@ def delete_account_task(account_id, reason: str): tenant_account_joins = db.session.query(TenantAccountJoin).filter(TenantAccountJoin.account_id == account.id).all() for ta in tenant_account_joins: try: - with db.session.begin(): - if ta.role == TenantAccountJoinRole.OWNER: - # dismiss all members of the tenant - members = db.session.query(TenantAccountJoin).filter(TenantAccountJoin.tenant_id == ta.tenant_id).delete() - logging.info(f"Dismissed {members} members from tenant {ta.tenant_id}.") + if ta.role == TenantAccountJoinRole.OWNER: + # dismiss all members of the tenant + members = db.session.query(TenantAccountJoin).filter(TenantAccountJoin.tenant_id == ta.tenant_id).delete() + logging.info(f"Dismissed {members} members from tenant {ta.tenant_id}.") - # delete the tenant - db.session.query(Tenant).filter(Tenant.id == ta.tenant_id).delete() - logging.info(f"Deleted tenant {ta.tenant_id}.") + # delete the tenant + db.session.query(Tenant).filter(Tenant.id == ta.tenant_id).delete() + logging.info(f"Deleted tenant {ta.tenant_id}.") - # delete subscription - try: - BillingService.delete_tenant_customer(ta.tenant_id) - except Exception as e: - logging.error(f"Failed to delete subscription for tenant {ta.tenant_id}: {e}.") - raise - else: - # remove the account from tenant - db.session.delete(ta) - logging.info(f"Removed account {account.email} from tenant {ta.tenant_id}.") + # delete subscription + try: + BillingService.delete_tenant_customer(ta.tenant_id) + except Exception as e: + logging.error(f"Failed to delete subscription for tenant {ta.tenant_id}: {e}.") + raise + else: + # remove the account from tenant + db.session.delete(ta) + logging.info(f"Removed account {account.email} from tenant {ta.tenant_id}.") - # delete the account - db.session.delete(account) + # delete the account + db.session.delete(account) - # prepare account deletion log - account_deletion_log = AccountDeletionLogService.create_account_deletion_log(account.email, reason) - db.session.add(account_deletion_log) + # prepare account deletion log + account_deletion_log = AccountDeletionLogService.create_account_deletion_log(account.email, reason) + db.session.add(account_deletion_log) + + db.session.commit() except Exception as e: - logging.error(f"Failed to delete account {account.email}.") + db.session.rollback() + logging.error(f"Failed to delete account {account.email}: {e}.") send_deletion_fail_task.delay(account.interface_language, account.email) return From ed97f768f3f9cb8e3807c68f4cf9a5e04a403806 Mon Sep 17 00:00:00 2001 From: GareArc Date: Sat, 21 Dec 2024 23:11:43 -0500 Subject: [PATCH 09/34] fix: refactor task --- api/tasks/delete_account_task.py | 114 ++++++++++++++++++------------- 1 file changed, 68 insertions(+), 46 deletions(-) diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index bc13830f3a86ab..b6599b7395ddc9 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -17,53 +17,75 @@ @shared_task(queue="dataset") def delete_account_task(account_id, reason: str): account = db.session.query(Account).filter(Account.id == account_id).first() - logger.info(click.style("Start delete account task.", fg="green")) + if not account: + logging.error(f"Account with ID {account_id} not found.") + return + + logger.info(click.style(f"Start deletion task for account {account.email}.", fg="green")) start_at = time.perf_counter() - logger.info(f"Start deletion of account {account.email}.") - # find all tenants this account belongs to - tenant_account_joins = db.session.query(TenantAccountJoin).filter(TenantAccountJoin.account_id == account.id).all() - for ta in tenant_account_joins: - try: - if ta.role == TenantAccountJoinRole.OWNER: - # dismiss all members of the tenant - members = db.session.query(TenantAccountJoin).filter(TenantAccountJoin.tenant_id == ta.tenant_id).delete() - logging.info(f"Dismissed {members} members from tenant {ta.tenant_id}.") - - # delete the tenant - db.session.query(Tenant).filter(Tenant.id == ta.tenant_id).delete() - logging.info(f"Deleted tenant {ta.tenant_id}.") - - # delete subscription - try: - BillingService.delete_tenant_customer(ta.tenant_id) - except Exception as e: - logging.error(f"Failed to delete subscription for tenant {ta.tenant_id}: {e}.") - raise - else: - # remove the account from tenant - db.session.delete(ta) - logging.info(f"Removed account {account.email} from tenant {ta.tenant_id}.") - - # delete the account - db.session.delete(account) - - # prepare account deletion log - account_deletion_log = AccountDeletionLogService.create_account_deletion_log(account.email, reason) - db.session.add(account_deletion_log) - - db.session.commit() - - except Exception as e: - db.session.rollback() - logging.error(f"Failed to delete account {account.email}: {e}.") - send_deletion_fail_task.delay(account.interface_language, account.email) - return - - send_deletion_success_task.delay(account.interface_language, account.email) - end_at = time.perf_counter() - logging.info( - click.style( - "Account deletion task completed: latency: {}".format(end_at - start_at), fg="green" + try: + _process_account_deletion(account, reason) + db.session.commit() + send_deletion_success_task.delay(account.interface_language, account.email) + logger.info( + click.style( + f"Account deletion task completed for {account.email}: latency: {time.perf_counter() - start_at}", + fg="green", + ) ) + except Exception as e: + db.session.rollback() + logging.error(f"Failed to delete account {account.email}: {e}.") + send_deletion_fail_task.delay(account.interface_language, account.email) + raise + + +def _process_account_deletion(account, reason): + # Fetch all tenant-account associations + tenant_account_joins = db.session.query(TenantAccountJoin).filter( + TenantAccountJoin.account_id == account.id + ).all() + + for ta in tenant_account_joins: + if ta.role == TenantAccountJoinRole.OWNER: + _handle_owner_tenant_deletion(ta) + else: + _remove_account_from_tenant(ta, account.email) + + account_deletion_log = AccountDeletionLogService.create_account_deletion_log( + account.email, reason ) + db.session.add(account_deletion_log) + db.session.delete(account) + logger.info(f"Account {account.email} successfully deleted.") + + +def _handle_owner_tenant_deletion(ta): + """Handle deletion of a tenant where the account is an owner.""" + tenant_id = ta.tenant_id + + # Dismiss all tenant members + members_deleted = db.session.query(TenantAccountJoin).filter( + TenantAccountJoin.tenant_id == tenant_id + ).delete() + logger.info(f"Dismissed {members_deleted} members from tenant {tenant_id}.") + + # Delete the tenant + db.session.query(Tenant).filter(Tenant.id == tenant_id).delete() + logger.info(f"Deleted tenant {tenant_id}.") + + # Delete subscription + try: + BillingService.delete_tenant_customer(tenant_id) + logger.info(f"Subscription for tenant {tenant_id} deleted successfully.") + except Exception as e: + logger.error(f"Failed to delete subscription for tenant {tenant_id}: {e}.") + raise + + +def _remove_account_from_tenant(ta, email): + """Remove the account from a tenant.""" + tenant_id = ta.tenant_id + db.session.delete(ta) + logger.info(f"Removed account {email} from tenant {tenant_id}.") From 3ee1de6017e6af067be603cd221da50e5bc94209 Mon Sep 17 00:00:00 2001 From: GareArc Date: Sun, 22 Dec 2024 05:03:37 +0000 Subject: [PATCH 10/34] minor fix --- api/services/billing_service.py | 2 +- api/tasks/delete_account_task.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/services/billing_service.py b/api/services/billing_service.py index 8c836284a8bd19..ea7e23e0f5a347 100644 --- a/api/services/billing_service.py +++ b/api/services/billing_service.py @@ -59,7 +59,7 @@ def is_tenant_owner_or_admin(current_user): if not TenantAccountRole.is_privileged_role(join.role): raise ValueError("Only team owner or team admin can perform this action") - @staticmethod + @classmethod def delete_tenant_customer(cls, tenant_id: str): """ Delete related customer in billing service. Used when tenant is deleted.""" params = {"tenant_id": tenant_id} diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index b6599b7395ddc9..6f746177ed8546 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -48,7 +48,7 @@ def _process_account_deletion(account, reason): ).all() for ta in tenant_account_joins: - if ta.role == TenantAccountJoinRole.OWNER: + if ta.role == TenantAccountJoinRole.OWNER.value: _handle_owner_tenant_deletion(ta) else: _remove_account_from_tenant(ta, account.email) From 424c7ee3d0086836fc923d60ca6f52c6d589f4c4 Mon Sep 17 00:00:00 2001 From: NFish Date: Mon, 23 Dec 2024 10:39:43 +0800 Subject: [PATCH 11/34] feat: add email templates --- ...ete_account_code_email_template_en-US.html | 74 +++++++++++++++++++ .../delete_account_code_email_zh-CN.html | 74 +++++++++++++++++++ .../delete_account_fail_template_en-US.html | 70 ++++++++++++++++++ .../delete_account_fail_template_zh-CN.html | 70 ++++++++++++++++++ ...delete_account_success_template_en-US.html | 70 ++++++++++++++++++ ...delete_account_success_template_zh-CN.html | 70 ++++++++++++++++++ 6 files changed, 428 insertions(+) create mode 100644 api/templates/delete_account_code_email_template_en-US.html create mode 100644 api/templates/delete_account_code_email_zh-CN.html create mode 100644 api/templates/delete_account_fail_template_en-US.html create mode 100644 api/templates/delete_account_fail_template_zh-CN.html create mode 100644 api/templates/delete_account_success_template_en-US.html create mode 100644 api/templates/delete_account_success_template_zh-CN.html diff --git a/api/templates/delete_account_code_email_template_en-US.html b/api/templates/delete_account_code_email_template_en-US.html new file mode 100644 index 00000000000000..5a1649402cefcb --- /dev/null +++ b/api/templates/delete_account_code_email_template_en-US.html @@ -0,0 +1,74 @@ + + + + + + +
+
+ + Dify Logo +
+

Dify 的删除账户验证码

+

复制并粘贴此验证码,注意验证码仅在接下来的 5 分钟内有效。

+
+ {{code}} +
+

如果您没有请求删除账户,请不要担心。您可以安全地忽略此电子邮件。

+
+ + diff --git a/api/templates/delete_account_code_email_zh-CN.html b/api/templates/delete_account_code_email_zh-CN.html new file mode 100644 index 00000000000000..b7f236be3c2da1 --- /dev/null +++ b/api/templates/delete_account_code_email_zh-CN.html @@ -0,0 +1,74 @@ + + + + + + +
+
+ + Dify Logo +
+

Delete your Dify account

+

Copy and paste this code, this code will only be valid for the next 5 minutes.

+
+ {{code}} +
+

If you didn't request, don't worry. You can safely ignore this email.

+
+ + diff --git a/api/templates/delete_account_fail_template_en-US.html b/api/templates/delete_account_fail_template_en-US.html new file mode 100644 index 00000000000000..84a724c582b8e8 --- /dev/null +++ b/api/templates/delete_account_fail_template_en-US.html @@ -0,0 +1,70 @@ + + + + + + +
+
+ + Dify Logo +
+

Ops! Your account delete failed

+

You recently deleted your {{email}} Dify account. It's done processing and is now deleted. As a reminder, you no longer have access to all the workspaces you used to be in with this account.

+
+ + diff --git a/api/templates/delete_account_fail_template_zh-CN.html b/api/templates/delete_account_fail_template_zh-CN.html new file mode 100644 index 00000000000000..e9a3234c121d72 --- /dev/null +++ b/api/templates/delete_account_fail_template_zh-CN.html @@ -0,0 +1,70 @@ + + + + + + +
+
+ + Dify Logo +
+

账户删除失败

+

You recently deleted your {{email}} Dify account. It's done processing and is now deleted. As a reminder, you no longer have access to all the workspaces you used to be in with this account.

+
+ + diff --git a/api/templates/delete_account_success_template_en-US.html b/api/templates/delete_account_success_template_en-US.html new file mode 100644 index 00000000000000..30e592bdc1b9fa --- /dev/null +++ b/api/templates/delete_account_success_template_en-US.html @@ -0,0 +1,70 @@ + + + + + + +
+
+ + Dify Logo +
+

Your account has finished deleting

+

You recently deleted your {{email}} Dify account. It's done processing and is now deleted. As a reminder, you no longer have access to all the workspaces you used to be in with this account.

+
+ + diff --git a/api/templates/delete_account_success_template_zh-CN.html b/api/templates/delete_account_success_template_zh-CN.html new file mode 100644 index 00000000000000..622c1d90f07a8a --- /dev/null +++ b/api/templates/delete_account_success_template_zh-CN.html @@ -0,0 +1,70 @@ + + + + + + +
+
+ + Dify Logo +
+

您的账户已删除

+

您最近删除了您的 {{email}} Dify 账户。该操作已完成,账户现已删除。请注意,您将不再能够访问此账户中曾经加入的所有工作区。

+
+ + From cd437efbca2e96633f224c160edba4120de6b355 Mon Sep 17 00:00:00 2001 From: NFish Date: Mon, 23 Dec 2024 11:29:04 +0800 Subject: [PATCH 12/34] fix: update emial template style --- .../delete_account_success_template_en-US.html | 16 +++++++++++----- .../delete_account_success_template_zh-CN.html | 15 +++++++++++---- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/api/templates/delete_account_success_template_en-US.html b/api/templates/delete_account_success_template_en-US.html index 30e592bdc1b9fa..d10a913da685d2 100644 --- a/api/templates/delete_account_success_template_en-US.html +++ b/api/templates/delete_account_success_template_en-US.html @@ -33,10 +33,10 @@ line-height: 28.8px; } .description { - font-size: 13px; - line-height: 16px; - color: #676f83; - margin-top: 12px; + color: #354052; + font-weight: 400; + line-height: 20px; + font-size: 14px; } .code-content { padding: 16px 32px; @@ -55,6 +55,12 @@ color: #676f83; font-size: 13px; } + .email{ + color: #354052; + font-weight: 600; + line-height: 20px; + font-size: 14px; + } @@ -64,7 +70,7 @@ Dify Logo

Your account has finished deleting

-

You recently deleted your {{email}} Dify account. It's done processing and is now deleted. As a reminder, you no longer have access to all the workspaces you used to be in with this account.

+

You recently deleted your Dify account. It's done processing and is now deleted. As a reminder, you no longer have access to all the workspaces you used to be in with this account.

diff --git a/api/templates/delete_account_success_template_zh-CN.html b/api/templates/delete_account_success_template_zh-CN.html index 622c1d90f07a8a..7381dba81c19d1 100644 --- a/api/templates/delete_account_success_template_zh-CN.html +++ b/api/templates/delete_account_success_template_zh-CN.html @@ -33,9 +33,10 @@ line-height: 28.8px; } .description { - font-size: 13px; - line-height: 16px; - color: #676f83; + color: #354052; + font-weight: 400; + line-height: 20px; + font-size: 14px; margin-top: 12px; } .code-content { @@ -55,6 +56,12 @@ color: #676f83; font-size: 13px; } + .email{ + color: #354052; + font-weight: 600; + line-height: 20px; + font-size: 14px; + } @@ -64,7 +71,7 @@ Dify Logo

您的账户已删除

-

您最近删除了您的 {{email}} Dify 账户。该操作已完成,账户现已删除。请注意,您将不再能够访问此账户中曾经加入的所有工作区。

+

您最近删除了您的 Dify 账户 。系统已完成处理,现在此账户已被删除。请注意,您不再有权访问此帐户曾经所在的所有空间。

From 6d93a09c807ce9d5af8088452480b1b518545277 Mon Sep 17 00:00:00 2001 From: GareArc Date: Mon, 23 Dec 2024 00:28:49 -0500 Subject: [PATCH 13/34] fix: add detailed deletion log --- api/libs/helper.py | 42 ++++++++++- api/models/account.py | 18 +++++ api/services/account_deletion_log_service.py | 10 +-- api/services/billing_service.py | 10 +-- api/tasks/delete_account_task.py | 49 +++++++++---- api/tasks/mail_account_deletion_task.py | 53 +++----------- ...l => delete_account_code_email_en-US.html} | 0 ...te_account_code_email_template_zh-CN.html} | 0 .../delete_account_fail_template_en-US.html | 70 ------------------- .../delete_account_fail_template_zh-CN.html | 70 ------------------- 10 files changed, 109 insertions(+), 213 deletions(-) rename api/templates/{delete_account_code_email_zh-CN.html => delete_account_code_email_en-US.html} (100%) rename api/templates/{delete_account_code_email_template_en-US.html => delete_account_code_email_template_zh-CN.html} (100%) delete mode 100644 api/templates/delete_account_fail_template_en-US.html delete mode 100644 api/templates/delete_account_fail_template_zh-CN.html diff --git a/api/libs/helper.py b/api/libs/helper.py index 349a75d73e6737..56d4ab395cd969 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -7,7 +7,7 @@ import time import uuid from collections.abc import Generator, Mapping -from datetime import datetime +from datetime import date, datetime from datetime import timezone as tz from hashlib import sha256 from typing import Any, Optional, Union, cast @@ -301,3 +301,43 @@ def increment_rate_limit(self, email: str): def get_current_datetime(): return datetime.now(tz.utc).replace(tzinfo=None) + + +def to_json(obj): + """ + Convert a Python object to a compact JSON string. + + Supports basic types, datetime, UUID, SQLAlchemy models, and nested structures. + + Args: + obj: The object to convert. + + Returns: + A compact JSON string representation of the object. + """ + def serialize(obj, seen=None): + if seen is None: + seen = set() # Track seen objects to prevent recursion + + if id(obj) in seen: + return None # Avoid circular references + + seen.add(id(obj)) + + if isinstance(obj, (datetime, date)): + return obj.isoformat() # Convert datetime and date to ISO 8601 string + elif isinstance(obj, uuid.UUID): + return str(obj) # Convert UUID to string + elif isinstance(obj, (int, float, str, bool, type(None))): + return obj # Leave basic types unchanged + elif isinstance(obj, dict): + return {key: serialize(value, seen) for key, value in obj.items() if value is not None} # Exclude None values + elif isinstance(obj, list): + return [serialize(item, seen) for item in obj] # Recursively serialize lists + else: + try: + return str(obj) # Fallback to string for unsupported types + except Exception: + return None # Return None if serialization fails + + return json.dumps(serialize(obj), separators=(',', ':')) diff --git a/api/models/account.py b/api/models/account.py index d5ed206e959eca..b6843e79067ba9 100644 --- a/api/models/account.py +++ b/api/models/account.py @@ -278,6 +278,24 @@ class AccountDeletionLog(db.Model): id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()")) email = db.Column(db.String(255), nullable=False) reason = db.Column(db.Text, nullable=True) + account_id = db.Column(StringUUID, nullable=False) + snapshot = db.Column(db.Text, nullable=False) + + created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")) + updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")) + + +class AccountDeletionLogDetail(db.Model): + __tablename__ = "account_deletion_log_details" + __table_args__ = ( + db.PrimaryKeyConstraint("id", name="account_deletion_log_detail_pkey"), + ) + + id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()")) + account_id = db.Column(StringUUID, nullable=False) + tenant_id = db.Column(StringUUID, nullable=False) + role = db.Column(db.String(16), nullable=False) + snapshot = db.Column(db.Text, nullable=False) created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")) updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")) diff --git a/api/services/account_deletion_log_service.py b/api/services/account_deletion_log_service.py index 205b16a6b0afe8..2dc729c829c3e9 100644 --- a/api/services/account_deletion_log_service.py +++ b/api/services/account_deletion_log_service.py @@ -3,16 +3,18 @@ from configs import dify_config from extensions.ext_database import db -from libs.helper import get_current_datetime -from models.account import AccountDeletionLog +from libs.helper import get_current_datetime, to_json +from models.account import Account, AccountDeletionLog class AccountDeletionLogService: @staticmethod - def create_account_deletion_log(email, reason): + def create_account_deletion_log(account: Account, reason): account_deletion_log = AccountDeletionLog() - account_deletion_log.email = email + account_deletion_log.email = account.email account_deletion_log.reason = reason + account_deletion_log.account_id = account.id + account_deletion_log.snapshot = to_json(account) account_deletion_log.updated_at = get_current_datetime() return account_deletion_log diff --git a/api/services/billing_service.py b/api/services/billing_service.py index ea7e23e0f5a347..f8d9bac7cad984 100644 --- a/api/services/billing_service.py +++ b/api/services/billing_service.py @@ -60,13 +60,7 @@ def is_tenant_owner_or_admin(current_user): raise ValueError("Only team owner or team admin can perform this action") @classmethod - def delete_tenant_customer(cls, tenant_id: str): + def unsubscripbe_tenant_customer(cls, tenant_id: str): """ Delete related customer in billing service. Used when tenant is deleted.""" params = {"tenant_id": tenant_id} - headers = {"Content-Type": "application/json", "Billing-Api-Secret-Key": cls.secret_key} - - url = f"{cls.base_url}/customer" - response = requests.request("DELETE", url, params=params, headers=headers) - if response.status_code != 200: - raise Exception(f"Failed to delete customer for tenant {tenant_id}.") - return response.json() + return cls._send_request("DELETE", "/subscription", params=params) diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index 6f746177ed8546..2a702c39180f1f 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -4,12 +4,13 @@ import click from celery import shared_task from extensions.ext_database import db -from models.account import (Account, Tenant, TenantAccountJoin, - TenantAccountJoinRole) +from models.account import (Account, AccountDeletionLogDetail, + TenantAccountJoin, TenantAccountJoinRole) from services.account_deletion_log_service import AccountDeletionLogService from services.billing_service import BillingService -from tasks.mail_account_deletion_task import (send_deletion_fail_task, - send_deletion_success_task) +from tasks.mail_account_deletion_task import send_deletion_success_task + +from api.libs.helper import to_json logger = logging.getLogger(__name__) @@ -37,7 +38,6 @@ def delete_account_task(account_id, reason: str): except Exception as e: db.session.rollback() logging.error(f"Failed to delete account {account.email}: {e}.") - send_deletion_fail_task.delay(account.interface_language, account.email) raise @@ -54,38 +54,57 @@ def _process_account_deletion(account, reason): _remove_account_from_tenant(ta, account.email) account_deletion_log = AccountDeletionLogService.create_account_deletion_log( - account.email, reason + account, reason ) db.session.add(account_deletion_log) db.session.delete(account) logger.info(f"Account {account.email} successfully deleted.") -def _handle_owner_tenant_deletion(ta): +def _handle_owner_tenant_deletion(ta: TenantAccountJoin): """Handle deletion of a tenant where the account is an owner.""" tenant_id = ta.tenant_id # Dismiss all tenant members - members_deleted = db.session.query(TenantAccountJoin).filter( + members_to_dismiss = db.session.query(TenantAccountJoin).filter( TenantAccountJoin.tenant_id == tenant_id - ).delete() - logger.info(f"Dismissed {members_deleted} members from tenant {tenant_id}.") - - # Delete the tenant - db.session.query(Tenant).filter(Tenant.id == tenant_id).delete() - logger.info(f"Deleted tenant {tenant_id}.") + ).all() + for member in members_to_dismiss: + db.session.delete(member) + logger.info(f"Dismissed {len(members_to_dismiss)} members from tenant {tenant_id}.") # Delete subscription try: - BillingService.delete_tenant_customer(tenant_id) + BillingService.unsubscripbe_tenant_customer(tenant_id) logger.info(f"Subscription for tenant {tenant_id} deleted successfully.") except Exception as e: logger.error(f"Failed to delete subscription for tenant {tenant_id}: {e}.") raise + # create account deletion log detail + account_deletion_log_detail = AccountDeletionLogDetail() + account_deletion_log_detail.account_id = ta.account_id + account_deletion_log_detail.tenant_id = tenant_id + account_deletion_log_detail.snapshot = to_json({ + "tenant_account_join_info": ta, + "dismissed_members": members_to_dismiss + }) + account_deletion_log_detail.role = ta.role + db.session.add(account_deletion_log_detail) + def _remove_account_from_tenant(ta, email): """Remove the account from a tenant.""" tenant_id = ta.tenant_id db.session.delete(ta) logger.info(f"Removed account {email} from tenant {tenant_id}.") + + # create account deletion log detail + account_deletion_log_detail = AccountDeletionLogDetail() + account_deletion_log_detail.account_id = ta.account_id + account_deletion_log_detail.tenant_id = tenant_id + account_deletion_log_detail.snapshot = to_json({ + "tenant_account_join_info": ta + }) + account_deletion_log_detail.role = ta.role + db.session.add(account_deletion_log_detail) diff --git a/api/tasks/mail_account_deletion_task.py b/api/tasks/mail_account_deletion_task.py index 561cb5046cbae6..6e65733cf45459 100644 --- a/api/tasks/mail_account_deletion_task.py +++ b/api/tasks/mail_account_deletion_task.py @@ -25,16 +25,16 @@ def send_deletion_success_task(language, to): try: if language == "zh-Hans": html_content = render_template( - "delete_account_mail_template_zh-CN.html", + "delete_account_success_template_zh-CN.html", to=to, - # TODO: Add more template variables + email=to, ) mail.send(to=to, subject="Dify 账户删除成功", html=html_content) else: html_content = render_template( - "delete_account_mail_template_en-US.html", + "delete_account_success_template_en-US.html", to=to, - # TODO: Add more template variables + email=to, ) mail.send(to=to, subject="Dify Account Deleted", html=html_content) @@ -48,43 +48,6 @@ def send_deletion_success_task(language, to): logging.exception("Send account deletion success email to {} failed".format(to)) -@shared_task(queue="mail") -def send_deletion_fail_task(language, to): - """Send email to user regarding account deletion.""" - if not mail.is_inited(): - return - - logging.info( - click.style(f"Start send account deletion success email to {to}", fg="green") - ) - start_at = time.perf_counter() - - try: - if language == "zh-Hans": - html_content = render_template( - "delete_account_fail_mail_template_zh-CN.html", - to=to, - # TODO: Add more template variables - ) - mail.send(to=to, subject="Dify 账户删除失败", html=html_content) - else: - html_content = render_template( - "delete_account_fail_mail_template_en-US.html", - to=to, - # TODO: Add more template variables - ) - mail.send(to=to, subject="Dify Account Deletion Failed", html=html_content) - - end_at = time.perf_counter() - logging.info( - click.style( - "Send account deletion failed email to {}: latency: {}".format(to, end_at - start_at), fg="green" - ) - ) - except Exception: - logging.exception("Send account deletion success email to {} failed".format(to)) - - @shared_task(queue="mail") def send_account_deletion_verification_code(language, to, code): """Send email to user regarding account deletion verification code. @@ -104,18 +67,18 @@ def send_account_deletion_verification_code(language, to, code): try: if language == "zh-Hans": html_content = render_template( - "delete_account_verification_code_mail_template_zh-CN.html", + "delete_account_code_email_template_zh-CN.html", to=to, code=code ) - mail.send(to=to, subject="Dify 删除账户验证码", html=html_content) + mail.send(to=to, subject="Dify 的删除账户验证码", html=html_content) else: html_content = render_template( - "delete_account_verification_code_mail_template_en-US.html", + "delete_account_code_email_en-US.html", to=to, code=code ) - mail.send(to=to, subject="Dify Account Deletion Verification Code", html=html_content) + mail.send(to=to, subject="Delete Your Dify Account", html=html_content) end_at = time.perf_counter() logging.info( diff --git a/api/templates/delete_account_code_email_zh-CN.html b/api/templates/delete_account_code_email_en-US.html similarity index 100% rename from api/templates/delete_account_code_email_zh-CN.html rename to api/templates/delete_account_code_email_en-US.html diff --git a/api/templates/delete_account_code_email_template_en-US.html b/api/templates/delete_account_code_email_template_zh-CN.html similarity index 100% rename from api/templates/delete_account_code_email_template_en-US.html rename to api/templates/delete_account_code_email_template_zh-CN.html diff --git a/api/templates/delete_account_fail_template_en-US.html b/api/templates/delete_account_fail_template_en-US.html deleted file mode 100644 index 84a724c582b8e8..00000000000000 --- a/api/templates/delete_account_fail_template_en-US.html +++ /dev/null @@ -1,70 +0,0 @@ - - - - - - -
-
- - Dify Logo -
-

Ops! Your account delete failed

-

You recently deleted your {{email}} Dify account. It's done processing and is now deleted. As a reminder, you no longer have access to all the workspaces you used to be in with this account.

-
- - diff --git a/api/templates/delete_account_fail_template_zh-CN.html b/api/templates/delete_account_fail_template_zh-CN.html deleted file mode 100644 index e9a3234c121d72..00000000000000 --- a/api/templates/delete_account_fail_template_zh-CN.html +++ /dev/null @@ -1,70 +0,0 @@ - - - - - - -
-
- - Dify Logo -
-

账户删除失败

-

You recently deleted your {{email}} Dify account. It's done processing and is now deleted. As a reminder, you no longer have access to all the workspaces you used to be in with this account.

-
- - From 3c090e7d4cd4a11a02bb17285c81ddb99bdbb770 Mon Sep 17 00:00:00 2001 From: GareArc Date: Mon, 23 Dec 2024 00:48:03 -0500 Subject: [PATCH 14/34] fix: migration --- ...dded_account_deletion_logs_and_details.py} | 21 +++++++++++++++---- api/tasks/delete_account_task.py | 3 +-- 2 files changed, 18 insertions(+), 6 deletions(-) rename api/migrations/versions/{2024_12_20_2136-582e52119c70_added_account_deletion_logs.py => 2024_12_23_0047-35e1a3223204_added_account_deletion_logs_and_details.py} (50%) diff --git a/api/migrations/versions/2024_12_20_2136-582e52119c70_added_account_deletion_logs.py b/api/migrations/versions/2024_12_23_0047-35e1a3223204_added_account_deletion_logs_and_details.py similarity index 50% rename from api/migrations/versions/2024_12_20_2136-582e52119c70_added_account_deletion_logs.py rename to api/migrations/versions/2024_12_23_0047-35e1a3223204_added_account_deletion_logs_and_details.py index 17fd5dbe3a8c0f..02b49b21be3455 100644 --- a/api/migrations/versions/2024_12_20_2136-582e52119c70_added_account_deletion_logs.py +++ b/api/migrations/versions/2024_12_23_0047-35e1a3223204_added_account_deletion_logs_and_details.py @@ -1,8 +1,8 @@ -"""added account_deletion_logs +"""added account_deletion_logs and details -Revision ID: 582e52119c70 +Revision ID: 35e1a3223204 Revises: e1944c35e15e -Create Date: 2024-12-20 21:36:46.856033 +Create Date: 2024-12-23 00:47:44.483419 """ from alembic import op @@ -11,7 +11,7 @@ # revision identifiers, used by Alembic. -revision = '582e52119c70' +revision = '35e1a3223204' down_revision = 'e1944c35e15e' branch_labels = None depends_on = None @@ -19,10 +19,22 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### + op.create_table('account_deletion_log_details', + sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('account_id', models.types.StringUUID(), nullable=False), + sa.Column('tenant_id', models.types.StringUUID(), nullable=False), + sa.Column('role', sa.String(length=16), nullable=False), + sa.Column('snapshot', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.PrimaryKeyConstraint('id', name='account_deletion_log_detail_pkey') + ) op.create_table('account_deletion_logs', sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), sa.Column('email', sa.String(length=255), nullable=False), sa.Column('reason', sa.Text(), nullable=True), + sa.Column('account_id', models.types.StringUUID(), nullable=False), + sa.Column('snapshot', sa.Text(), nullable=False), sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), sa.PrimaryKeyConstraint('id', name='account_deletion_log_pkey') @@ -33,4 +45,5 @@ def upgrade(): def downgrade(): # ### commands auto generated by Alembic - please adjust! ### op.drop_table('account_deletion_logs') + op.drop_table('account_deletion_log_details') # ### end Alembic commands ### diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index 2a702c39180f1f..5850183245fa0e 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -4,14 +4,13 @@ import click from celery import shared_task from extensions.ext_database import db +from libs.helper import to_json from models.account import (Account, AccountDeletionLogDetail, TenantAccountJoin, TenantAccountJoinRole) from services.account_deletion_log_service import AccountDeletionLogService from services.billing_service import BillingService from tasks.mail_account_deletion_task import send_deletion_success_task -from api.libs.helper import to_json - logger = logging.getLogger(__name__) From a4921c1647d6ca99a663d3d108dd5b65c9fdcf9c Mon Sep 17 00:00:00 2001 From: GareArc Date: Mon, 23 Dec 2024 01:17:32 -0500 Subject: [PATCH 15/34] fix: remove custom json serializer --- api/libs/helper.py | 42 +------------------- api/services/account_deletion_log_service.py | 5 ++- api/tasks/delete_account_task.py | 6 +-- 3 files changed, 7 insertions(+), 46 deletions(-) diff --git a/api/libs/helper.py b/api/libs/helper.py index 56d4ab395cd969..349a75d73e6737 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -7,7 +7,7 @@ import time import uuid from collections.abc import Generator, Mapping -from datetime import date, datetime +from datetime import datetime from datetime import timezone as tz from hashlib import sha256 from typing import Any, Optional, Union, cast @@ -301,43 +301,3 @@ def increment_rate_limit(self, email: str): def get_current_datetime(): return datetime.now(tz.utc).replace(tzinfo=None) - - -def to_json(obj): - """ - Convert a Python object to a compact JSON string. - - Supports basic types, datetime, UUID, SQLAlchemy models, and nested structures. - - Args: - obj: The object to convert. - - Returns: - A compact JSON string representation of the object. - """ - def serialize(obj, seen=None): - if seen is None: - seen = set() # Track seen objects to prevent recursion - - if id(obj) in seen: - return None # Avoid circular references - - seen.add(id(obj)) - - if isinstance(obj, (datetime, date)): - return obj.isoformat() # Convert datetime and date to ISO 8601 string - elif isinstance(obj, uuid.UUID): - return str(obj) # Convert UUID to string - elif isinstance(obj, (int, float, str, bool, type(None))): - return obj # Leave basic types unchanged - elif isinstance(obj, dict): - return {key: serialize(value, seen) for key, value in obj.items() if value is not None} # Exclude None values - elif isinstance(obj, list): - return [serialize(item, seen) for item in obj] # Recursively serialize lists - else: - try: - return str(obj) # Fallback to string for unsupported types - except Exception: - return None # Return None if serialization fails - - return json.dumps(serialize(obj), separators=(',', ':')) diff --git a/api/services/account_deletion_log_service.py b/api/services/account_deletion_log_service.py index 2dc729c829c3e9..19c385ed41ee99 100644 --- a/api/services/account_deletion_log_service.py +++ b/api/services/account_deletion_log_service.py @@ -3,7 +3,8 @@ from configs import dify_config from extensions.ext_database import db -from libs.helper import get_current_datetime, to_json +from flask import jsonify +from libs.helper import get_current_datetime from models.account import Account, AccountDeletionLog @@ -14,7 +15,7 @@ def create_account_deletion_log(account: Account, reason): account_deletion_log.email = account.email account_deletion_log.reason = reason account_deletion_log.account_id = account.id - account_deletion_log.snapshot = to_json(account) + account_deletion_log.snapshot = jsonify(account) account_deletion_log.updated_at = get_current_datetime() return account_deletion_log diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index 5850183245fa0e..ab997dba5d6599 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -4,7 +4,7 @@ import click from celery import shared_task from extensions.ext_database import db -from libs.helper import to_json +from flask import jsonify from models.account import (Account, AccountDeletionLogDetail, TenantAccountJoin, TenantAccountJoinRole) from services.account_deletion_log_service import AccountDeletionLogService @@ -84,7 +84,7 @@ def _handle_owner_tenant_deletion(ta: TenantAccountJoin): account_deletion_log_detail = AccountDeletionLogDetail() account_deletion_log_detail.account_id = ta.account_id account_deletion_log_detail.tenant_id = tenant_id - account_deletion_log_detail.snapshot = to_json({ + account_deletion_log_detail.snapshot = jsonify({ "tenant_account_join_info": ta, "dismissed_members": members_to_dismiss }) @@ -102,7 +102,7 @@ def _remove_account_from_tenant(ta, email): account_deletion_log_detail = AccountDeletionLogDetail() account_deletion_log_detail.account_id = ta.account_id account_deletion_log_detail.tenant_id = tenant_id - account_deletion_log_detail.snapshot = to_json({ + account_deletion_log_detail.snapshot = jsonify({ "tenant_account_join_info": ta }) account_deletion_log_detail.role = ta.role From 577eb8f9a1e6ae20935908026f926451b79b8a55 Mon Sep 17 00:00:00 2001 From: GareArc Date: Mon, 23 Dec 2024 01:39:55 -0500 Subject: [PATCH 16/34] fix: serializer --- api/libs/helper.py | 17 +++++++++++++++++ api/services/account_deletion_log_service.py | 6 +++--- api/tasks/delete_account_task.py | 18 ++++++++++-------- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/api/libs/helper.py b/api/libs/helper.py index 349a75d73e6737..38244b4cc18247 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -301,3 +301,20 @@ def increment_rate_limit(self, email: str): def get_current_datetime(): return datetime.now(tz.utc).replace(tzinfo=None) + + +def serialize_sqlalchemy(obj): + """ + Serializes an SQLAlchemy object into a JSON string. + """ + data = {} + for column in obj.__table__.columns: + value = getattr(obj, column.name) + if isinstance(value, datetime): + data[column.name] = value.isoformat() # ISO 8601 format for datetime + elif isinstance(value, uuid.UUID): + data[column.name] = str(value) # String representation for UUID + else: + data[column.name] = value + + return json.dumps(data, separators=(",", ":")) diff --git a/api/services/account_deletion_log_service.py b/api/services/account_deletion_log_service.py index 19c385ed41ee99..4a44e3d90fd7f6 100644 --- a/api/services/account_deletion_log_service.py +++ b/api/services/account_deletion_log_service.py @@ -1,10 +1,10 @@ +import json from datetime import timedelta from configs import dify_config from extensions.ext_database import db -from flask import jsonify -from libs.helper import get_current_datetime +from libs.helper import get_current_datetime, serialize_sqlalchemy from models.account import Account, AccountDeletionLog @@ -15,7 +15,7 @@ def create_account_deletion_log(account: Account, reason): account_deletion_log.email = account.email account_deletion_log.reason = reason account_deletion_log.account_id = account.id - account_deletion_log.snapshot = jsonify(account) + account_deletion_log.snapshot = json.dumps(serialize_sqlalchemy(account), separators=(",", ":")) account_deletion_log.updated_at = get_current_datetime() return account_deletion_log diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index ab997dba5d6599..087e48ad5d82b4 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -1,16 +1,18 @@ +import json import logging import time import click from celery import shared_task from extensions.ext_database import db -from flask import jsonify from models.account import (Account, AccountDeletionLogDetail, TenantAccountJoin, TenantAccountJoinRole) from services.account_deletion_log_service import AccountDeletionLogService from services.billing_service import BillingService from tasks.mail_account_deletion_task import send_deletion_success_task +from api.libs.helper import serialize_sqlalchemy + logger = logging.getLogger(__name__) @@ -84,10 +86,10 @@ def _handle_owner_tenant_deletion(ta: TenantAccountJoin): account_deletion_log_detail = AccountDeletionLogDetail() account_deletion_log_detail.account_id = ta.account_id account_deletion_log_detail.tenant_id = tenant_id - account_deletion_log_detail.snapshot = jsonify({ - "tenant_account_join_info": ta, - "dismissed_members": members_to_dismiss - }) + account_deletion_log_detail.snapshot = json.dumps({ + "tenant_account_join_info": serialize_sqlalchemy(ta), + "dismissed_members": [serialize_sqlalchemy(member) for member in members_to_dismiss] + }, separators=(",", ":")) account_deletion_log_detail.role = ta.role db.session.add(account_deletion_log_detail) @@ -102,8 +104,8 @@ def _remove_account_from_tenant(ta, email): account_deletion_log_detail = AccountDeletionLogDetail() account_deletion_log_detail.account_id = ta.account_id account_deletion_log_detail.tenant_id = tenant_id - account_deletion_log_detail.snapshot = jsonify({ - "tenant_account_join_info": ta - }) + account_deletion_log_detail.snapshot = json.dumps({ + "tenant_account_join_info": serialize_sqlalchemy(ta), + }, separators=(",", ":")) account_deletion_log_detail.role = ta.role db.session.add(account_deletion_log_detail) From 44c2bfbbeeeb56ce2b844f124c70be0501a1ca97 Mon Sep 17 00:00:00 2001 From: GareArc Date: Mon, 23 Dec 2024 01:42:29 -0500 Subject: [PATCH 17/34] fix: bad import --- api/tasks/delete_account_task.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index 087e48ad5d82b4..2ba20f5a3dae12 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -5,14 +5,13 @@ import click from celery import shared_task from extensions.ext_database import db +from libs.helper import serialize_sqlalchemy from models.account import (Account, AccountDeletionLogDetail, TenantAccountJoin, TenantAccountJoinRole) from services.account_deletion_log_service import AccountDeletionLogService from services.billing_service import BillingService from tasks.mail_account_deletion_task import send_deletion_success_task -from api.libs.helper import serialize_sqlalchemy - logger = logging.getLogger(__name__) From d9687ee5345df1821e70ff1424e896b1dcc3e49b Mon Sep 17 00:00:00 2001 From: GareArc Date: Mon, 23 Dec 2024 01:58:22 -0500 Subject: [PATCH 18/34] fix: add email check in register service --- api/services/account_service.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/services/account_service.py b/api/services/account_service.py index fe1ea9bc1b3de1..38409ed785960a 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -839,6 +839,8 @@ def register( ) -> Account: db.session.begin_nested() """Register account""" + if AccountDeletionLogService.email_in_freeze(email): + raise AccountRegisterError("Email is in freeze.") try: account = AccountService.create_account( email=email, From d9bba39e752eb0909224173e76d89f679e7f8f13 Mon Sep 17 00:00:00 2001 From: GareArc Date: Mon, 23 Dec 2024 02:19:22 -0500 Subject: [PATCH 19/34] reformat --- api/configs/feature/__init__.py | 16 ++++-- api/controllers/console/workspace/account.py | 26 ++++----- api/libs/helper.py | 5 +- api/models/account.py | 8 +-- api/services/account_deletion_log_service.py | 9 +-- api/services/account_service.py | 60 +++++++++++--------- api/services/billing_service.py | 3 +- api/tasks/delete_account_task.py | 40 ++++++------- api/tasks/mail_account_deletion_task.py | 28 ++++----- 9 files changed, 100 insertions(+), 95 deletions(-) diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index 0730c8a0b569a9..460e5ce3383f85 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -1,11 +1,19 @@ from typing import Annotated, Literal, Optional -from configs.feature.hosted_service import HostedServiceConfig -from pydantic import (AliasChoices, Field, HttpUrl, NegativeInt, - NonNegativeInt, PositiveFloat, PositiveInt, - computed_field) +from pydantic import ( + AliasChoices, + Field, + HttpUrl, + NegativeInt, + NonNegativeInt, + PositiveFloat, + PositiveInt, + computed_field, +) from pydantic_settings import BaseSettings +from configs.feature.hosted_service import HostedServiceConfig + class SecurityConfig(BaseSettings): """ diff --git a/api/controllers/console/workspace/account.py b/api/controllers/console/workspace/account.py index 412b8a8b212f54..6e7f7e494b71da 100644 --- a/api/controllers/console/workspace/account.py +++ b/api/controllers/console/workspace/account.py @@ -1,27 +1,27 @@ import datetime import pytz +from flask import request +from flask_login import current_user +from flask_restful import Resource, fields, marshal_with, reqparse + from configs import dify_config from constants.languages import supported_language from controllers.console import api -from controllers.console.workspace.error import (AccountAlreadyInitedError, - CurrentPasswordIncorrectError, - InvalidInvitationCodeError, - RepeatPasswordNotMatchError) -from controllers.console.wraps import (account_initialization_required, - enterprise_license_required, - setup_required) +from controllers.console.workspace.error import ( + AccountAlreadyInitedError, + CurrentPasswordIncorrectError, + InvalidInvitationCodeError, + RepeatPasswordNotMatchError, +) +from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required from extensions.ext_database import db from fields.member_fields import account_fields -from flask import request -from flask_login import current_user -from flask_restful import Resource, fields, marshal_with, reqparse from libs.helper import TimestampField, timezone from libs.login import login_required from models import AccountIntegrate, InvitationCode from services.account_service import AccountService -from services.errors.account import \ - CurrentPasswordIncorrectError as ServiceCurrentPasswordIncorrectError +from services.errors.account import CurrentPasswordIncorrectError as ServiceCurrentPasswordIncorrectError class AccountInitApi(Resource): @@ -243,7 +243,6 @@ def get(self): class AccountDeleteVerifyApi(Resource): - @setup_required @login_required @account_initialization_required @@ -260,7 +259,6 @@ def get(self): class AccountDeleteApi(Resource): - @setup_required @login_required @account_initialization_required diff --git a/api/libs/helper.py b/api/libs/helper.py index 38244b4cc18247..f43e0727e0e30c 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -13,12 +13,13 @@ from typing import Any, Optional, Union, cast from zoneinfo import available_timezones +from flask import Response, stream_with_context +from flask_restful import fields + from configs import dify_config from core.app.features.rate_limiting.rate_limit import RateLimitGenerator from core.file import helpers as file_helpers from extensions.ext_redis import redis_client -from flask import Response, stream_with_context -from flask_restful import fields from models.account import Account diff --git a/api/models/account.py b/api/models/account.py index b6843e79067ba9..a1ef04bca586e7 100644 --- a/api/models/account.py +++ b/api/models/account.py @@ -271,9 +271,7 @@ class InvitationCode(db.Model): class AccountDeletionLog(db.Model): __tablename__ = "account_deletion_logs" - __table_args__ = ( - db.PrimaryKeyConstraint("id", name="account_deletion_log_pkey"), - ) + __table_args__ = (db.PrimaryKeyConstraint("id", name="account_deletion_log_pkey"),) id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()")) email = db.Column(db.String(255), nullable=False) @@ -287,9 +285,7 @@ class AccountDeletionLog(db.Model): class AccountDeletionLogDetail(db.Model): __tablename__ = "account_deletion_log_details" - __table_args__ = ( - db.PrimaryKeyConstraint("id", name="account_deletion_log_detail_pkey"), - ) + __table_args__ = (db.PrimaryKeyConstraint("id", name="account_deletion_log_detail_pkey"),) id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()")) account_id = db.Column(StringUUID, nullable=False) diff --git a/api/services/account_deletion_log_service.py b/api/services/account_deletion_log_service.py index 4a44e3d90fd7f6..1a053bc07739f4 100644 --- a/api/services/account_deletion_log_service.py +++ b/api/services/account_deletion_log_service.py @@ -1,4 +1,3 @@ - import json from datetime import timedelta @@ -22,10 +21,12 @@ def create_account_deletion_log(account: Account, reason): @staticmethod def email_in_freeze(email): - log = db.session.query(AccountDeletionLog) \ - .filter(AccountDeletionLog.email == email) \ - .order_by(AccountDeletionLog.created_at.desc()) \ + log = ( + db.session.query(AccountDeletionLog) + .filter(AccountDeletionLog.email == email) + .order_by(AccountDeletionLog.created_at.desc()) .first() + ) if not log: return False diff --git a/api/services/account_service.py b/api/services/account_service.py index 38409ed785960a..d044b9f4e0b389 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -8,6 +8,10 @@ from hashlib import sha256 from typing import Any, Optional +from pydantic import BaseModel +from sqlalchemy import func +from werkzeug.exceptions import Unauthorized + from configs import dify_config from constants.languages import language_timezone_mapping, languages from events.tenant_event import tenant_was_created @@ -17,34 +21,41 @@ from libs.passport import PassportService from libs.password import compare_password, hash_password, valid_password from libs.rsa import generate_key_pair -from models.account import (Account, AccountIntegrate, AccountStatus, Tenant, - TenantAccountJoin, TenantAccountJoinRole, - TenantAccountRole, TenantStatus) +from models.account import ( + Account, + AccountIntegrate, + AccountStatus, + Tenant, + TenantAccountJoin, + TenantAccountJoinRole, + TenantAccountRole, + TenantStatus, +) from models.model import DifySetup -from pydantic import BaseModel from services.account_deletion_log_service import AccountDeletionLogService -from services.errors.account import (AccountAlreadyInTenantError, - AccountLoginError, AccountNotFoundError, - AccountNotLinkTenantError, - AccountPasswordError, - AccountRegisterError, - CannotOperateSelfError, - CurrentPasswordIncorrectError, - InvalidActionError, - LinkAccountIntegrateError, - MemberNotInTenantError, NoPermissionError, - RoleAlreadyAssignedError, - TenantNotFoundError) +from services.errors.account import ( + AccountAlreadyInTenantError, + AccountLoginError, + AccountNotFoundError, + AccountNotLinkTenantError, + AccountPasswordError, + AccountRegisterError, + CannotOperateSelfError, + CurrentPasswordIncorrectError, + InvalidActionError, + LinkAccountIntegrateError, + MemberNotInTenantError, + NoPermissionError, + RoleAlreadyAssignedError, + TenantNotFoundError, +) from services.errors.workspace import WorkSpaceNotAllowedCreateError from services.feature_service import FeatureService -from sqlalchemy import func from tasks.delete_account_task import delete_account_task -from tasks.mail_account_deletion_task import \ - send_account_deletion_verification_code +from tasks.mail_account_deletion_task import send_account_deletion_verification_code from tasks.mail_email_code_login import send_email_code_login_mail_task from tasks.mail_invite_member_task import send_invite_member_mail_task from tasks.mail_reset_password_task import send_reset_password_mail_task -from werkzeug.exceptions import Unauthorized class TokenPair(BaseModel): @@ -250,8 +261,7 @@ def generate_account_deletion_verification_code(account: Account) -> tuple[str, @classmethod def send_account_deletion_verification_email(cls, account: Account, code: str): if cls.email_code_account_deletion_rate_limiter.is_rate_limited(email): - from controllers.console.auth.error import \ - EmailCodeAccountDeletionRateLimitExceededError + from controllers.console.auth.error import EmailCodeAccountDeletionRateLimitExceededError raise EmailCodeAccountDeletionRateLimitExceededError() @@ -385,8 +395,7 @@ def send_reset_password_email( account_email = account.email if account else email if cls.reset_password_rate_limiter.is_rate_limited(account_email): - from controllers.console.auth.error import \ - PasswordResetRateLimitExceededError + from controllers.console.auth.error import PasswordResetRateLimitExceededError raise PasswordResetRateLimitExceededError() @@ -420,8 +429,7 @@ def send_email_code_login_email( cls, account: Optional[Account] = None, email: Optional[str] = None, language: Optional[str] = "en-US" ): if cls.email_code_login_rate_limiter.is_rate_limited(email): - from controllers.console.auth.error import \ - EmailCodeLoginRateLimitExceededError + from controllers.console.auth.error import EmailCodeLoginRateLimitExceededError raise EmailCodeLoginRateLimitExceededError() diff --git a/api/services/billing_service.py b/api/services/billing_service.py index f8d9bac7cad984..2167c76c135262 100644 --- a/api/services/billing_service.py +++ b/api/services/billing_service.py @@ -1,6 +1,7 @@ import os import requests + from extensions.ext_database import db from models.account import TenantAccountJoin, TenantAccountRole @@ -61,6 +62,6 @@ def is_tenant_owner_or_admin(current_user): @classmethod def unsubscripbe_tenant_customer(cls, tenant_id: str): - """ Delete related customer in billing service. Used when tenant is deleted.""" + """Delete related customer in billing service. Used when tenant is deleted.""" params = {"tenant_id": tenant_id} return cls._send_request("DELETE", "/subscription", params=params) diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index 2ba20f5a3dae12..0db11c7caf0a1a 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -4,10 +4,10 @@ import click from celery import shared_task + from extensions.ext_database import db from libs.helper import serialize_sqlalchemy -from models.account import (Account, AccountDeletionLogDetail, - TenantAccountJoin, TenantAccountJoinRole) +from models.account import Account, AccountDeletionLogDetail, TenantAccountJoin, TenantAccountJoinRole from services.account_deletion_log_service import AccountDeletionLogService from services.billing_service import BillingService from tasks.mail_account_deletion_task import send_deletion_success_task @@ -37,15 +37,13 @@ def delete_account_task(account_id, reason: str): ) except Exception as e: db.session.rollback() - logging.error(f"Failed to delete account {account.email}: {e}.") + logging.exception(f"Failed to delete account {account.email}.") raise def _process_account_deletion(account, reason): # Fetch all tenant-account associations - tenant_account_joins = db.session.query(TenantAccountJoin).filter( - TenantAccountJoin.account_id == account.id - ).all() + tenant_account_joins = db.session.query(TenantAccountJoin).filter(TenantAccountJoin.account_id == account.id).all() for ta in tenant_account_joins: if ta.role == TenantAccountJoinRole.OWNER.value: @@ -53,9 +51,7 @@ def _process_account_deletion(account, reason): else: _remove_account_from_tenant(ta, account.email) - account_deletion_log = AccountDeletionLogService.create_account_deletion_log( - account, reason - ) + account_deletion_log = AccountDeletionLogService.create_account_deletion_log(account, reason) db.session.add(account_deletion_log) db.session.delete(account) logger.info(f"Account {account.email} successfully deleted.") @@ -66,9 +62,7 @@ def _handle_owner_tenant_deletion(ta: TenantAccountJoin): tenant_id = ta.tenant_id # Dismiss all tenant members - members_to_dismiss = db.session.query(TenantAccountJoin).filter( - TenantAccountJoin.tenant_id == tenant_id - ).all() + members_to_dismiss = db.session.query(TenantAccountJoin).filter(TenantAccountJoin.tenant_id == tenant_id).all() for member in members_to_dismiss: db.session.delete(member) logger.info(f"Dismissed {len(members_to_dismiss)} members from tenant {tenant_id}.") @@ -78,17 +72,20 @@ def _handle_owner_tenant_deletion(ta: TenantAccountJoin): BillingService.unsubscripbe_tenant_customer(tenant_id) logger.info(f"Subscription for tenant {tenant_id} deleted successfully.") except Exception as e: - logger.error(f"Failed to delete subscription for tenant {tenant_id}: {e}.") + logger.exception(f"Failed to delete subscription for tenant {tenant_id}.") raise # create account deletion log detail account_deletion_log_detail = AccountDeletionLogDetail() account_deletion_log_detail.account_id = ta.account_id account_deletion_log_detail.tenant_id = tenant_id - account_deletion_log_detail.snapshot = json.dumps({ - "tenant_account_join_info": serialize_sqlalchemy(ta), - "dismissed_members": [serialize_sqlalchemy(member) for member in members_to_dismiss] - }, separators=(",", ":")) + account_deletion_log_detail.snapshot = json.dumps( + { + "tenant_account_join_info": serialize_sqlalchemy(ta), + "dismissed_members": [serialize_sqlalchemy(member) for member in members_to_dismiss], + }, + separators=(",", ":"), + ) account_deletion_log_detail.role = ta.role db.session.add(account_deletion_log_detail) @@ -103,8 +100,11 @@ def _remove_account_from_tenant(ta, email): account_deletion_log_detail = AccountDeletionLogDetail() account_deletion_log_detail.account_id = ta.account_id account_deletion_log_detail.tenant_id = tenant_id - account_deletion_log_detail.snapshot = json.dumps({ - "tenant_account_join_info": serialize_sqlalchemy(ta), - }, separators=(",", ":")) + account_deletion_log_detail.snapshot = json.dumps( + { + "tenant_account_join_info": serialize_sqlalchemy(ta), + }, + separators=(",", ":"), + ) account_deletion_log_detail.role = ta.role db.session.add(account_deletion_log_detail) diff --git a/api/tasks/mail_account_deletion_task.py b/api/tasks/mail_account_deletion_task.py index 6e65733cf45459..d38df3613bd450 100644 --- a/api/tasks/mail_account_deletion_task.py +++ b/api/tasks/mail_account_deletion_task.py @@ -3,9 +3,10 @@ import click from celery import shared_task -from extensions.ext_mail import mail from flask import render_template +from extensions.ext_mail import mail + @shared_task(queue="mail") def send_deletion_success_task(language, to): @@ -17,9 +18,7 @@ def send_deletion_success_task(language, to): if not mail.is_inited(): return - logging.info( - click.style(f"Start send account deletion success email to {to}", fg="green") - ) + logging.info(click.style(f"Start send account deletion success email to {to}", fg="green")) start_at = time.perf_counter() try: @@ -59,31 +58,24 @@ def send_account_deletion_verification_code(language, to, code): if not mail.is_inited(): return - logging.info( - click.style(f"Start send account deletion verification code email to {to}", fg="green") - ) + logging.info(click.style(f"Start send account deletion verification code email to {to}", fg="green")) start_at = time.perf_counter() try: if language == "zh-Hans": - html_content = render_template( - "delete_account_code_email_template_zh-CN.html", - to=to, - code=code - ) + html_content = render_template("delete_account_code_email_template_zh-CN.html", to=to, code=code) mail.send(to=to, subject="Dify 的删除账户验证码", html=html_content) else: - html_content = render_template( - "delete_account_code_email_en-US.html", - to=to, - code=code - ) + html_content = render_template("delete_account_code_email_en-US.html", to=to, code=code) mail.send(to=to, subject="Delete Your Dify Account", html=html_content) end_at = time.perf_counter() logging.info( click.style( - "Send account deletion verification code email to {} succeeded: latency: {}".format(to, end_at - start_at), fg="green" + "Send account deletion verification code email to {} succeeded: latency: {}".format( + to, end_at - start_at + ), + fg="green", ) ) except Exception: From ab4afa5d47b3ac95cc547a2dea80510deccb51bf Mon Sep 17 00:00:00 2001 From: GareArc Date: Mon, 23 Dec 2024 02:51:59 -0500 Subject: [PATCH 20/34] fix: rebase migration head --- ...added_account_deletion_logs_and_details.py | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/api/migrations/versions/2024_12_23_0047-35e1a3223204_added_account_deletion_logs_and_details.py b/api/migrations/versions/2024_12_23_0047-35e1a3223204_added_account_deletion_logs_and_details.py index 02b49b21be3455..d6e210bde83292 100644 --- a/api/migrations/versions/2024_12_23_0047-35e1a3223204_added_account_deletion_logs_and_details.py +++ b/api/migrations/versions/2024_12_23_0047-35e1a3223204_added_account_deletion_logs_and_details.py @@ -1,18 +1,17 @@ """added account_deletion_logs and details Revision ID: 35e1a3223204 -Revises: e1944c35e15e +Revises: d7999dfa4aae Create Date: 2024-12-23 00:47:44.483419 """ -from alembic import op import models as models import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '35e1a3223204' -down_revision = 'e1944c35e15e' +down_revision = 'd7999dfa4aae' branch_labels = None depends_on = None @@ -20,25 +19,25 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.create_table('account_deletion_log_details', - sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), - sa.Column('account_id', models.types.StringUUID(), nullable=False), - sa.Column('tenant_id', models.types.StringUUID(), nullable=False), - sa.Column('role', sa.String(length=16), nullable=False), - sa.Column('snapshot', sa.Text(), nullable=False), - sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), - sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), - sa.PrimaryKeyConstraint('id', name='account_deletion_log_detail_pkey') - ) + sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('account_id', models.types.StringUUID(), nullable=False), + sa.Column('tenant_id', models.types.StringUUID(), nullable=False), + sa.Column('role', sa.String(length=16), nullable=False), + sa.Column('snapshot', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.PrimaryKeyConstraint('id', name='account_deletion_log_detail_pkey') + ) op.create_table('account_deletion_logs', - sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), - sa.Column('email', sa.String(length=255), nullable=False), - sa.Column('reason', sa.Text(), nullable=True), - sa.Column('account_id', models.types.StringUUID(), nullable=False), - sa.Column('snapshot', sa.Text(), nullable=False), - sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), - sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), - sa.PrimaryKeyConstraint('id', name='account_deletion_log_pkey') - ) + sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('email', sa.String(length=255), nullable=False), + sa.Column('reason', sa.Text(), nullable=True), + sa.Column('account_id', models.types.StringUUID(), nullable=False), + sa.Column('snapshot', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), + sa.PrimaryKeyConstraint('id', name='account_deletion_log_pkey') + ) # ### end Alembic commands ### From 356c3d533bca6a242f91e5a1d5a5618c5b122a59 Mon Sep 17 00:00:00 2001 From: GareArc Date: Mon, 23 Dec 2024 23:04:22 -0500 Subject: [PATCH 21/34] fix: remove deletion logic --- api/libs/helper.py | 27 +---- api/models/account.py | 28 ------ api/services/account_deletion_log_service.py | 37 ------- api/services/account_service.py | 67 ++++++------- api/services/billing_service.py | 18 +++- api/tasks/delete_account_task.py | 100 ++----------------- 6 files changed, 51 insertions(+), 226 deletions(-) delete mode 100644 api/services/account_deletion_log_service.py diff --git a/api/libs/helper.py b/api/libs/helper.py index f43e0727e0e30c..0401e36811682c 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -8,18 +8,16 @@ import uuid from collections.abc import Generator, Mapping from datetime import datetime -from datetime import timezone as tz from hashlib import sha256 from typing import Any, Optional, Union, cast from zoneinfo import available_timezones -from flask import Response, stream_with_context -from flask_restful import fields - from configs import dify_config from core.app.features.rate_limiting.rate_limit import RateLimitGenerator from core.file import helpers as file_helpers from extensions.ext_redis import redis_client +from flask import Response, stream_with_context +from flask_restful import fields from models.account import Account @@ -298,24 +296,3 @@ def increment_rate_limit(self, email: str): redis_client.zadd(key, {current_time: current_time}) redis_client.expire(key, self.time_window * 2) - - -def get_current_datetime(): - return datetime.now(tz.utc).replace(tzinfo=None) - - -def serialize_sqlalchemy(obj): - """ - Serializes an SQLAlchemy object into a JSON string. - """ - data = {} - for column in obj.__table__.columns: - value = getattr(obj, column.name) - if isinstance(value, datetime): - data[column.name] = value.isoformat() # ISO 8601 format for datetime - elif isinstance(value, uuid.UUID): - data[column.name] = str(value) # String representation for UUID - else: - data[column.name] = value - - return json.dumps(data, separators=(",", ":")) diff --git a/api/models/account.py b/api/models/account.py index a1ef04bca586e7..932ba1da578b8a 100644 --- a/api/models/account.py +++ b/api/models/account.py @@ -267,31 +267,3 @@ class InvitationCode(db.Model): used_by_account_id = db.Column(StringUUID) deprecated_at = db.Column(db.DateTime) created_at = db.Column(db.DateTime, nullable=False, server_default=func.current_timestamp()) - - -class AccountDeletionLog(db.Model): - __tablename__ = "account_deletion_logs" - __table_args__ = (db.PrimaryKeyConstraint("id", name="account_deletion_log_pkey"),) - - id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()")) - email = db.Column(db.String(255), nullable=False) - reason = db.Column(db.Text, nullable=True) - account_id = db.Column(StringUUID, nullable=False) - snapshot = db.Column(db.Text, nullable=False) - - created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")) - updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")) - - -class AccountDeletionLogDetail(db.Model): - __tablename__ = "account_deletion_log_details" - __table_args__ = (db.PrimaryKeyConstraint("id", name="account_deletion_log_detail_pkey"),) - - id = db.Column(StringUUID, server_default=db.text("uuid_generate_v4()")) - account_id = db.Column(StringUUID, nullable=False) - tenant_id = db.Column(StringUUID, nullable=False) - role = db.Column(db.String(16), nullable=False) - snapshot = db.Column(db.Text, nullable=False) - - created_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")) - updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")) diff --git a/api/services/account_deletion_log_service.py b/api/services/account_deletion_log_service.py deleted file mode 100644 index 1a053bc07739f4..00000000000000 --- a/api/services/account_deletion_log_service.py +++ /dev/null @@ -1,37 +0,0 @@ -import json -from datetime import timedelta - -from configs import dify_config -from extensions.ext_database import db -from libs.helper import get_current_datetime, serialize_sqlalchemy -from models.account import Account, AccountDeletionLog - - -class AccountDeletionLogService: - @staticmethod - def create_account_deletion_log(account: Account, reason): - account_deletion_log = AccountDeletionLog() - account_deletion_log.email = account.email - account_deletion_log.reason = reason - account_deletion_log.account_id = account.id - account_deletion_log.snapshot = json.dumps(serialize_sqlalchemy(account), separators=(",", ":")) - account_deletion_log.updated_at = get_current_datetime() - - return account_deletion_log - - @staticmethod - def email_in_freeze(email): - log = ( - db.session.query(AccountDeletionLog) - .filter(AccountDeletionLog.email == email) - .order_by(AccountDeletionLog.created_at.desc()) - .first() - ) - - if not log: - return False - - # check if email is in freeze - if log.created_at + timedelta(days=dify_config.EMAIL_FREEZE_PERIOD_IN_DAYS) > get_current_datetime(): - return True - return False diff --git a/api/services/account_service.py b/api/services/account_service.py index d044b9f4e0b389..220609331cc71a 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -8,10 +8,6 @@ from hashlib import sha256 from typing import Any, Optional -from pydantic import BaseModel -from sqlalchemy import func -from werkzeug.exceptions import Unauthorized - from configs import dify_config from constants.languages import language_timezone_mapping, languages from events.tenant_event import tenant_was_created @@ -21,41 +17,35 @@ from libs.passport import PassportService from libs.password import compare_password, hash_password, valid_password from libs.rsa import generate_key_pair -from models.account import ( - Account, - AccountIntegrate, - AccountStatus, - Tenant, - TenantAccountJoin, - TenantAccountJoinRole, - TenantAccountRole, - TenantStatus, -) +from models.account import (Account, AccountIntegrate, AccountStatus, Tenant, + TenantAccountJoin, TenantAccountJoinRole, + TenantAccountRole, TenantStatus) from models.model import DifySetup -from services.account_deletion_log_service import AccountDeletionLogService -from services.errors.account import ( - AccountAlreadyInTenantError, - AccountLoginError, - AccountNotFoundError, - AccountNotLinkTenantError, - AccountPasswordError, - AccountRegisterError, - CannotOperateSelfError, - CurrentPasswordIncorrectError, - InvalidActionError, - LinkAccountIntegrateError, - MemberNotInTenantError, - NoPermissionError, - RoleAlreadyAssignedError, - TenantNotFoundError, -) +from pydantic import BaseModel +from services.errors.account import (AccountAlreadyInTenantError, + AccountLoginError, AccountNotFoundError, + AccountNotLinkTenantError, + AccountPasswordError, + AccountRegisterError, + CannotOperateSelfError, + CurrentPasswordIncorrectError, + InvalidActionError, + LinkAccountIntegrateError, + MemberNotInTenantError, NoPermissionError, + RoleAlreadyAssignedError, + TenantNotFoundError) from services.errors.workspace import WorkSpaceNotAllowedCreateError from services.feature_service import FeatureService +from sqlalchemy import func from tasks.delete_account_task import delete_account_task -from tasks.mail_account_deletion_task import send_account_deletion_verification_code +from tasks.mail_account_deletion_task import \ + send_account_deletion_verification_code from tasks.mail_email_code_login import send_email_code_login_mail_task from tasks.mail_invite_member_task import send_invite_member_mail_task from tasks.mail_reset_password_task import send_reset_password_mail_task +from werkzeug.exceptions import Unauthorized + +from api.services.billing_service import BillingService class TokenPair(BaseModel): @@ -208,7 +198,7 @@ def create_account( raise AccountNotFound() - if AccountDeletionLogService.email_in_freeze(email): + if dify_config.BILLING_ENABLED and BillingService.is_email_in_freeze(email): raise AccountRegisterError("Email is in freeze.") account = Account() @@ -261,7 +251,8 @@ def generate_account_deletion_verification_code(account: Account) -> tuple[str, @classmethod def send_account_deletion_verification_email(cls, account: Account, code: str): if cls.email_code_account_deletion_rate_limiter.is_rate_limited(email): - from controllers.console.auth.error import EmailCodeAccountDeletionRateLimitExceededError + from controllers.console.auth.error import \ + EmailCodeAccountDeletionRateLimitExceededError raise EmailCodeAccountDeletionRateLimitExceededError() @@ -395,7 +386,8 @@ def send_reset_password_email( account_email = account.email if account else email if cls.reset_password_rate_limiter.is_rate_limited(account_email): - from controllers.console.auth.error import PasswordResetRateLimitExceededError + from controllers.console.auth.error import \ + PasswordResetRateLimitExceededError raise PasswordResetRateLimitExceededError() @@ -429,7 +421,8 @@ def send_email_code_login_email( cls, account: Optional[Account] = None, email: Optional[str] = None, language: Optional[str] = "en-US" ): if cls.email_code_login_rate_limiter.is_rate_limited(email): - from controllers.console.auth.error import EmailCodeLoginRateLimitExceededError + from controllers.console.auth.error import \ + EmailCodeLoginRateLimitExceededError raise EmailCodeLoginRateLimitExceededError() @@ -847,7 +840,7 @@ def register( ) -> Account: db.session.begin_nested() """Register account""" - if AccountDeletionLogService.email_in_freeze(email): + if dify_config.BILLING_ENABLED and BillingService.is_email_in_freeze(email): raise AccountRegisterError("Email is in freeze.") try: account = AccountService.create_account( diff --git a/api/services/billing_service.py b/api/services/billing_service.py index 2167c76c135262..6aba5065233fd3 100644 --- a/api/services/billing_service.py +++ b/api/services/billing_service.py @@ -1,7 +1,6 @@ import os import requests - from extensions.ext_database import db from models.account import TenantAccountJoin, TenantAccountRole @@ -61,7 +60,16 @@ def is_tenant_owner_or_admin(current_user): raise ValueError("Only team owner or team admin can perform this action") @classmethod - def unsubscripbe_tenant_customer(cls, tenant_id: str): - """Delete related customer in billing service. Used when tenant is deleted.""" - params = {"tenant_id": tenant_id} - return cls._send_request("DELETE", "/subscription", params=params) + def delete_account(cls, account_id: str, reason: str): + """Delete account.""" + params = {"account_id": account_id, "reason": reason} + return cls._send_request("DELETE", "/account", params=params) + + @classmethod + def is_email_in_freeze(cls, email: str) -> bool: + params = {"email": email} + try: + response = cls._send_request("GET", "/account/in-freeze", params=params) + return response["data"] + except Exception: + return False diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index 0db11c7caf0a1a..465567c4ff175c 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -1,14 +1,8 @@ -import json import logging -import time -import click from celery import shared_task - from extensions.ext_database import db -from libs.helper import serialize_sqlalchemy -from models.account import Account, AccountDeletionLogDetail, TenantAccountJoin, TenantAccountJoinRole -from services.account_deletion_log_service import AccountDeletionLogService +from models.account import Account from services.billing_service import BillingService from tasks.mail_account_deletion_task import send_deletion_success_task @@ -17,94 +11,12 @@ @shared_task(queue="dataset") def delete_account_task(account_id, reason: str): - account = db.session.query(Account).filter(Account.id == account_id).first() - if not account: - logging.error(f"Account with ID {account_id} not found.") - return - - logger.info(click.style(f"Start deletion task for account {account.email}.", fg="green")) - start_at = time.perf_counter() - try: - _process_account_deletion(account, reason) - db.session.commit() - send_deletion_success_task.delay(account.interface_language, account.email) - logger.info( - click.style( - f"Account deletion task completed for {account.email}: latency: {time.perf_counter() - start_at}", - fg="green", - ) - ) + BillingService.delete_account(account_id, reason) except Exception as e: - db.session.rollback() - logging.exception(f"Failed to delete account {account.email}.") + logger.error(f"Failed to delete account {account_id} from billing service: {e}") raise - -def _process_account_deletion(account, reason): - # Fetch all tenant-account associations - tenant_account_joins = db.session.query(TenantAccountJoin).filter(TenantAccountJoin.account_id == account.id).all() - - for ta in tenant_account_joins: - if ta.role == TenantAccountJoinRole.OWNER.value: - _handle_owner_tenant_deletion(ta) - else: - _remove_account_from_tenant(ta, account.email) - - account_deletion_log = AccountDeletionLogService.create_account_deletion_log(account, reason) - db.session.add(account_deletion_log) - db.session.delete(account) - logger.info(f"Account {account.email} successfully deleted.") - - -def _handle_owner_tenant_deletion(ta: TenantAccountJoin): - """Handle deletion of a tenant where the account is an owner.""" - tenant_id = ta.tenant_id - - # Dismiss all tenant members - members_to_dismiss = db.session.query(TenantAccountJoin).filter(TenantAccountJoin.tenant_id == tenant_id).all() - for member in members_to_dismiss: - db.session.delete(member) - logger.info(f"Dismissed {len(members_to_dismiss)} members from tenant {tenant_id}.") - - # Delete subscription - try: - BillingService.unsubscripbe_tenant_customer(tenant_id) - logger.info(f"Subscription for tenant {tenant_id} deleted successfully.") - except Exception as e: - logger.exception(f"Failed to delete subscription for tenant {tenant_id}.") - raise - - # create account deletion log detail - account_deletion_log_detail = AccountDeletionLogDetail() - account_deletion_log_detail.account_id = ta.account_id - account_deletion_log_detail.tenant_id = tenant_id - account_deletion_log_detail.snapshot = json.dumps( - { - "tenant_account_join_info": serialize_sqlalchemy(ta), - "dismissed_members": [serialize_sqlalchemy(member) for member in members_to_dismiss], - }, - separators=(",", ":"), - ) - account_deletion_log_detail.role = ta.role - db.session.add(account_deletion_log_detail) - - -def _remove_account_from_tenant(ta, email): - """Remove the account from a tenant.""" - tenant_id = ta.tenant_id - db.session.delete(ta) - logger.info(f"Removed account {email} from tenant {tenant_id}.") - - # create account deletion log detail - account_deletion_log_detail = AccountDeletionLogDetail() - account_deletion_log_detail.account_id = ta.account_id - account_deletion_log_detail.tenant_id = tenant_id - account_deletion_log_detail.snapshot = json.dumps( - { - "tenant_account_join_info": serialize_sqlalchemy(ta), - }, - separators=(",", ":"), - ) - account_deletion_log_detail.role = ta.role - db.session.add(account_deletion_log_detail) + account = db.session.query(Account).filter(Account.id == account_id).first() + # send success email + send_deletion_success_task.delay(account.interface_language, account.email) From ec5f3129996b3a81c820621cdbb3c8f5f70ef2a2 Mon Sep 17 00:00:00 2001 From: GareArc Date: Mon, 23 Dec 2024 23:07:38 -0500 Subject: [PATCH 22/34] fix: delete migration and config --- api/configs/feature/__init__.py | 24 ++-------- ...added_account_deletion_logs_and_details.py | 48 ------------------- 2 files changed, 4 insertions(+), 68 deletions(-) delete mode 100644 api/migrations/versions/2024_12_23_0047-35e1a3223204_added_account_deletion_logs_and_details.py diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index 460e5ce3383f85..9ae01812894491 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -1,18 +1,10 @@ from typing import Annotated, Literal, Optional -from pydantic import ( - AliasChoices, - Field, - HttpUrl, - NegativeInt, - NonNegativeInt, - PositiveFloat, - PositiveInt, - computed_field, -) -from pydantic_settings import BaseSettings - from configs.feature.hosted_service import HostedServiceConfig +from pydantic import (AliasChoices, Field, HttpUrl, NegativeInt, + NonNegativeInt, PositiveFloat, PositiveInt, + computed_field) +from pydantic_settings import BaseSettings class SecurityConfig(BaseSettings): @@ -767,13 +759,6 @@ class LoginConfig(BaseSettings): ) -class RegisterConfig(BaseSettings): - EMAIL_FREEZE_PERIOD_IN_DAYS: PositiveInt = Field( - description="Freeze period in days for re-registering with the same email", - default=30, - ) - - class AccountConfig(BaseSettings): ACCOUNT_DELETION_TOKEN_EXPIRY_MINUTES: PositiveInt = Field( description="Duration in minutes for which a account deletion token remains valid", @@ -808,7 +793,6 @@ class FeatureConfig( WorkflowNodeExecutionConfig, WorkspaceConfig, LoginConfig, - RegisterConfig, AccountConfig, # hosted services config HostedServiceConfig, diff --git a/api/migrations/versions/2024_12_23_0047-35e1a3223204_added_account_deletion_logs_and_details.py b/api/migrations/versions/2024_12_23_0047-35e1a3223204_added_account_deletion_logs_and_details.py deleted file mode 100644 index d6e210bde83292..00000000000000 --- a/api/migrations/versions/2024_12_23_0047-35e1a3223204_added_account_deletion_logs_and_details.py +++ /dev/null @@ -1,48 +0,0 @@ -"""added account_deletion_logs and details - -Revision ID: 35e1a3223204 -Revises: d7999dfa4aae -Create Date: 2024-12-23 00:47:44.483419 - -""" -import models as models -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = '35e1a3223204' -down_revision = 'd7999dfa4aae' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('account_deletion_log_details', - sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), - sa.Column('account_id', models.types.StringUUID(), nullable=False), - sa.Column('tenant_id', models.types.StringUUID(), nullable=False), - sa.Column('role', sa.String(length=16), nullable=False), - sa.Column('snapshot', sa.Text(), nullable=False), - sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), - sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), - sa.PrimaryKeyConstraint('id', name='account_deletion_log_detail_pkey') - ) - op.create_table('account_deletion_logs', - sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), - sa.Column('email', sa.String(length=255), nullable=False), - sa.Column('reason', sa.Text(), nullable=True), - sa.Column('account_id', models.types.StringUUID(), nullable=False), - sa.Column('snapshot', sa.Text(), nullable=False), - sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), - sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False), - sa.PrimaryKeyConstraint('id', name='account_deletion_log_pkey') - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('account_deletion_logs') - op.drop_table('account_deletion_log_details') - # ### end Alembic commands ### From f4754317eedd5a18d7f71570689edee49ab77122 Mon Sep 17 00:00:00 2001 From: GareArc Date: Mon, 23 Dec 2024 23:08:29 -0500 Subject: [PATCH 23/34] reformat --- api/configs/feature/__init__.py | 16 ++++++-- api/libs/helper.py | 5 ++- api/services/account_service.py | 63 ++++++++++++++++++-------------- api/services/billing_service.py | 1 + api/tasks/delete_account_task.py | 3 +- 5 files changed, 53 insertions(+), 35 deletions(-) diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index 9ae01812894491..bceaa7ca2e8237 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -1,11 +1,19 @@ from typing import Annotated, Literal, Optional -from configs.feature.hosted_service import HostedServiceConfig -from pydantic import (AliasChoices, Field, HttpUrl, NegativeInt, - NonNegativeInt, PositiveFloat, PositiveInt, - computed_field) +from pydantic import ( + AliasChoices, + Field, + HttpUrl, + NegativeInt, + NonNegativeInt, + PositiveFloat, + PositiveInt, + computed_field, +) from pydantic_settings import BaseSettings +from configs.feature.hosted_service import HostedServiceConfig + class SecurityConfig(BaseSettings): """ diff --git a/api/libs/helper.py b/api/libs/helper.py index 0401e36811682c..91b1d1fe173d6f 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -12,12 +12,13 @@ from typing import Any, Optional, Union, cast from zoneinfo import available_timezones +from flask import Response, stream_with_context +from flask_restful import fields + from configs import dify_config from core.app.features.rate_limiting.rate_limit import RateLimitGenerator from core.file import helpers as file_helpers from extensions.ext_redis import redis_client -from flask import Response, stream_with_context -from flask_restful import fields from models.account import Account diff --git a/api/services/account_service.py b/api/services/account_service.py index 220609331cc71a..03a39ccbddef63 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -8,6 +8,11 @@ from hashlib import sha256 from typing import Any, Optional +from api.services.billing_service import BillingService +from pydantic import BaseModel +from sqlalchemy import func +from werkzeug.exceptions import Unauthorized + from configs import dify_config from constants.languages import language_timezone_mapping, languages from events.tenant_event import tenant_was_created @@ -17,35 +22,40 @@ from libs.passport import PassportService from libs.password import compare_password, hash_password, valid_password from libs.rsa import generate_key_pair -from models.account import (Account, AccountIntegrate, AccountStatus, Tenant, - TenantAccountJoin, TenantAccountJoinRole, - TenantAccountRole, TenantStatus) +from models.account import ( + Account, + AccountIntegrate, + AccountStatus, + Tenant, + TenantAccountJoin, + TenantAccountJoinRole, + TenantAccountRole, + TenantStatus, +) from models.model import DifySetup -from pydantic import BaseModel -from services.errors.account import (AccountAlreadyInTenantError, - AccountLoginError, AccountNotFoundError, - AccountNotLinkTenantError, - AccountPasswordError, - AccountRegisterError, - CannotOperateSelfError, - CurrentPasswordIncorrectError, - InvalidActionError, - LinkAccountIntegrateError, - MemberNotInTenantError, NoPermissionError, - RoleAlreadyAssignedError, - TenantNotFoundError) +from services.errors.account import ( + AccountAlreadyInTenantError, + AccountLoginError, + AccountNotFoundError, + AccountNotLinkTenantError, + AccountPasswordError, + AccountRegisterError, + CannotOperateSelfError, + CurrentPasswordIncorrectError, + InvalidActionError, + LinkAccountIntegrateError, + MemberNotInTenantError, + NoPermissionError, + RoleAlreadyAssignedError, + TenantNotFoundError, +) from services.errors.workspace import WorkSpaceNotAllowedCreateError from services.feature_service import FeatureService -from sqlalchemy import func from tasks.delete_account_task import delete_account_task -from tasks.mail_account_deletion_task import \ - send_account_deletion_verification_code +from tasks.mail_account_deletion_task import send_account_deletion_verification_code from tasks.mail_email_code_login import send_email_code_login_mail_task from tasks.mail_invite_member_task import send_invite_member_mail_task from tasks.mail_reset_password_task import send_reset_password_mail_task -from werkzeug.exceptions import Unauthorized - -from api.services.billing_service import BillingService class TokenPair(BaseModel): @@ -251,8 +261,7 @@ def generate_account_deletion_verification_code(account: Account) -> tuple[str, @classmethod def send_account_deletion_verification_email(cls, account: Account, code: str): if cls.email_code_account_deletion_rate_limiter.is_rate_limited(email): - from controllers.console.auth.error import \ - EmailCodeAccountDeletionRateLimitExceededError + from controllers.console.auth.error import EmailCodeAccountDeletionRateLimitExceededError raise EmailCodeAccountDeletionRateLimitExceededError() @@ -386,8 +395,7 @@ def send_reset_password_email( account_email = account.email if account else email if cls.reset_password_rate_limiter.is_rate_limited(account_email): - from controllers.console.auth.error import \ - PasswordResetRateLimitExceededError + from controllers.console.auth.error import PasswordResetRateLimitExceededError raise PasswordResetRateLimitExceededError() @@ -421,8 +429,7 @@ def send_email_code_login_email( cls, account: Optional[Account] = None, email: Optional[str] = None, language: Optional[str] = "en-US" ): if cls.email_code_login_rate_limiter.is_rate_limited(email): - from controllers.console.auth.error import \ - EmailCodeLoginRateLimitExceededError + from controllers.console.auth.error import EmailCodeLoginRateLimitExceededError raise EmailCodeLoginRateLimitExceededError() diff --git a/api/services/billing_service.py b/api/services/billing_service.py index 6aba5065233fd3..f930db40cc245a 100644 --- a/api/services/billing_service.py +++ b/api/services/billing_service.py @@ -1,6 +1,7 @@ import os import requests + from extensions.ext_database import db from models.account import TenantAccountJoin, TenantAccountRole diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index 465567c4ff175c..575f0bc4714472 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -1,6 +1,7 @@ import logging from celery import shared_task + from extensions.ext_database import db from models.account import Account from services.billing_service import BillingService @@ -14,7 +15,7 @@ def delete_account_task(account_id, reason: str): try: BillingService.delete_account(account_id, reason) except Exception as e: - logger.error(f"Failed to delete account {account_id} from billing service: {e}") + logger.exception(f"Failed to delete account {account_id} from billing service.") raise account = db.session.query(Account).filter(Account.id == account_id).first() From 5bf44316628daa5b5cd2e5721c42838101a86152 Mon Sep 17 00:00:00 2001 From: GareArc Date: Tue, 24 Dec 2024 04:50:13 +0000 Subject: [PATCH 24/34] fix wrong import --- api/services/account_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/services/account_service.py b/api/services/account_service.py index 03a39ccbddef63..76da0a1cfc3608 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -8,7 +8,7 @@ from hashlib import sha256 from typing import Any, Optional -from api.services.billing_service import BillingService +from services.billing_service import BillingService from pydantic import BaseModel from sqlalchemy import func from werkzeug.exceptions import Unauthorized From ec1c1269878fe0bbcfb2dd2ca8aa8b5ce4a4fc05 Mon Sep 17 00:00:00 2001 From: GareArc Date: Wed, 25 Dec 2024 00:05:34 -0500 Subject: [PATCH 25/34] fix: change celery queue name --- api/services/account_service.py | 10 +++++++--- api/tasks/delete_account_task.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/api/services/account_service.py b/api/services/account_service.py index 76da0a1cfc3608..005083988a92af 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -8,7 +8,6 @@ from hashlib import sha256 from typing import Any, Optional -from services.billing_service import BillingService from pydantic import BaseModel from sqlalchemy import func from werkzeug.exceptions import Unauthorized @@ -33,6 +32,7 @@ TenantStatus, ) from models.model import DifySetup +from services.billing_service import BillingService from services.errors.account import ( AccountAlreadyInTenantError, AccountLoginError, @@ -209,7 +209,9 @@ def create_account( raise AccountNotFound() if dify_config.BILLING_ENABLED and BillingService.is_email_in_freeze(email): - raise AccountRegisterError("Email is in freeze.") + raise AccountRegisterError( + "Unable to re-register the account because the deletion occurred less than 30 days ago" + ) account = Account() account.email = email @@ -848,7 +850,9 @@ def register( db.session.begin_nested() """Register account""" if dify_config.BILLING_ENABLED and BillingService.is_email_in_freeze(email): - raise AccountRegisterError("Email is in freeze.") + raise AccountRegisterError( + "Unable to re-register the account because the deletion occurred less than 30 days ago" + ) try: account = AccountService.create_account( email=email, diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index 575f0bc4714472..86e382a32c8b17 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) -@shared_task(queue="dataset") +@shared_task(queue="account_deletion") def delete_account_task(account_id, reason: str): try: BillingService.delete_account(account_id, reason) From 53e11274e692b8b82013d65b933358770120133e Mon Sep 17 00:00:00 2001 From: GareArc Date: Wed, 25 Dec 2024 00:14:29 -0500 Subject: [PATCH 26/34] fix: type check --- api/tasks/delete_account_task.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index 86e382a32c8b17..961bc43f94c7f0 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -19,5 +19,8 @@ def delete_account_task(account_id, reason: str): raise account = db.session.query(Account).filter(Account.id == account_id).first() + if not account: + logger.error(f"Account {account_id} not found.") + return # send success email send_deletion_success_task.delay(account.interface_language, account.email) From 5f4bb97210b52c26aaff02c2b93ddc177b0b5330 Mon Sep 17 00:00:00 2001 From: GareArc Date: Wed, 25 Dec 2024 00:26:43 -0500 Subject: [PATCH 27/34] fix: bugs --- api/services/account_service.py | 2 +- api/services/billing_service.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/services/account_service.py b/api/services/account_service.py index 005083988a92af..099fec94d63592 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -262,12 +262,12 @@ def generate_account_deletion_verification_code(account: Account) -> tuple[str, @classmethod def send_account_deletion_verification_email(cls, account: Account, code: str): + language, email = account.interface_language, account.email if cls.email_code_account_deletion_rate_limiter.is_rate_limited(email): from controllers.console.auth.error import EmailCodeAccountDeletionRateLimitExceededError raise EmailCodeAccountDeletionRateLimitExceededError() - language, email = account.interface_language, account.email send_account_deletion_verification_code.delay(language=language, to=email, code=code) cls.email_code_account_deletion_rate_limiter.increment_rate_limit(email) diff --git a/api/services/billing_service.py b/api/services/billing_service.py index f930db40cc245a..2b0e8f54a013fe 100644 --- a/api/services/billing_service.py +++ b/api/services/billing_service.py @@ -71,6 +71,6 @@ def is_email_in_freeze(cls, email: str) -> bool: params = {"email": email} try: response = cls._send_request("GET", "/account/in-freeze", params=params) - return response["data"] + return bool(response.get("data", False)) except Exception: return False From c99254db9f518e989df1f13c68a8f84ee84f1e81 Mon Sep 17 00:00:00 2001 From: GareArc Date: Wed, 25 Dec 2024 00:48:38 -0500 Subject: [PATCH 28/34] fix: ignore type for celey in mypy --- api/tasks/delete_account_task.py | 3 +-- api/tasks/mail_account_deletion_task.py | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index 961bc43f94c7f0..65b3f7758f8180 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -1,7 +1,6 @@ import logging -from celery import shared_task - +from celery import shared_task # type: ignore from extensions.ext_database import db from models.account import Account from services.billing_service import BillingService diff --git a/api/tasks/mail_account_deletion_task.py b/api/tasks/mail_account_deletion_task.py index d38df3613bd450..820924fc89a4c6 100644 --- a/api/tasks/mail_account_deletion_task.py +++ b/api/tasks/mail_account_deletion_task.py @@ -2,10 +2,9 @@ import time import click -from celery import shared_task -from flask import render_template - +from celery import shared_task # type: ignore from extensions.ext_mail import mail +from flask import render_template @shared_task(queue="mail") From 3a62679018a4d450e9bffed87470573894705a72 Mon Sep 17 00:00:00 2001 From: GareArc Date: Wed, 25 Dec 2024 00:51:36 -0500 Subject: [PATCH 29/34] reformat --- api/tasks/delete_account_task.py | 1 + api/tasks/mail_account_deletion_task.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index 65b3f7758f8180..3f61425c62f036 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -1,6 +1,7 @@ import logging from celery import shared_task # type: ignore + from extensions.ext_database import db from models.account import Account from services.billing_service import BillingService diff --git a/api/tasks/mail_account_deletion_task.py b/api/tasks/mail_account_deletion_task.py index 820924fc89a4c6..8d1434dc55098e 100644 --- a/api/tasks/mail_account_deletion_task.py +++ b/api/tasks/mail_account_deletion_task.py @@ -3,9 +3,10 @@ import click from celery import shared_task # type: ignore -from extensions.ext_mail import mail from flask import render_template +from extensions.ext_mail import mail + @shared_task(queue="mail") def send_deletion_success_task(language, to): From dd6747d3fd3789ca55061e7a41b42b18efaaa915 Mon Sep 17 00:00:00 2001 From: GareArc Date: Wed, 25 Dec 2024 02:28:21 -0500 Subject: [PATCH 30/34] feat: add seperate feedback api --- api/controllers/console/workspace/account.py | 17 +++++++++++++++++ api/services/billing_service.py | 6 ++++++ 2 files changed, 23 insertions(+) diff --git a/api/controllers/console/workspace/account.py b/api/controllers/console/workspace/account.py index 6e7f7e494b71da..8e3874a6fb7ce0 100644 --- a/api/controllers/console/workspace/account.py +++ b/api/controllers/console/workspace/account.py @@ -1,6 +1,7 @@ import datetime import pytz +from api.services.billing_service import BillingService from flask import request from flask_login import current_user from flask_restful import Resource, fields, marshal_with, reqparse @@ -279,6 +280,21 @@ def post(self): return {"result": "success"} +class AccountDeleleUpdateFeedbackApi(Resource): + @setup_required + def post(self): + account = current_user + + parser = reqparse.RequestParser() + parser.add_argument("email", type=str, required=True, location="json") + parser.add_argument("feedback", type=str, required=True, location="json") + args = parser.parse_args() + + BillingService.update_account_eletion_feedback(args["email"], args["feedback"]) + + return {"result": "success"} + + # Register API resources api.add_resource(AccountInitApi, "/account/init") api.add_resource(AccountProfileApi, "/account/profile") @@ -291,5 +307,6 @@ def post(self): api.add_resource(AccountIntegrateApi, "/account/integrates") api.add_resource(AccountDeleteVerifyApi, "/account/delete/verify") api.add_resource(AccountDeleteApi, "/account/delete") +api.add_resource(AccountDeleleUpdateFeedbackApi, "/account/delete/feedback") # api.add_resource(AccountEmailApi, '/account/email') # api.add_resource(AccountEmailVerifyApi, '/account/email-verify') diff --git a/api/services/billing_service.py b/api/services/billing_service.py index 2b0e8f54a013fe..053daa135b1d57 100644 --- a/api/services/billing_service.py +++ b/api/services/billing_service.py @@ -74,3 +74,9 @@ def is_email_in_freeze(cls, email: str) -> bool: return bool(response.get("data", False)) except Exception: return False + + @classmethod + def update_account_eletion_feedback(cls, email: str, feedback: str): + """Update account deletion feedback.""" + json = {"email": email, "feedback": feedback} + return cls._send_request("POST", "/account/deletion-feedback", json=json) From 2e9ac73a3d72fa425af7c826d1687f21b193d093 Mon Sep 17 00:00:00 2001 From: GareArc Date: Wed, 25 Dec 2024 02:48:25 -0500 Subject: [PATCH 31/34] fix: change task queue --- api/controllers/console/workspace/account.py | 4 ++-- api/services/billing_service.py | 2 +- api/tasks/delete_account_task.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/controllers/console/workspace/account.py b/api/controllers/console/workspace/account.py index 8e3874a6fb7ce0..70dd8b94aecd73 100644 --- a/api/controllers/console/workspace/account.py +++ b/api/controllers/console/workspace/account.py @@ -1,7 +1,6 @@ import datetime import pytz -from api.services.billing_service import BillingService from flask import request from flask_login import current_user from flask_restful import Resource, fields, marshal_with, reqparse @@ -22,6 +21,7 @@ from libs.login import login_required from models import AccountIntegrate, InvitationCode from services.account_service import AccountService +from services.billing_service import BillingService from services.errors.account import CurrentPasswordIncorrectError as ServiceCurrentPasswordIncorrectError @@ -290,7 +290,7 @@ def post(self): parser.add_argument("feedback", type=str, required=True, location="json") args = parser.parse_args() - BillingService.update_account_eletion_feedback(args["email"], args["feedback"]) + BillingService.update_account_deletion_feedback(args["email"], args["feedback"]) return {"result": "success"} diff --git a/api/services/billing_service.py b/api/services/billing_service.py index 053daa135b1d57..df9475bef8d3cc 100644 --- a/api/services/billing_service.py +++ b/api/services/billing_service.py @@ -76,7 +76,7 @@ def is_email_in_freeze(cls, email: str) -> bool: return False @classmethod - def update_account_eletion_feedback(cls, email: str, feedback: str): + def update_account_deletion_feedback(cls, email: str, feedback: str): """Update account deletion feedback.""" json = {"email": email, "feedback": feedback} return cls._send_request("POST", "/account/deletion-feedback", json=json) diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index 3f61425c62f036..fa7a0ce0bff6ab 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -10,7 +10,7 @@ logger = logging.getLogger(__name__) -@shared_task(queue="account_deletion") +@shared_task(queue="dataset") def delete_account_task(account_id, reason: str): try: BillingService.delete_account(account_id, reason) From ea30458a9d280c1262d56913e2bb585ce90240e1 Mon Sep 17 00:00:00 2001 From: GareArc Date: Wed, 25 Dec 2024 03:31:52 -0500 Subject: [PATCH 32/34] fix: wrong api route --- api/controllers/console/workspace/account.py | 2 +- api/services/billing_service.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/controllers/console/workspace/account.py b/api/controllers/console/workspace/account.py index 70dd8b94aecd73..1036f1284444bc 100644 --- a/api/controllers/console/workspace/account.py +++ b/api/controllers/console/workspace/account.py @@ -273,7 +273,7 @@ def post(self): args = parser.parse_args() if not AccountService.verify_account_deletion_code(args["token"], args["code"]): - return {"result": "fail", "error": "Verification code is invalid."}, 400 + raise ValueError("Invalid verification code.") AccountService.delete_account(account, args["reason"]) diff --git a/api/services/billing_service.py b/api/services/billing_service.py index df9475bef8d3cc..5d8494e9e61fd4 100644 --- a/api/services/billing_service.py +++ b/api/services/billing_service.py @@ -64,7 +64,7 @@ def is_tenant_owner_or_admin(current_user): def delete_account(cls, account_id: str, reason: str): """Delete account.""" params = {"account_id": account_id, "reason": reason} - return cls._send_request("DELETE", "/account", params=params) + return cls._send_request("DELETE", "/account/", params=params) @classmethod def is_email_in_freeze(cls, email: str) -> bool: From f55eb6cd987260ba803f3b52db8d1795798c9182 Mon Sep 17 00:00:00 2001 From: GareArc Date: Wed, 25 Dec 2024 04:08:02 -0500 Subject: [PATCH 33/34] fix: bug --- api/tasks/delete_account_task.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index fa7a0ce0bff6ab..921f9300253f4a 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -1,7 +1,6 @@ import logging from celery import shared_task # type: ignore - from extensions.ext_database import db from models.account import Account from services.billing_service import BillingService @@ -12,13 +11,13 @@ @shared_task(queue="dataset") def delete_account_task(account_id, reason: str): + account = db.session.query(Account).filter(Account.id == account_id).first() try: BillingService.delete_account(account_id, reason) except Exception as e: logger.exception(f"Failed to delete account {account_id} from billing service.") raise - account = db.session.query(Account).filter(Account.id == account_id).first() if not account: logger.error(f"Account {account_id} not found.") return From 49268d811fb67a350e60e0fb7552891f952eccce Mon Sep 17 00:00:00 2001 From: GareArc Date: Wed, 25 Dec 2024 04:08:22 -0500 Subject: [PATCH 34/34] reformat --- api/tasks/delete_account_task.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/tasks/delete_account_task.py b/api/tasks/delete_account_task.py index 921f9300253f4a..d005e1178f9eb7 100644 --- a/api/tasks/delete_account_task.py +++ b/api/tasks/delete_account_task.py @@ -1,6 +1,7 @@ import logging from celery import shared_task # type: ignore + from extensions.ext_database import db from models.account import Account from services.billing_service import BillingService