Merge branch 'refs/heads/fix/dataset_operator' into deploy/dev

* refs/heads/fix/dataset_operator: (33 commits)
  feat: update dataset sort
  feat: add dataset_permissions tenant_id
  chore: optimize memory fetch performance (#6039)
  feat: support moonshot and glm base models for volcengine provider (#6029)
  Optimize db config (#6011)
  fix: token count includes base64 string of input images (#5868)
  chore: skip pip upgrade preparation in api dockerfile (#5999)
  feat(*): Swtich to dify_config. (#6025)
  fix: the input field of tool panel not worked as expected (#6003)
  Add 2 firecrawl tools : Scrape and Search (#6016)
  test(test_rerank): Remove duplicate test cases. (#6024)
  chore: optimize memory messages fetch count limit (#6021)
  Revert "feat: knowledge admin role" (#6018)
  feat: add Llama 3 and Mixtral model options to ddgo_ai.yaml (#5979)
  fix: add status_code 304 (#6000)
  6014 i18n add support for spanish (#6017)
  [Feature] Support loading for mermaid. (#6004)
  fix: update workflow trace query (#6010)
  Removed firecrawl-py, fixed and improved firecrawl tool (#5896)
  fix API tool's schema not support array (#6006)
  ...

.gitignore

@@ -174,3 +174,5 @@ sdks/python-client/dify_client.egg-info
 .vscode/*
 !.vscode/launch.json
 pyrightconfig.json
+
+.idea/

api/Dockerfile

@@ -5,8 +5,7 @@ WORKDIR /app/api
 
 # Install Poetry
 ENV POETRY_VERSION=1.8.3
-RUN pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir --upgrade poetry==${POETRY_VERSION}
+RUN pip install --no-cache-dir poetry==${POETRY_VERSION}
 
 # Configure Poetry
 ENV POETRY_CACHE_DIR=/tmp/poetry_cache

api/README.md

@@ -11,7 +11,7 @@
 
    ```bash
    cd ../docker
-   cp .middleware.env.example .middleware.env
+   cp middleware.env.example middleware.env
    docker compose -f docker-compose.middleware.yaml -p dify up -d
    cd ../api
    ```

api/app.py

@@ -2,7 +2,7 @@
 
 from configs import dify_config
 
-if not os.environ.get("DEBUG") or os.environ.get("DEBUG", "false").lower() != 'true':
+if os.environ.get("DEBUG", "false").lower() != 'true':
     from gevent import monkey
 
     monkey.patch_all()
@@ -43,6 +43,8 @@
 from extensions.ext_database import db
 from extensions.ext_login import login_manager
 from libs.passport import PassportService
+
+# TODO: Find a way to avoid importing models here
 from models import account, dataset, model, source, task, tool, tools, web
 from services.account_service import AccountService
 

api/configs/app_config.py

@@ -1,3 +1,4 @@
+from pydantic import computed_field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from configs.deploy import DeploymentConfig
@@ -43,3 +44,28 @@ class DifyConfig(
         # ignore extra attributes
         extra='ignore',
     )
+
+    CODE_MAX_NUMBER: int = 9223372036854775807
+    CODE_MIN_NUMBER: int = -9223372036854775808
+    CODE_MAX_STRING_LENGTH: int = 80000
+    CODE_MAX_STRING_ARRAY_LENGTH: int = 30
+    CODE_MAX_OBJECT_ARRAY_LENGTH: int = 30
+    CODE_MAX_NUMBER_ARRAY_LENGTH: int = 1000
+
+    HTTP_REQUEST_MAX_CONNECT_TIMEOUT: int = 300
+    HTTP_REQUEST_MAX_READ_TIMEOUT: int = 600
+    HTTP_REQUEST_MAX_WRITE_TIMEOUT: int = 600
+    HTTP_REQUEST_NODE_MAX_BINARY_SIZE: int = 1024 * 1024 * 10
+
+    @computed_field
+    def HTTP_REQUEST_NODE_READABLE_MAX_BINARY_SIZE(self) -> str:
+        return f'{self.HTTP_REQUEST_NODE_MAX_BINARY_SIZE / 1024 / 1024:.2f}MB'
+
+    HTTP_REQUEST_NODE_MAX_TEXT_SIZE: int = 1024 * 1024
+
+    @computed_field
+    def HTTP_REQUEST_NODE_READABLE_MAX_TEXT_SIZE(self) -> str:
+        return f'{self.HTTP_REQUEST_NODE_MAX_TEXT_SIZE / 1024 / 1024:.2f}MB'
+
+    SSRF_PROXY_HTTP_URL: str | None = None
+    SSRF_PROXY_HTTPS_URL: str | None = None

api/configs/feature/__init__.py

@@ -17,6 +17,10 @@ class SecurityConfig(BaseModel):
         default=None,
     )
 
+    RESET_PASSWORD_TOKEN_EXPIRY_HOURS: PositiveInt = Field(
+        description='Expiry time in hours for reset token',
+        default=24,
+    )
 
 class AppExecutionConfig(BaseModel):
     """

api/configs/middleware/__init__.py

@@ -81,6 +81,11 @@ class DatabaseConfig:
         default='',
     )
 
+    DB_EXTRAS: str = Field(
+        description='db extras options. Example: keepalives_idle=60&keepalives=1',
+        default='',
+    )
+
     SQLALCHEMY_DATABASE_URI_SCHEME: str = Field(
         description='db uri scheme',
         default='postgresql',
@@ -89,7 +94,12 @@ class DatabaseConfig:
     @computed_field
     @property
     def SQLALCHEMY_DATABASE_URI(self) -> str:
-        db_extras = f"?client_encoding={self.DB_CHARSET}" if self.DB_CHARSET else ""
+        db_extras = (
+            f"{self.DB_EXTRAS}&client_encoding={self.DB_CHARSET}"
+            if self.DB_CHARSET
+            else self.DB_EXTRAS
+        ).strip("&")
+        db_extras = f"?{db_extras}" if db_extras else ""
         return (f"{self.SQLALCHEMY_DATABASE_URI_SCHEME}://"
                 f"{self.DB_USERNAME}:{self.DB_PASSWORD}@{self.DB_HOST}:{self.DB_PORT}/{self.DB_DATABASE}"
                 f"{db_extras}")
@@ -114,7 +124,7 @@ def SQLALCHEMY_DATABASE_URI(self) -> str:
         default=False,
     )
 
-    SQLALCHEMY_ECHO: bool = Field(
+    SQLALCHEMY_ECHO: bool | str = Field(
         description='whether to enable SqlAlchemy echo',
         default=False,
     )

api/configs/middleware/vdb/tencent_vector_config.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import BaseModel, Field, PositiveInt
+from pydantic import BaseModel, Field, NonNegativeInt, PositiveInt
 
 
 class TencentVectorDBConfig(BaseModel):
@@ -24,7 +24,7 @@ class TencentVectorDBConfig(BaseModel):
     )
 
     TENCENT_VECTOR_DB_USERNAME: Optional[str] = Field(
-        description='Tencent Vector password',
+        description='Tencent Vector username',
         default=None,
     )
 
@@ -38,7 +38,7 @@ class TencentVectorDBConfig(BaseModel):
         default=1,
     )
 
-    TENCENT_VECTOR_DB_REPLICAS: PositiveInt = Field(
+    TENCENT_VECTOR_DB_REPLICAS: NonNegativeInt = Field(
         description='Tencent Vector replicas',
         default=2,
     )

api/constants/languages.py

@@ -14,7 +14,7 @@
     'vi-VN': 'Asia/Ho_Chi_Minh',
     'ro-RO': 'Europe/Bucharest',
     'pl-PL': 'Europe/Warsaw',
-    'hi-IN': 'Asia/Kolkata'
+    'hi-IN': 'Asia/Kolkata',
 }
 
 languages = list(language_timezone_mapping.keys())

api/controllers/console/__init__.py

@@ -30,7 +30,7 @@
 )
 
 # Import auth controllers
-from .auth import activate, data_source_bearer_auth, data_source_oauth, login, oauth
+from .auth import activate, data_source_bearer_auth, data_source_oauth, forgot_password, login, oauth
 
 # Import billing controllers
 from .billing import billing

api/controllers/console/auth/data_source_oauth.py

@@ -6,6 +6,7 @@
 from flask_restful import Resource
 from werkzeug.exceptions import Forbidden
 
+from configs import dify_config
 from controllers.console import api
 from libs.login import login_required
 from libs.oauth_data_source import NotionOAuth
@@ -16,11 +17,11 @@
 
 def get_oauth_providers():
     with current_app.app_context():
-        notion_oauth = NotionOAuth(client_id=current_app.config.get('NOTION_CLIENT_ID'),
-                                   client_secret=current_app.config.get(
-                                       'NOTION_CLIENT_SECRET'),
-                                   redirect_uri=current_app.config.get(
-                                       'CONSOLE_API_URL') + '/console/api/oauth/data-source/callback/notion')
+        if not dify_config.NOTION_CLIENT_ID or not dify_config.NOTION_CLIENT_SECRET:
+            return {}
+        notion_oauth = NotionOAuth(client_id=dify_config.NOTION_CLIENT_ID,
+                                   client_secret=dify_config.NOTION_CLIENT_SECRET,
+                                   redirect_uri=dify_config.CONSOLE_API_URL + '/console/api/oauth/data-source/callback/notion')
 
         OAUTH_PROVIDERS = {
             'notion': notion_oauth
@@ -39,8 +40,10 @@ def get(self, provider: str):
             print(vars(oauth_provider))
         if not oauth_provider:
             return {'error': 'Invalid provider'}, 400
-        if current_app.config.get('NOTION_INTEGRATION_TYPE') == 'internal':
-            internal_secret = current_app.config.get('NOTION_INTERNAL_SECRET')
+        if dify_config.NOTION_INTEGRATION_TYPE == 'internal':
+            internal_secret = dify_config.NOTION_INTERNAL_SECRET
+            if not internal_secret:
+                return {'error': 'Internal secret is not set'},
             oauth_provider.save_internal_access_token(internal_secret)
             return { 'data': '' }
         else:
@@ -60,13 +63,13 @@ def get(self, provider: str):
         if 'code' in request.args:
             code = request.args.get('code')
 
-            return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?type=notion&code={code}')
+            return redirect(f'{dify_config.CONSOLE_WEB_URL}?type=notion&code={code}')
         elif 'error' in request.args:
             error = request.args.get('error')
 
-            return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?type=notion&error={error}')
+            return redirect(f'{dify_config.CONSOLE_WEB_URL}?type=notion&error={error}')
         else:
-            return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?type=notion&error=Access denied')
+            return redirect(f'{dify_config.CONSOLE_WEB_URL}?type=notion&error=Access denied')
 
 
 class OAuthDataSourceBinding(Resource):

api/controllers/console/auth/error.py

@@ -5,3 +5,28 @@ class ApiKeyAuthFailedError(BaseHTTPException):
     error_code = 'auth_failed'
     description = "{message}"
     code = 500
+
+
+class InvalidEmailError(BaseHTTPException):
+    error_code = 'invalid_email'
+    description = "The email address is not valid."
+    code = 400
+
+
+class PasswordMismatchError(BaseHTTPException):
+    error_code = 'password_mismatch'
+    description = "The passwords do not match."
+    code = 400
+
+
+class InvalidTokenError(BaseHTTPException):
+    error_code = 'invalid_or_expired_token'
+    description = "The token is invalid or has expired."
+    code = 400
+
+
+class PasswordResetRateLimitExceededError(BaseHTTPException):
+    error_code = 'password_reset_rate_limit_exceeded'
+    description = "Password reset rate limit exceeded. Try again later."
+    code = 429
+

api/controllers/console/auth/forgot_password.py

@@ -0,0 +1,107 @@
+import base64
+import logging
+import secrets
+
+from flask_restful import Resource, reqparse
+
+from controllers.console import api
+from controllers.console.auth.error import (
+    InvalidEmailError,
+    InvalidTokenError,
+    PasswordMismatchError,
+    PasswordResetRateLimitExceededError,
+)
+from controllers.console.setup import setup_required
+from extensions.ext_database import db
+from libs.helper import email as email_validate
+from libs.password import hash_password, valid_password
+from models.account import Account
+from services.account_service import AccountService
+from services.errors.account import RateLimitExceededError
+
+
+class ForgotPasswordSendEmailApi(Resource):
+
+    @setup_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('email', type=str, required=True, location='json')
+        args = parser.parse_args()
+
+        email = args['email']
+
+        if not email_validate(email):
+            raise InvalidEmailError()
+
+        account = Account.query.filter_by(email=email).first()
+
+        if account:
+            try:
+                AccountService.send_reset_password_email(account=account)
+            except RateLimitExceededError:
+                logging.warning(f"Rate limit exceeded for email: {account.email}")
+                raise PasswordResetRateLimitExceededError()
+        else:
+            # Return success to avoid revealing email registration status
+            logging.warning(f"Attempt to reset password for unregistered email: {email}")
+
+        return {"result": "success"}
+
+
+class ForgotPasswordCheckApi(Resource):
+
+    @setup_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('token', type=str, required=True, nullable=False, location='json')
+        args = parser.parse_args()
+        token = args['token']
+
+        reset_data = AccountService.get_reset_password_data(token)
+
+        if reset_data is None:
+            return {'is_valid': False, 'email': None}
+        return {'is_valid': True, 'email': reset_data.get('email')}
+
+
+class ForgotPasswordResetApi(Resource):
+
+    @setup_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('token', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('new_password', type=valid_password, required=True, nullable=False, location='json')
+        parser.add_argument('password_confirm', type=valid_password, required=True, nullable=False, location='json')
+        args = parser.parse_args()
+
+        new_password = args['new_password']
+        password_confirm = args['password_confirm']
+
+        if str(new_password).strip() != str(password_confirm).strip():
+            raise PasswordMismatchError()
+
+        token = args['token']
+        reset_data = AccountService.get_reset_password_data(token)
+
+        if reset_data is None:
+            raise InvalidTokenError()
+
+        AccountService.revoke_reset_password_token(token)
+
+        salt = secrets.token_bytes(16)
+        base64_salt = base64.b64encode(salt).decode()
+
+        password_hashed = hash_password(new_password, salt)
+        base64_password_hashed = base64.b64encode(password_hashed).decode()
+
+        account = Account.query.filter_by(email=reset_data.get('email')).first()
+        account.password = base64_password_hashed
+        account.password_salt = base64_salt
+        db.session.commit()
+
+        return {'result': 'success'}
+
+
+api.add_resource(ForgotPasswordSendEmailApi, '/forgot-password')
+api.add_resource(ForgotPasswordCheckApi, '/forgot-password/validity')
+api.add_resource(ForgotPasswordResetApi, '/forgot-password/resets')