Merge branch 'refs/heads/main' into feat/update-prompt-generate

* refs/heads/main: (62 commits)
  fix: extract tool calls correctly while arguments is empty (#6503)
  clean vector collection redis cache (#6494)
  refactor(api): switch to dify_config in controllers/console (#6485)
  update delete embeddings by id (#6489)
  update clean embedding cache query logic (#6483)
  Fix/extra table tracing app config (#6487)
  feat(tool): getimg.ai integration (#6260)
  fix: next suggest question logic problem (#6451)
  fix: build error (#6480)
  fix: correct gpt-4o-mini max token (#6472)
  Update CONTRIBUTING_CN "安装常见问题解答" link. (#6470)
  Update CONTRIBUTING "installation FAQ" link. (#6471)
  Update CONTRIBUTING_JA "installation FAQ" link. (#6469)
  chore: some components upgrage to new ui (#6468)
  fix wrong method using (#6459)
  feat: support gpt-4o-mini for openrouter provider (#6447)
  update ernie models (#6454)
  chore: make text generation timeout duration configurable (#6450)
  refactor(myscale):Set the default value of the myscale vector db in DifyConfig. (#6441)
  [EMERGENCY] Fix Anthropic header issue (#6445)
  ...

CONTRIBUTING.md

@@ -81,7 +81,7 @@ Dify requires the following dependencies to build, make sure they're installed o
 
 Dify is composed of a backend and a frontend. Navigate to the backend directory by `cd api/`, then follow the [Backend README](api/README.md) to install it. In a separate terminal, navigate to the frontend directory by `cd web/`, then follow the [Frontend README](web/README.md) to install.
 
-Check the [installation FAQ](https://docs.dify.ai/getting-started/faq/install-faq) for a list of common issues and steps to troubleshoot.
+Check the [installation FAQ](https://docs.dify.ai/learn-more/faq/self-host-faq) for a list of common issues and steps to troubleshoot.
 
 ### 5. Visit dify in your browser
 

CONTRIBUTING_CN.md

@@ -77,7 +77,7 @@ Dify 依赖以下工具和库：
 
 Dify 由后端和前端组成。通过 `cd api/` 导航到后端目录，然后按照 [后端 README](api/README.md) 进行安装。在另一个终端中，通过 `cd web/` 导航到前端目录，然后按照 [前端 README](web/README.md) 进行安装。
 
-查看 [安装常见问题解答](https://docs.dify.ai/getting-started/faq/install-faq) 以获取常见问题列表和故障排除步骤。
+查看 [安装常见问题解答](https://docs.dify.ai/v/zh-hans/learn-more/faq/install-faq) 以获取常见问题列表和故障排除步骤。
 
 ### 5. 在浏览器中访问 Dify
 

CONTRIBUTING_JA.md

@@ -82,7 +82,7 @@ Dify はバックエンドとフロントエンドから構成されています
 まず`cd api/`でバックエンドのディレクトリに移動し、[Backend README](api/README.md)に従ってインストールします。
 次に別のターミナルで、`cd web/`でフロントエンドのディレクトリに移動し、[Frontend README](web/README.md)に従ってインストールしてください。
 
-よくある問題とトラブルシューティングの手順については、[installation FAQ](https://docs.dify.ai/getting-started/faq/install-faq) を確認してください。
+よくある問題とトラブルシューティングの手順については、[installation FAQ](https://docs.dify.ai/v/japanese/learn-more/faq/install-faq) を確認してください。
 
 ### 5. ブラウザで dify にアクセスする
 

api/.env.example

@@ -256,3 +256,7 @@ WORKFLOW_CALL_MAX_DEPTH=5
 # App configuration
 APP_MAX_EXECUTION_TIME=1200
 APP_MAX_ACTIVE_REQUESTS=0
+
+
+# Celery beat configuration
+CELERY_BEAT_SCHEDULER_TIME=1

api/configs/feature/__init__.py

@@ -23,6 +23,7 @@ class SecurityConfig(BaseSettings):
         default=24,
     )
 
+
 class AppExecutionConfig(BaseSettings):
     """
     App Execution configs
@@ -435,6 +436,13 @@ class ImageFormatConfig(BaseSettings):
     )
 
 
+class CeleryBeatConfig(BaseSettings):
+    CELERY_BEAT_SCHEDULER_TIME: int = Field(
+        description='the time of the celery scheduler, default to 1 day',
+        default=1,
+    )
+
+
 class FeatureConfig(
     # place the configs in alphabet order
     AppExecutionConfig,
@@ -462,5 +470,6 @@ class FeatureConfig(
 
     # hosted services config
     HostedServiceConfig,
+    CeleryBeatConfig,
 ):
     pass

api/configs/feature/hosted_service/__init__.py

@@ -79,7 +79,7 @@ class HostedAzureOpenAiConfig(BaseSettings):
         default=False,
     )
 
-    HOSTED_OPENAI_API_KEY: Optional[str] = Field(
+    HOSTED_AZURE_OPENAI_API_KEY: Optional[str] = Field(
         description='',
         default=None,
     )

api/configs/middleware/vdb/myscale_config.py

@@ -1,4 +1,3 @@
-from typing import Optional
 
 from pydantic import BaseModel, Field, PositiveInt
 
@@ -8,32 +7,32 @@ class MyScaleConfig(BaseModel):
     MyScale configs
     """
 
-    MYSCALE_HOST: Optional[str] = Field(
+    MYSCALE_HOST: str = Field(
         description='MyScale host',
-        default=None,
+        default='localhost',
     )
 
-    MYSCALE_PORT: Optional[PositiveInt] = Field(
+    MYSCALE_PORT: PositiveInt = Field(
         description='MyScale port',
         default=8123,
     )
 
-    MYSCALE_USER: Optional[str] = Field(
+    MYSCALE_USER: str = Field(
         description='MyScale user',
-        default=None,
+        default='default',
     )
 
-    MYSCALE_PASSWORD: Optional[str] = Field(
+    MYSCALE_PASSWORD: str = Field(
         description='MyScale password',
-        default=None,
+        default='',
     )
 
-    MYSCALE_DATABASE: Optional[str] = Field(
+    MYSCALE_DATABASE: str = Field(
         description='MyScale database name',
-        default=None,
+        default='default',
     )
 
-    MYSCALE_FTS_PARAMS: Optional[str] = Field(
+    MYSCALE_FTS_PARAMS: str = Field(
         description='MyScale fts index parameters',
-        default=None,
+        default='',
     )

api/controllers/console/datasets/datasets.py

@@ -1,10 +1,11 @@
 import flask_restful
-from flask import current_app, request
+from flask import request
 from flask_login import current_user
 from flask_restful import Resource, marshal, marshal_with, reqparse
 from werkzeug.exceptions import Forbidden, NotFound
 
 import services
+from configs import dify_config
 from controllers.console import api
 from controllers.console.apikey import api_key_fields, api_key_list
 from controllers.console.app.error import ProviderNotInitializeError
@@ -530,7 +531,7 @@ class DatasetApiBaseUrlApi(Resource):
     @account_initialization_required
     def get(self):
         return {
-            'api_base_url': (current_app.config['SERVICE_API_URL'] if current_app.config['SERVICE_API_URL']
+            'api_base_url': (dify_config.SERVICE_API_URL if dify_config.SERVICE_API_URL
                              else request.host_url.rstrip('/')) + '/v1'
         }
 
@@ -540,20 +541,20 @@ class DatasetRetrievalSettingApi(Resource):
     @login_required
     @account_initialization_required
     def get(self):
-        vector_type = current_app.config['VECTOR_STORE']
+        vector_type = dify_config.VECTOR_STORE
         match vector_type:
             case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE:
                 return {
                     'retrieval_method': [
-                        RetrievalMethod.SEMANTIC_SEARCH
+                        RetrievalMethod.SEMANTIC_SEARCH.value
                     ]
                 }
             case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH | VectorType.ANALYTICDB | VectorType.MYSCALE:
                 return {
                     'retrieval_method': [
-                        RetrievalMethod.SEMANTIC_SEARCH,
-                        RetrievalMethod.FULL_TEXT_SEARCH,
-                        RetrievalMethod.HYBRID_SEARCH,
+                        RetrievalMethod.SEMANTIC_SEARCH.value,
+                        RetrievalMethod.FULL_TEXT_SEARCH.value,
+                        RetrievalMethod.HYBRID_SEARCH.value,
                     ]
                 }
             case _:
@@ -569,15 +570,15 @@ def get(self, vector_type):
             case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE:
                 return {
                     'retrieval_method': [
-                        RetrievalMethod.SEMANTIC_SEARCH
+                        RetrievalMethod.SEMANTIC_SEARCH.value
                     ]
                 }
             case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH| VectorType.ANALYTICDB | VectorType.MYSCALE:
                 return {
                     'retrieval_method': [
-                        RetrievalMethod.SEMANTIC_SEARCH,
-                        RetrievalMethod.FULL_TEXT_SEARCH,
-                        RetrievalMethod.HYBRID_SEARCH,
+                        RetrievalMethod.SEMANTIC_SEARCH.value,
+                        RetrievalMethod.FULL_TEXT_SEARCH.value,
+                        RetrievalMethod.HYBRID_SEARCH.value,
                     ]
                 }
             case _:

api/controllers/console/datasets/datasets_segments.py

@@ -75,7 +75,7 @@ def get(self, dataset_id, document_id):
         )
 
         if last_id is not None:
-            last_segment = DocumentSegment.query.get(str(last_id))
+            last_segment = db.session.get(DocumentSegment, str(last_id))
             if last_segment:
                 query = query.filter(
                     DocumentSegment.position > last_segment.position)

api/controllers/console/datasets/file.py

@@ -1,8 +1,9 @@
-from flask import current_app, request
+from flask import request
 from flask_login import current_user
 from flask_restful import Resource, marshal_with
 
 import services
+from configs import dify_config
 from controllers.console import api
 from controllers.console.datasets.error import (
     FileTooLargeError,
@@ -26,9 +27,9 @@ class FileApi(Resource):
     @account_initialization_required
     @marshal_with(upload_config_fields)
     def get(self):
-        file_size_limit = current_app.config.get("UPLOAD_FILE_SIZE_LIMIT")
-        batch_count_limit = current_app.config.get("UPLOAD_FILE_BATCH_LIMIT")
-        image_file_size_limit = current_app.config.get("UPLOAD_IMAGE_FILE_SIZE_LIMIT")
+        file_size_limit = dify_config.UPLOAD_FILE_SIZE_LIMIT
+        batch_count_limit = dify_config.UPLOAD_FILE_BATCH_LIMIT
+        image_file_size_limit = dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT
         return {
             'file_size_limit': file_size_limit,
             'batch_count_limit': batch_count_limit,
@@ -76,7 +77,7 @@ class FileSupportTypeApi(Resource):
     @login_required
     @account_initialization_required
     def get(self):
-        etl_type = current_app.config['ETL_TYPE']
+        etl_type = dify_config.ETL_TYPE
         allowed_extensions = UNSTRUCTURED_ALLOWED_EXTENSIONS if etl_type == 'Unstructured' else ALLOWED_EXTENSIONS
         return {'allowed_extensions': allowed_extensions}
 

api/controllers/console/explore/audio.py

@@ -78,10 +78,12 @@ def post(self, installed_app):
             parser = reqparse.RequestParser()
             parser.add_argument('message_id', type=str, required=False, location='json')
             parser.add_argument('voice', type=str, location='json')
+            parser.add_argument('text', type=str, location='json')
             parser.add_argument('streaming', type=bool, location='json')
             args = parser.parse_args()
 
-            message_id = args.get('message_id')
+            message_id = args.get('message_id', None)
+            text = args.get('text', None)
             if (app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]
                     and app_model.workflow
                     and app_model.workflow.features_dict):
@@ -95,7 +97,8 @@ def post(self, installed_app):
             response = AudioService.transcript_tts(
                 app_model=app_model,
                 message_id=message_id,
-                voice=voice
+                voice=voice,
+                text=text
             )
             return response
         except services.errors.app_model_config.AppModelConfigBrokenError:

api/controllers/console/explore/parameter.py

@@ -1,7 +1,7 @@
 
-from flask import current_app
 from flask_restful import fields, marshal_with
 
+from configs import dify_config
 from controllers.console import api
 from controllers.console.app.error import AppUnavailableError
 from controllers.console.explore.wraps import InstalledAppResource
@@ -78,7 +78,7 @@ def get(self, installed_app: InstalledApp):
                                                      "transfer_methods": ["remote_url", "local_file"]
                                                  }}),
             'system_parameters': {
-                'image_file_size_limit': current_app.config.get('UPLOAD_IMAGE_FILE_SIZE_LIMIT')
+                'image_file_size_limit': dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT
             }
         }
 

api/controllers/console/init_validate.py

@@ -1,8 +1,9 @@
 import os
 
-from flask import current_app, session
+from flask import session
 from flask_restful import Resource, reqparse
 
+from configs import dify_config
 from libs.helper import str_len
 from models.model import DifySetup
 from services.account_service import TenantService
@@ -40,7 +41,7 @@ def post(self):
         return {'result': 'success'}, 201
 
 def get_init_validate_status():
-    if current_app.config['EDITION'] == 'SELF_HOSTED':
+    if dify_config.EDITION == 'SELF_HOSTED':
         if os.environ.get('INIT_PASSWORD'):
             return session.get('is_init_validated') or DifySetup.query.first()
 

api/controllers/console/setup.py

@@ -1,8 +1,9 @@
 from functools import wraps
 
-from flask import current_app, request
+from flask import request
 from flask_restful import Resource, reqparse
 
+from configs import dify_config
 from libs.helper import email, get_remote_ip, str_len
 from libs.password import valid_password
 from models.model import DifySetup
@@ -17,7 +18,7 @@
 class SetupApi(Resource):
 
     def get(self):
-        if current_app.config['EDITION'] == 'SELF_HOSTED':
+        if dify_config.EDITION == 'SELF_HOSTED':
             setup_status = get_setup_status()
             if setup_status:
                 return {
@@ -77,7 +78,7 @@ def decorated(*args, **kwargs):
 
 
 def get_setup_status():
-    if current_app.config['EDITION'] == 'SELF_HOSTED':
+    if dify_config.EDITION == 'SELF_HOSTED':
         return DifySetup.query.first()
     else:
         return True

api/controllers/console/version.py

@@ -3,9 +3,10 @@
 import logging
 
 import requests
-from flask import current_app
 from flask_restful import Resource, reqparse
 
+from configs import dify_config
+
 from . import api
 
 
@@ -15,16 +16,16 @@ def get(self):
         parser = reqparse.RequestParser()
         parser.add_argument('current_version', type=str, required=True, location='args')
         args = parser.parse_args()
-        check_update_url = current_app.config['CHECK_UPDATE_URL']
+        check_update_url = dify_config.CHECK_UPDATE_URL
 
         result = {
-            'version': current_app.config['CURRENT_VERSION'],
+            'version': dify_config.CURRENT_VERSION,
             'release_date': '',
             'release_notes': '',
             'can_auto_update': False,
             'features': {
-                'can_replace_logo': current_app.config['CAN_REPLACE_LOGO'],
-                'model_load_balancing_enabled': current_app.config['MODEL_LB_ENABLED']
+                'can_replace_logo': dify_config.CAN_REPLACE_LOGO,
+                'model_load_balancing_enabled': dify_config.MODEL_LB_ENABLED
             }
         }
 

api/controllers/console/workspace/account.py

@@ -1,10 +1,11 @@
 import datetime
 
 import pytz
-from flask import current_app, request
+from flask import request
 from flask_login import current_user
 from flask_restful import Resource, fields, marshal_with, reqparse
 
+from configs import dify_config
 from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.setup import setup_required
@@ -36,7 +37,7 @@ def post(self):
 
         parser = reqparse.RequestParser()
 
-        if current_app.config['EDITION'] == 'CLOUD':
+        if dify_config.EDITION == 'CLOUD':
             parser.add_argument('invitation_code', type=str, location='json')
 
         parser.add_argument(
@@ -45,7 +46,7 @@ def post(self):
                             required=True, location='json')
         args = parser.parse_args()
 
-        if current_app.config['EDITION'] == 'CLOUD':
+        if dify_config.EDITION == 'CLOUD':
             if not args['invitation_code']:
                 raise ValueError('invitation_code is required')