Merge branch 'refs/heads/feat/knowledge-admin-role' into deploy/dev

* refs/heads/feat/knowledge-admin-role:
  feat: dataset documents rename permission
  feat: add claude3 function calling (#5889)
  refactor: Create a `dify_config` with Pydantic. (#5938)
  fix: zhipuai pytest correction (#5934)
  docs(api/core/tools/docs/en_US/tool_scale_out.md): Format by markdownlint. (#5903)
  chore: remove dify SaaS URL in default configs (#5888)
  feat: pr template (#5886)
  fix bug : TencentVectorDBConfig Add  TENCENT_VECTOR_DB_DATABASE (#5879)
  refactor(api/core/app/apps/base_app_generator.py): improve input validation and sanitization in BaseAppGenerator (#5866)
  chore:  click area that trigger showing tracing config is too large (#5878)
  Fix/remove tsne position test (#5858)
  Fix/docker env namings (#5857)
  doc: docker-compose won't start due to wrong README (#5859)
  fix:retieval setting document link 404 (#5861)
  chore:remove .env.example duplicate key (#5853)
  fix: not show opening question if the opening message is empty (#5856)
  fix: react.js error 185 maximum update depth exceeded in streaming responses during conversation (#5849)

.github/pull_request_template.md

@@ -1,32 +1,34 @@
+# Checklist:
+
+> [!IMPORTANT]  
+> Please review the checklist below before submitting your pull request.
+
+- [ ] Please open an issue before creating a PR or link to an existing issue
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I ran `dev/reformat`(backend) and `cd web && npx lint-staged`(frontend) to appease the lint gods
+
 # Description
 
-Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
+Describe the big picture of your changes here to communicate to the maintainers why we should accept this pull request. If it fixes a bug or resolves a feature request, be sure to link to that issue. Close issue syntax: `Fixes #<issue number>`, see [documentation](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) for more details.
 
-Fixes # (issue)
+Fixes 
 
 ## Type of Change
 
-Please delete options that are not relevant.
-
 - [ ] Bug fix (non-breaking change which fixes an issue)
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
 - [ ] This change requires a documentation update, included: [Dify Document](https://github.com/langgenius/dify-docs)
 - [ ] Improvement, including but not limited to code refactoring, performance optimization, and UI/UX improvement
 - [ ] Dependency upgrade
 
-# How Has This Been Tested?
+# Testing Instructions
 
 Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
 
-- [ ] TODO
+- [ ] Test A
+- [ ] Test B
+
 
-# Suggested Checklist:
 
-- [ ] I have performed a self-review of my own code
-- [ ] I have commented my code, particularly in hard-to-understand areas
-- [ ] My changes generate no new warnings
-- [ ] I ran `dev/reformat`(backend) and `cd web && npx lint-staged`(frontend) to appease the lint gods
-- [ ] `optional` I have made corresponding changes to the documentation 
-- [ ] `optional` I have added tests that prove my fix is effective or that my feature works
-- [ ] `optional` New and existing unit tests pass locally with my changes

api/README.md

@@ -11,6 +11,7 @@
 
    ```bash
    cd ../docker
+   cp .middleware.env.example .middleware.env
    docker compose -f docker-compose.middleware.yaml -p dify up -d
    cd ../api
    ```

api/app.py

@@ -1,6 +1,6 @@
 import os
 
-from configs.app_config import DifyConfig
+from configs import dify_config
 
 if not os.environ.get("DEBUG") or os.environ.get("DEBUG", "false").lower() != 'true':
     from gevent import monkey
@@ -81,7 +81,7 @@ def create_flask_app_with_configs() -> Flask:
     with configs loaded from .env file
     """
     dify_app = DifyApp(__name__)
-    dify_app.config.from_mapping(DifyConfig().model_dump())
+    dify_app.config.from_mapping(dify_config.model_dump())
 
     # populate configs into system environment variables
     for key, value in dify_app.config.items():

api/commands.py

@@ -8,6 +8,7 @@
 from flask import current_app
 from werkzeug.exceptions import NotFound
 
+from configs import dify_config
 from constants.languages import languages
 from core.rag.datasource.vdb.vector_factory import Vector
 from core.rag.datasource.vdb.vector_type import VectorType
@@ -112,7 +113,7 @@ def reset_encrypt_key_pair():
     After the reset, all LLM credentials will become invalid, requiring re-entry.
     Only support SELF_HOSTED mode.
     """
-    if current_app.config['EDITION'] != 'SELF_HOSTED':
+    if dify_config.EDITION != 'SELF_HOSTED':
         click.echo(click.style('Sorry, only support SELF_HOSTED mode.', fg='red'))
         return
 

api/configs/__init__.py

@@ -0,0 +1,4 @@
+
+from .app_config import DifyConfig
+
+dify_config = DifyConfig()

api/configs/feature/__init__.py

@@ -50,25 +50,25 @@ class EndpointConfig(BaseModel):
     CONSOLE_API_URL: str = Field(
         description='The backend URL prefix of the console API.'
                     'used to concatenate the login authorization callback or notion integration callback.',
-        default='https://cloud.dify.ai',
+        default='',
     )
 
     CONSOLE_WEB_URL: str = Field(
         description='The front-end URL prefix of the console web.'
                     'used to concatenate some front-end addresses and for CORS configuration use.',
-        default='https://cloud.dify.ai',
+        default='',
     )
 
     SERVICE_API_URL: str = Field(
         description='Service API Url prefix.'
                     'used to display Service API Base Url to the front-end.',
-        default='https://api.dify.ai',
+        default='',
     )
 
     APP_WEB_URL: str = Field(
         description='WebApp Url prefix.'
                     'used to display WebAPP API Base Url to the front-end.',
-        default='https://udify.app',
+        default='',
     )
 
 
@@ -82,7 +82,7 @@ class FileAccessConfig(BaseModel):
                     'Url is signed and has expiration time.',
         validation_alias=AliasChoices('FILES_URL', 'CONSOLE_API_URL'),
         alias_priority=1,
-        default='https://cloud.dify.ai',
+        default='',
     )
 
     FILES_ACCESS_TIMEOUT: int = Field(

api/configs/middleware/vdb/tencent_vector_config.py

@@ -42,3 +42,8 @@ class TencentVectorDBConfig(BaseModel):
         description='Tencent Vector replicas',
         default=2,
     )
+
+    TENCENT_VECTOR_DB_DATABASE: Optional[str] = Field(
+        description='Tencent Vector Database',
+        default=None,
+    )

api/controllers/console/datasets/datasets_document.py

@@ -964,10 +964,11 @@ class DocumentRenameApi(DocumentResource):
     @account_initialization_required
     @marshal_with(document_fields)
     def post(self, dataset_id, document_id):
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
+        if not current_user.is_dataset_editor:
             raise Forbidden()
-
+        dataset = DatasetService.get_dataset(dataset_id)
+        DatasetService.check_dataset_operator_permission(current_user, dataset)
         parser = reqparse.RequestParser()
         parser.add_argument('name', type=str, required=True, nullable=False, location='json')
         args = parser.parse_args()

api/core/app/app_config/entities.py

@@ -114,6 +114,10 @@ def value_of(cls, value: str) -> 'VariableEntity.Type':
     default: Optional[str] = None
     hint: Optional[str] = None
 
+    @property
+    def name(self) -> str:
+        return self.variable
+
 
 class ExternalDataVariableEntity(BaseModel):
     """

api/core/app/apps/base_app_generator.py

@@ -1,52 +1,56 @@
+from collections.abc import Mapping
+from typing import Any, Optional
+
 from core.app.app_config.entities import AppConfig, VariableEntity
 
 
 class BaseAppGenerator:
-    def _get_cleaned_inputs(self, user_inputs: dict, app_config: AppConfig):
-        if user_inputs is None:
-            user_inputs = {}
-
-        filtered_inputs = {}
-
+    def _get_cleaned_inputs(self, user_inputs: Optional[Mapping[str, Any]], app_config: AppConfig) -> Mapping[str, Any]:
+        user_inputs = user_inputs or {}
         # Filter input variables from form configuration, handle required fields, default values, and option values
         variables = app_config.variables
-        for variable_config in variables:
-            variable = variable_config.variable
-
-            if (variable not in user_inputs
-                    or user_inputs[variable] is None
-                    or (isinstance(user_inputs[variable], str) and user_inputs[variable] == '')):
-                if variable_config.required:
-                    raise ValueError(f"{variable} is required in input form")
-                else:
-                    filtered_inputs[variable] = variable_config.default if variable_config.default is not None else ""
-                    continue
-
-            value = user_inputs[variable]
-
-            if value is not None:
-                if variable_config.type != VariableEntity.Type.NUMBER and not isinstance(value, str):
-                    raise ValueError(f"{variable} in input form must be a string")
-                elif variable_config.type == VariableEntity.Type.NUMBER and isinstance(value, str):
-                    if '.' in value:
-                        value = float(value)
-                    else:
-                        value = int(value)
-
-            if variable_config.type == VariableEntity.Type.SELECT:
-                options = variable_config.options if variable_config.options is not None else []
-                if value not in options:
-                    raise ValueError(f"{variable} in input form must be one of the following: {options}")
-            elif variable_config.type in [VariableEntity.Type.TEXT_INPUT, VariableEntity.Type.PARAGRAPH]:
-                if variable_config.max_length is not None:
-                    max_length = variable_config.max_length
-                    if len(value) > max_length:
-                        raise ValueError(f'{variable} in input form must be less than {max_length} characters')
-
-            if value and isinstance(value, str):
-                filtered_inputs[variable] = value.replace('\x00', '')
-            else:
-                filtered_inputs[variable] = value if value is not None else None
-
+        filtered_inputs = {var.name: self._validate_input(inputs=user_inputs, var=var) for var in variables}
+        filtered_inputs = {k: self._sanitize_value(v) for k, v in filtered_inputs.items()}
         return filtered_inputs
 
+    def _validate_input(self, *, inputs: Mapping[str, Any], var: VariableEntity):
+        user_input_value = inputs.get(var.name)
+        if var.required and not user_input_value:
+            raise ValueError(f'{var.name} is required in input form')
+        if not var.required and not user_input_value:
+            # TODO: should we return None here if the default value is None?
+            return var.default or ''
+        if (
+            var.type
+            in (
+                VariableEntity.Type.TEXT_INPUT,
+                VariableEntity.Type.SELECT,
+                VariableEntity.Type.PARAGRAPH,
+            )
+            and user_input_value
+            and not isinstance(user_input_value, str)
+        ):
+            raise ValueError(f"(type '{var.type}') {var.name} in input form must be a string")
+        if var.type == VariableEntity.Type.NUMBER and isinstance(user_input_value, str):
+            # may raise ValueError if user_input_value is not a valid number
+            try:
+                if '.' in user_input_value:
+                    return float(user_input_value)
+                else:
+                    return int(user_input_value)
+            except ValueError:
+                raise ValueError(f"{var.name} in input form must be a valid number")
+        if var.type == VariableEntity.Type.SELECT:
+            options = var.options or []
+            if user_input_value not in options:
+                raise ValueError(f'{var.name} in input form must be one of the following: {options}')
+        elif var.type in (VariableEntity.Type.TEXT_INPUT, VariableEntity.Type.PARAGRAPH):
+            if var.max_length and user_input_value and len(user_input_value) > var.max_length:
+                raise ValueError(f'{var.name} in input form must be less than {var.max_length} characters')
+
+        return user_input_value
+
+    def _sanitize_value(self, value: Any) -> Any:
+        if isinstance(value, str):
+            return value.replace('\x00', '')
+        return value

api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-3-haiku-v1.yaml

@@ -5,6 +5,8 @@ model_type: llm
 features:
   - agent-thought
   - vision
+  - tool-call
+  - stream-tool-call
 model_properties:
   mode: chat
   context_size: 200000

api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-3-opus-v1.yaml

@@ -5,6 +5,8 @@ model_type: llm
 features:
   - agent-thought
   - vision
+  - tool-call
+  - stream-tool-call
 model_properties:
   mode: chat
   context_size: 200000

api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-3-sonnet-v1.5.yaml

@@ -5,6 +5,8 @@ model_type: llm
 features:
   - agent-thought
   - vision
+  - tool-call
+  - stream-tool-call
 model_properties:
   mode: chat
   context_size: 200000

api/core/model_runtime/model_providers/bedrock/llm/anthropic.claude-3-sonnet-v1.yaml

@@ -5,6 +5,8 @@ model_type: llm
 features:
   - agent-thought
   - vision
+  - tool-call
+  - stream-tool-call
 model_properties:
   mode: chat
   context_size: 200000