Merge branch 'feat/workflow-backend' into deploy/dev

# Conflicts:
#	api/controllers/console/app/app.py
#	api/controllers/console/app/workflow.py
#	api/core/agent/base_agent_runner.py
#	api/core/app/app_config/easy_ui_based_app/dataset/manager.py
#	api/core/app/apps/advanced_chat/app_generator.py
#	api/core/app/apps/advanced_chat/app_runner.py
#	api/core/app/apps/advanced_chat/generate_task_pipeline.py
#	api/core/app/apps/advanced_chat/workflow_event_trigger_callback.py
#	api/core/app/apps/agent_chat/app_config_manager.py
#	api/core/app/apps/agent_chat/app_generator.py
#	api/core/app/apps/base_app_queue_manager.py
#	api/core/app/apps/chat/app_generator.py
#	api/core/app/apps/completion/app_config_manager.py
#	api/core/app/apps/completion/app_generator.py
#	api/core/app/apps/completion/app_runner.py
#	api/core/app/apps/message_based_app_generator.py
#	api/core/app/apps/message_based_app_queue_manager.py
#	api/core/app/apps/workflow/app_generator.py
#	api/core/app/apps/workflow/app_queue_manager.py
#	api/core/app/apps/workflow/app_runner.py
#	api/core/app/apps/workflow/generate_task_pipeline.py
#	api/core/app/apps/workflow/workflow_event_trigger_callback.py
#	api/core/app/entities/queue_entities.py
#	api/core/workflow/callbacks/base_workflow_callback.py
#	api/core/workflow/entities/variable_pool.py
#	api/core/workflow/entities/workflow_entities.py
#	api/core/workflow/nodes/base_node.py
#	api/core/workflow/nodes/code/code_node.py
#	api/core/workflow/nodes/direct_answer/direct_answer_node.py
#	api/core/workflow/nodes/end/end_node.py
#	api/core/workflow/nodes/http_request/http_request_node.py
#	api/core/workflow/nodes/llm/llm_node.py
#	api/core/workflow/nodes/start/start_node.py
#	api/core/workflow/nodes/template_transform/template_transform_node.py
#	api/core/workflow/nodes/tool/tool_node.py
#	api/core/workflow/workflow_engine_manager.py
#	api/models/model.py
#	api/models/workflow.py
#	api/services/completion_service.py
#	api/services/workflow_service.py

.github/workflows/api-workflow-tests.yaml

@@ -0,0 +1,31 @@
+name: Run Pytest
+
+on:
+  pull_request:
+    branches:
+      - main
+      - deploy/dev
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    env:
+      MOCK_SWITCH: true
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+          cache-dependency-path: ./api/requirements.txt
+
+      - name: Install dependencies
+        run: pip install -r ./api/requirements.txt
+
+      - name: Run pytest
+        run: pytest api/tests/integration_tests/workflow

api/.env.example

@@ -132,3 +132,7 @@ SSRF_PROXY_HTTP_URL=
 SSRF_PROXY_HTTPS_URL=
 
 BATCH_UPLOAD_LIMIT=10
+
+# CODE EXECUTION CONFIGURATION
+CODE_EXECUTION_ENDPOINT=
+CODE_EXECUTINO_API_KEY=

api/config.py

@@ -27,6 +27,7 @@
     'CHECK_UPDATE_URL': 'https://updates.dify.ai',
     'DEPLOY_ENV': 'PRODUCTION',
     'SQLALCHEMY_POOL_SIZE': 30,
+    'SQLALCHEMY_MAX_OVERFLOW': 10,
     'SQLALCHEMY_POOL_RECYCLE': 3600,
     'SQLALCHEMY_ECHO': 'False',
     'SENTRY_TRACES_SAMPLE_RATE': 1.0,
@@ -59,7 +60,9 @@
     'CAN_REPLACE_LOGO': 'False',
     'ETL_TYPE': 'dify',
     'KEYWORD_STORE': 'jieba',
-    'BATCH_UPLOAD_LIMIT': 20
+    'BATCH_UPLOAD_LIMIT': 20,
+    'CODE_EXECUTION_ENDPOINT': '',
+    'CODE_EXECUTION_API_KEY': ''
 }
 
 
@@ -146,6 +149,7 @@ def __init__(self):
         self.SQLALCHEMY_DATABASE_URI = f"postgresql://{db_credentials['DB_USERNAME']}:{db_credentials['DB_PASSWORD']}@{db_credentials['DB_HOST']}:{db_credentials['DB_PORT']}/{db_credentials['DB_DATABASE']}{db_extras}"
         self.SQLALCHEMY_ENGINE_OPTIONS = {
             'pool_size': int(get_env('SQLALCHEMY_POOL_SIZE')),
+            'max_overflow': int(get_env('SQLALCHEMY_MAX_OVERFLOW')),
             'pool_recycle': int(get_env('SQLALCHEMY_POOL_RECYCLE'))
         }
 
@@ -293,6 +297,9 @@ def __init__(self):
 
         self.BATCH_UPLOAD_LIMIT = get_env('BATCH_UPLOAD_LIMIT')
 
+        self.CODE_EXECUTION_ENDPOINT = get_env('CODE_EXECUTION_ENDPOINT')
+        self.CODE_EXECUTION_API_KEY = get_env('CODE_EXECUTION_API_KEY')
+
         self.API_COMPRESSION_ENABLED = get_bool_env('API_COMPRESSION_ENABLED')
 
 

api/controllers/console/app/app.py

@@ -1,3 +1,5 @@
+import json
+
 from flask_login import current_user
 from flask_restful import Resource, inputs, marshal_with, reqparse
 from werkzeug.exceptions import Forbidden, BadRequest
@@ -6,13 +8,18 @@
 from controllers.console.app.wraps import get_app_model
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
+from core.agent.entities import AgentToolEntity
+from extensions.ext_database import db
 from fields.app_fields import (
     app_detail_fields,
     app_detail_fields_with_site,
     app_pagination_fields,
 )
 from libs.login import login_required
 from services.app_service import AppService
+from models.model import App, AppModelConfig, AppMode
+from core.tools.utils.configuration import ToolParameterConfigurationManager
+from core.tools.tool_manager import ToolManager
 
 
 ALLOW_CREATE_APP_MODES = ['chat', 'agent-chat', 'advanced-chat', 'workflow']
@@ -102,6 +109,40 @@ class AppApi(Resource):
     @marshal_with(app_detail_fields_with_site)
     def get(self, app_model):
         """Get app detail"""
+        # get original app model config
+        if app_model.mode == AppMode.AGENT_CHAT.value or app_model.is_agent:
+            model_config: AppModelConfig = app_model.app_model_config
+            agent_mode = model_config.agent_mode_dict
+            # decrypt agent tool parameters if it's secret-input
+            for tool in agent_mode.get('tools') or []:
+                agent_tool_entity = AgentToolEntity(**tool)
+                # get tool
+                tool_runtime = ToolManager.get_agent_tool_runtime(
+                    tenant_id=current_user.current_tenant_id,
+                    agent_tool=agent_tool_entity,
+                    agent_callback=None
+                )
+                manager = ToolParameterConfigurationManager(
+                    tenant_id=current_user.current_tenant_id,
+                    tool_runtime=tool_runtime,
+                    provider_name=agent_tool_entity.provider_id,
+                    provider_type=agent_tool_entity.provider_type,
+                )
+
+                # get decrypted parameters
+                if agent_tool_entity.tool_parameters:
+                    parameters = manager.decrypt_tool_parameters(agent_tool_entity.tool_parameters or {})
+                    masked_parameter = manager.mask_tool_parameters(parameters or {})
+                else:
+                    masked_parameter = {}
+
+                # override tool parameters
+                tool['tool_parameters'] = masked_parameter
+
+            # override agent mode
+            model_config.agent_mode = json.dumps(agent_mode)
+            db.session.commit()
+
         return app_model
 
     @setup_required

api/controllers/console/app/model_config.py

@@ -1,3 +1,4 @@
+import json
 
 from flask import request
 from flask_login import current_user
@@ -7,6 +8,9 @@
 from controllers.console.app.wraps import get_app_model
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
+from core.agent.entities import AgentToolEntity
+from core.tools.tool_manager import ToolManager
+from core.tools.utils.configuration import ToolParameterConfigurationManager
 from events.app_event import app_model_config_was_updated
 from extensions.ext_database import db
 from libs.login import login_required
@@ -34,6 +38,83 @@ def post(self, app_model):
         )
         new_app_model_config = new_app_model_config.from_model_config_dict(model_configuration)
 
+        if app_model.mode == AppMode.AGENT_CHAT.value or app_model.is_agent:
+            # get original app model config
+            original_app_model_config: AppModelConfig = db.session.query(AppModelConfig).filter(
+                AppModelConfig.id == app_model.app_model_config_id
+            ).first()
+            agent_mode = original_app_model_config.agent_mode_dict
+            # decrypt agent tool parameters if it's secret-input
+            parameter_map = {}
+            masked_parameter_map = {}
+            tool_map = {}
+            for tool in agent_mode.get('tools') or []:
+                agent_tool_entity = AgentToolEntity(**tool)
+                # get tool
+                tool_runtime = ToolManager.get_agent_tool_runtime(
+                    tenant_id=current_user.current_tenant_id,
+                    agent_tool=agent_tool_entity,
+                    agent_callback=None
+                )
+                manager = ToolParameterConfigurationManager(
+                    tenant_id=current_user.current_tenant_id,
+                    tool_runtime=tool_runtime,
+                    provider_name=agent_tool_entity.provider_id,
+                    provider_type=agent_tool_entity.provider_type,
+                )
+
+                # get decrypted parameters
+                if agent_tool_entity.tool_parameters:
+                    parameters = manager.decrypt_tool_parameters(agent_tool_entity.tool_parameters or {})
+                    masked_parameter = manager.mask_tool_parameters(parameters or {})
+                else:
+                    parameters = {}
+                    masked_parameter = {}
+
+                key = f'{agent_tool_entity.provider_id}.{agent_tool_entity.provider_type}.{agent_tool_entity.tool_name}'
+                masked_parameter_map[key] = masked_parameter
+                parameter_map[key] = parameters
+                tool_map[key] = tool_runtime
+
+            # encrypt agent tool parameters if it's secret-input
+            agent_mode = new_app_model_config.agent_mode_dict
+            for tool in agent_mode.get('tools') or []:
+                agent_tool_entity = AgentToolEntity(**tool)
+
+                # get tool
+                key = f'{agent_tool_entity.provider_id}.{agent_tool_entity.provider_type}.{agent_tool_entity.tool_name}'
+                if key in tool_map:
+                    tool_runtime = tool_map[key]
+                else:
+                    tool_runtime = ToolManager.get_agent_tool_runtime(
+                        tenant_id=current_user.current_tenant_id,
+                        agent_tool=agent_tool_entity,
+                        agent_callback=None
+                    )
+
+                manager = ToolParameterConfigurationManager(
+                    tenant_id=current_user.current_tenant_id,
+                    tool_runtime=tool_runtime,
+                    provider_name=agent_tool_entity.provider_id,
+                    provider_type=agent_tool_entity.provider_type,
+                )
+                manager.delete_tool_parameters_cache()
+
+                # override parameters if it equals to masked parameters
+                if agent_tool_entity.tool_parameters:
+                    if key not in masked_parameter_map:
+                        continue
+
+                    if agent_tool_entity.tool_parameters == masked_parameter_map[key]:
+                        agent_tool_entity.tool_parameters = parameter_map[key]
+
+                # encrypt parameters
+                if agent_tool_entity.tool_parameters:
+                    tool['tool_parameters'] = manager.encrypt_tool_parameters(agent_tool_entity.tool_parameters or {})
+
+            # update app model config
+            new_app_model_config.agent_mode = json.dumps(agent_mode)
+
         db.session.add(new_app_model_config)
         db.session.flush()
 

api/controllers/console/app/workflow.py

@@ -147,9 +147,12 @@ def post(self, app_model: App, task_id: str):
         """
         Stop workflow task
         """
-        # TODO
         workflow_service = WorkflowService()
-        workflow_service.stop_workflow_task(app_model=app_model, task_id=task_id, account=current_user)
+        workflow_service.stop_workflow_task(
+            task_id=task_id,
+            user=current_user,
+            invoke_from=InvokeFrom.DEBUGGER
+        )
 
         return {
             "result": "success"