feat/enhance the multi-modal support (langgenius#8818)

api/.env.example

@@ -233,6 +233,8 @@ VIKINGDB_SOCKET_TIMEOUT=30
 UPLOAD_FILE_SIZE_LIMIT=15
 UPLOAD_FILE_BATCH_LIMIT=5
 UPLOAD_IMAGE_FILE_SIZE_LIMIT=10
+UPLOAD_VIDEO_FILE_SIZE_LIMIT=100
+UPLOAD_AUDIO_FILE_SIZE_LIMIT=50
 
 # Model Configuration
 MULTIMODAL_SEND_IMAGE_FORMAT=base64
@@ -310,6 +312,7 @@ INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=1000
 WORKFLOW_MAX_EXECUTION_STEPS=500
 WORKFLOW_MAX_EXECUTION_TIME=1200
 WORKFLOW_CALL_MAX_DEPTH=5
+MAX_VARIABLE_SIZE=204800
 
 # App configuration
 APP_MAX_EXECUTION_TIME=1200

api/.vscode/launch.json.example

@@ -1,8 +1,15 @@
 {
     "version": "0.2.0",
+    "compounds": [
+        {
+            "name": "Launch Flask and Celery",
+            "configurations": ["Python: Flask", "Python: Celery"]
+        }
+    ],
     "configurations": [
         {
             "name": "Python: Flask",
+            "consoleName": "Flask",
             "type": "debugpy",
             "request": "launch",
             "python": "${workspaceFolder}/.venv/bin/python",
@@ -17,12 +24,12 @@
             },
             "args": [
                 "run",
-                "--host=0.0.0.0",
                 "--port=5001"
             ]
         },
         {
             "name": "Python: Celery",
+            "consoleName": "Celery",
             "type": "debugpy",
             "request": "launch",
             "python": "${workspaceFolder}/.venv/bin/python",
@@ -45,10 +52,10 @@
                 "-c",
                 "1",
                 "--loglevel",
-                "info",
+                "DEBUG",
                 "-Q",
                 "dataset,generation,mail,ops_trace,app_deletion"
             ]
-        },
+        }
     ]
-}
+}

api/commands.py

@@ -19,7 +19,7 @@
 from libs.helper import email as email_validate
 from libs.password import hash_password, password_pattern, valid_password
 from libs.rsa import generate_key_pair
-from models.account import Tenant
+from models import Tenant
 from models.dataset import Dataset, DatasetCollectionBinding, DocumentSegment
 from models.dataset import Document as DatasetDocument
 from models.model import Account, App, AppAnnotationSetting, AppMode, Conversation, MessageAnnotation
@@ -426,14 +426,14 @@ def convert_to_agent_apps():
         # fetch first 1000 apps
         sql_query = """SELECT a.id AS id FROM apps a
             INNER JOIN app_model_configs am ON a.app_model_config_id=am.id
-            WHERE a.mode = 'chat' 
-            AND am.agent_mode is not null 
+            WHERE a.mode = 'chat'
+            AND am.agent_mode is not null
             AND (
-				am.agent_mode like '%"strategy": "function_call"%' 
+				am.agent_mode like '%"strategy": "function_call"%'
                 OR am.agent_mode  like '%"strategy": "react"%'
-			) 
+			)
             AND (
-				am.agent_mode like '{"enabled": true%' 
+				am.agent_mode like '{"enabled": true%'
                 OR am.agent_mode like '{"max_iteration": %'
 			) ORDER BY a.created_at DESC LIMIT 1000
         """

api/configs/feature/__init__.py

@@ -20,11 +20,11 @@ class SecurityConfig(BaseSettings):
     Security-related configurations for the application
     """
 
-    SECRET_KEY: Optional[str] = Field(
+    SECRET_KEY: str = Field(
         description="Secret key for secure session cookie signing."
         "Make sure you are changing this key for your deployment with a strong key."
         "Generate a strong key using `openssl rand -base64 42` or set via the `SECRET_KEY` environment variable.",
-        default=None,
+        default="",
     )
 
     RESET_PASSWORD_TOKEN_EXPIRY_HOURS: PositiveInt = Field(
@@ -186,6 +186,16 @@ class FileUploadConfig(BaseSettings):
         default=10,
     )
 
+    UPLOAD_VIDEO_FILE_SIZE_LIMIT: NonNegativeInt = Field(
+        description="video file size limit in Megabytes for uploading files",
+        default=100,
+    )
+
+    UPLOAD_AUDIO_FILE_SIZE_LIMIT: NonNegativeInt = Field(
+        description="audio file size limit in Megabytes for uploading files",
+        default=50,
+    )
+
     BATCH_UPLOAD_LIMIT: NonNegativeInt = Field(
         description="Maximum number of files allowed in a batch upload operation",
         default=20,
@@ -364,8 +374,8 @@ class WorkflowConfig(BaseSettings):
     )
 
     MAX_VARIABLE_SIZE: PositiveInt = Field(
-        description="Maximum size in bytes for a single variable in workflows. Default to 5KB.",
-        default=5 * 1024,
+        description="Maximum size in bytes for a single variable in workflows. Default to 200 KB.",
+        default=200 * 1024,
     )
 
 
@@ -493,6 +503,7 @@ class RagEtlConfig(BaseSettings):
     Configuration for RAG ETL processes
     """
 
+    # TODO: This config is not only for rag etl, it is also for file upload, we should move it to file upload config
     ETL_TYPE: str = Field(
         description="RAG ETL type ('dify' or 'Unstructured'), default to 'dify'",
         default="dify",
@@ -559,7 +570,7 @@ class IndexingConfig(BaseSettings):
 
 
 class ImageFormatConfig(BaseSettings):
-    MULTIMODAL_SEND_IMAGE_FORMAT: str = Field(
+    MULTIMODAL_SEND_IMAGE_FORMAT: Literal["base64", "url"] = Field(
         description="Format for sending images in multimodal contexts ('base64' or 'url'), default is base64",
         default="base64",
     )

api/constants/__init__.py

@@ -1,2 +1,22 @@
+from configs import dify_config
+
 HIDDEN_VALUE = "[__HIDDEN__]"
 UUID_NIL = "00000000-0000-0000-0000-000000000000"
+
+IMAGE_EXTENSIONS = ["jpg", "jpeg", "png", "webp", "gif", "svg"]
+IMAGE_EXTENSIONS.extend([ext.upper() for ext in IMAGE_EXTENSIONS])
+
+VIDEO_EXTENSIONS = ["mp4", "mov", "mpeg", "mpga"]
+VIDEO_EXTENSIONS.extend([ext.upper() for ext in VIDEO_EXTENSIONS])
+
+AUDIO_EXTENSIONS = ["mp3", "m4a", "wav", "webm", "amr"]
+AUDIO_EXTENSIONS.extend([ext.upper() for ext in AUDIO_EXTENSIONS])
+
+
+if dify_config.ETL_TYPE == "Unstructured":
+    DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "pdf", "html", "htm", "xlsx", "xls"]
+    DOCUMENT_EXTENSIONS.extend(("docx", "csv", "eml", "msg", "pptx", "ppt", "xml", "epub"))
+    DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])
+else:
+    DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "pdf", "html", "htm", "xlsx", "xls", "docx", "csv"]
+    DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])

api/contexts/__init__.py

@@ -1,7 +1,9 @@
 from contextvars import ContextVar
+from typing import TYPE_CHECKING
 
-from core.workflow.entities.variable_pool import VariablePool
+if TYPE_CHECKING:
+    from core.workflow.entities.variable_pool import VariablePool
 
 tenant_id: ContextVar[str] = ContextVar("tenant_id")
 
-workflow_variable_pool: ContextVar[VariablePool] = ContextVar("workflow_variable_pool")
+workflow_variable_pool: ContextVar["VariablePool"] = ContextVar("workflow_variable_pool")

api/controllers/console/app/conversation.py

@@ -22,7 +22,8 @@
 )
 from libs.helper import DatetimeString
 from libs.login import login_required
-from models.model import AppMode, Conversation, EndUser, Message, MessageAnnotation
+from models import Conversation, EndUser, Message, MessageAnnotation
+from models.model import AppMode
 
 
 class CompletionConversationApi(Resource):

api/controllers/console/app/site.py

@@ -12,7 +12,7 @@
 from extensions.ext_database import db
 from fields.app_fields import app_site_fields
 from libs.login import login_required
-from models.model import Site
+from models import Site
 
 
 def parse_app_site_args():

api/controllers/console/app/workflow.py

@@ -13,14 +13,14 @@
 from controllers.console.wraps import account_initialization_required
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom
-from core.app.segments import factory
-from core.errors.error import AppInvokeQuotaExceededError
+from factories import variable_factory
 from fields.workflow_fields import workflow_fields
 from fields.workflow_run_fields import workflow_run_node_execution_fields
 from libs import helper
 from libs.helper import TimestampField, uuid_value
 from libs.login import current_user, login_required
-from models.model import App, AppMode
+from models import App
+from models.model import AppMode
 from services.app_dsl_service import AppDslService
 from services.app_generate_service import AppGenerateService
 from services.errors.app import WorkflowHashNotEqualError
@@ -101,9 +101,13 @@ def post(self, app_model: App):
 
         try:
             environment_variables_list = args.get("environment_variables") or []
-            environment_variables = [factory.build_variable_from_mapping(obj) for obj in environment_variables_list]
+            environment_variables = [
+                variable_factory.build_variable_from_mapping(obj) for obj in environment_variables_list
+            ]
             conversation_variables_list = args.get("conversation_variables") or []
-            conversation_variables = [factory.build_variable_from_mapping(obj) for obj in conversation_variables_list]
+            conversation_variables = [
+                variable_factory.build_variable_from_mapping(obj) for obj in conversation_variables_list
+            ]
             workflow = workflow_service.sync_draft_workflow(
                 app_model=app_model,
                 graph=args["graph"],
@@ -273,17 +277,15 @@ def post(self, app_model: App):
         parser.add_argument("files", type=list, required=False, location="json")
         args = parser.parse_args()
 
-        try:
-            response = AppGenerateService.generate(
-                app_model=app_model, user=current_user, args=args, invoke_from=InvokeFrom.DEBUGGER, streaming=True
-            )
+        response = AppGenerateService.generate(
+            app_model=app_model,
+            user=current_user,
+            args=args,
+            invoke_from=InvokeFrom.DEBUGGER,
+            streaming=True,
+        )
 
-            return helper.compact_generate_response(response)
-        except (ValueError, AppInvokeQuotaExceededError) as e:
-            raise e
-        except Exception as e:
-            logging.exception("internal server error.")
-            raise InternalServerError()
+        return helper.compact_generate_response(response)
 
 
 class WorkflowTaskStopApi(Resource):

api/controllers/console/app/workflow_app_log.py

@@ -7,7 +7,8 @@
 from controllers.console.wraps import account_initialization_required
 from fields.workflow_app_log_fields import workflow_app_log_pagination_fields
 from libs.login import login_required
-from models.model import App, AppMode
+from models import App
+from models.model import AppMode
 from services.workflow_app_service import WorkflowAppService
 
 

api/controllers/console/app/workflow_run.py

@@ -13,7 +13,8 @@
 )
 from libs.helper import uuid_value
 from libs.login import login_required
-from models.model import App, AppMode
+from models import App
+from models.model import AppMode
 from services.workflow_run_service import WorkflowRunService
 
 

api/controllers/console/app/workflow_statistic.py

@@ -13,8 +13,8 @@
 from extensions.ext_database import db
 from libs.helper import DatetimeString
 from libs.login import login_required
+from models.enums import WorkflowRunTriggeredFrom
 from models.model import AppMode
-from models.workflow import WorkflowRunTriggeredFrom
 
 
 class WorkflowDailyRunsStatistic(Resource):

api/controllers/console/app/wraps.py

@@ -5,7 +5,8 @@
 from controllers.console.app.error import AppNotFoundError
 from extensions.ext_database import db
 from libs.login import current_user
-from models.model import App, AppMode
+from models import App
+from models.model import AppMode
 
 
 def get_app_model(view: Optional[Callable] = None, *, mode: Union[AppMode, list[AppMode]] = None):

api/controllers/console/auth/oauth.py

@@ -13,7 +13,8 @@
 from extensions.ext_database import db
 from libs.helper import extract_remote_ip
 from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo
-from models.account import Account, AccountStatus
+from models import Account
+from models.account import AccountStatus
 from services.account_service import AccountService, RegisterService, TenantService
 from services.errors.account import AccountNotFoundError
 from services.errors.workspace import WorkSpaceNotAllowedCreateError, WorkSpaceNotFoundError

api/controllers/console/datasets/data_source.py

@@ -15,8 +15,7 @@
 from extensions.ext_database import db
 from fields.data_source_fields import integrate_list_fields, integrate_notion_info_list_fields
 from libs.login import login_required
-from models.dataset import Document
-from models.source import DataSourceOauthBinding
+from models import DataSourceOauthBinding, Document
 from services.dataset_service import DatasetService, DocumentService
 from tasks.document_indexing_sync_task import document_indexing_sync_task
 

api/controllers/console/datasets/datasets.py

@@ -24,8 +24,8 @@
 from fields.dataset_fields import dataset_detail_fields, dataset_query_detail_fields
 from fields.document_fields import document_status_fields
 from libs.login import login_required
-from models.dataset import Dataset, DatasetPermissionEnum, Document, DocumentSegment
-from models.model import ApiToken, UploadFile
+from models import ApiToken, Dataset, Document, DocumentSegment, UploadFile
+from models.dataset import DatasetPermissionEnum
 from services.dataset_service import DatasetPermissionService, DatasetService, DocumentService
 
 

api/controllers/console/datasets/datasets_document.py

@@ -46,8 +46,7 @@
     document_with_segments_fields,
 )
 from libs.login import login_required
-from models.dataset import Dataset, DatasetProcessRule, Document, DocumentSegment
-from models.model import UploadFile
+from models import Dataset, DatasetProcessRule, Document, DocumentSegment, UploadFile
 from services.dataset_service import DatasetService, DocumentService
 from tasks.add_document_to_index_task import add_document_to_index_task
 from tasks.remove_document_from_index_task import remove_document_from_index_task

api/controllers/console/datasets/datasets_segments.py

@@ -24,7 +24,7 @@
 from extensions.ext_redis import redis_client
 from fields.segment_fields import segment_fields
 from libs.login import login_required
-from models.dataset import DocumentSegment
+from models import DocumentSegment
 from services.dataset_service import DatasetService, DocumentService, SegmentService
 from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task
 from tasks.disable_segment_from_index_task import disable_segment_from_index_task