add form schemas.

.vscode/launch.json → api/.vscode/launch.json

@@ -10,7 +10,7 @@
             "request": "launch",
             "module": "flask",
             "env": {
-                "FLASK_APP": "api/app.py",
+                "FLASK_APP": "app.py",
                 "FLASK_DEBUG": "1",
                 "GEVENT_SUPPORT": "True"
             },

api/controllers/console/__init__.py

@@ -6,7 +6,7 @@
 api = ExternalApi(bp)
 
 # Import other controllers
-from . import setup, version, apikey, admin
+from . import extension, setup, version, apikey, admin
 
 # Import app controllers
 from .app import advanced_prompt_template, app, site, completion, model_config, statistic, conversation, message, generator, audio

api/controllers/console/extension.py

@@ -0,0 +1,23 @@
+from flask_restful import Resource, reqparse
+
+from controllers.console import api
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from libs.login import login_required
+from services.extension_service import ExtensionService
+
+
+class CodeBasedExtension(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('module', type=str, required=True, location='args')
+        args = parser.parse_args()
+
+        return ExtensionService.get_code_based_extensions(args['module'])
+
+
+api.add_resource(CodeBasedExtension, '/code-based-extensions')

api/core/__init__.py

@@ -0,0 +1 @@
+import core.moderation.base

api/core/helper/auto_register.py

@@ -0,0 +1,17 @@
+# Desc: Metaclass for auto-registering subclasses of a class.
+
+class AutoRegisterMeta(type):
+    def __init__(cls, name, bases, attrs):
+        super(AutoRegisterMeta, cls).__init__(name, bases, attrs)
+        if not hasattr(cls, 'subclasses'):
+            cls.subclasses = {}
+        else:
+            register_name = getattr(cls, 'register_name', name)
+            cls.subclasses[register_name] = cls
+
+class AutoRegisterBase(metaclass=AutoRegisterMeta):
+    @classmethod
+    def create_instance(cls, subclass_name, *args, **kwargs):
+        if subclass_name not in cls.subclasses:
+            raise ValueError(f"No register_name with name '{subclass_name}' found")
+        return cls.subclasses[subclass_name](*args, **kwargs)

api/core/helper/extensible.py

@@ -0,0 +1,34 @@
+import json
+import os
+import copy
+
+class Extensible:
+    __extensions = {}
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        cls.register()
+
+    @classmethod
+    def register(cls):
+        subclass_path = os.path.abspath(cls.__module__.replace(".", os.path.sep) + '.py')
+        subclass_dir_path = os.path.dirname(subclass_path)
+        parent_folder_name = os.path.basename(os.path.dirname(subclass_dir_path))
+
+        json_path = os.path.join(subclass_dir_path, 'schema.json')
+        json_data = {}
+        if os.path.exists(json_path):
+            with open(json_path, 'r') as f:
+                json_data = json.load(f)        
+
+        if parent_folder_name not in cls.__extensions:
+            cls.__extensions[parent_folder_name] = {
+                "module": parent_folder_name,
+                "data": []
+            }
+
+        cls.__extensions[parent_folder_name]["data"].append(json_data)
+
+    @classmethod
+    def get_extensions(cls) -> dict:
+        return copy.deepcopy(cls.__extensions)

api/core/moderation/__init__.py

@@ -0,0 +1,4 @@
+from core.moderation.openai.openai import OpenAIModeration
+from core.moderation.keywords.keywords import KeywordsModeration
+from core.moderation.api_based.api_based import ApiBasedModeration
+from core.moderation.cloud_service.cloud_service import CloudServiceModeration

api/core/moderation/api_based/api_based.py

@@ -0,0 +1,13 @@
+from core.moderation.base import BaseModeration
+
+
+class ApiBasedModeration(BaseModeration):
+    register_name = "api_based"
+
+    @classmethod
+    def validate_config(self, config: dict) -> None:
+        api_based_extension_id = config.get("api_based_extension_id")
+        if not api_based_extension_id:
+            raise ValueError("api_based_extension_id is required")
+
+        self._validate_inputs_and_outputs_config(config, False)

api/core/moderation/base.py

@@ -0,0 +1,46 @@
+from abc import abstractclassmethod
+from core.helper.auto_register import AutoRegisterBase
+
+
+class BaseModeration(AutoRegisterBase):
+
+    @abstractclassmethod
+    def validate_config(self, config: dict) -> None:
+        pass
+
+    @abstractclassmethod
+    def moderation_for_inputs(self, config: dict):
+        pass
+
+    @abstractclassmethod
+    def moderation_for_outputs(self, config: dict):
+        pass
+
+    @classmethod
+    def _validate_inputs_and_outputs_config(self, config: dict, is_preset_response_required: bool) -> None:
+        # inputs_configs
+        inputs_configs = config.get("inputs_configs")
+        if not isinstance(inputs_configs, dict):
+            raise ValueError("inputs_configs must be a dict")
+
+        # outputs_configs
+        outputs_configs = config.get("outputs_configs")
+        if not isinstance(outputs_configs, dict):
+            raise ValueError("outputs_configs must be a dict")
+
+        inputs_configs_enabled = inputs_configs.get("enabled")
+        outputs_configs_enabled = outputs_configs.get("enabled")
+        if not inputs_configs_enabled and not outputs_configs_enabled:
+            raise ValueError("At least one of inputs_configs or outputs_configs must be enabled")
+
+        # preset_response
+        if not is_preset_response_required:
+            return
+
+        if inputs_configs_enabled and not inputs_configs.get("preset_response"):
+            raise ValueError("inputs_configs.preset_response is required")
+
+        if outputs_configs_enabled and not outputs_configs.get("preset_response"):
+            raise ValueError("outputs_configs.preset_response is required")
+
+

api/core/moderation/cloud_service/cloud_service.py

@@ -0,0 +1,5 @@
+from core.moderation.base import BaseModeration
+from core.helper.extensible import Extensible
+
+class CloudServiceModeration(BaseModeration, Extensible):
+    register_name = "cloud_service"

api/core/moderation/cloud_service/schema.json

@@ -0,0 +1,51 @@
+{
+    "name": "cloud_service",
+    "label": {
+        "en-US": "Cloud Service",
+        "zh-Hans": "云服务"
+    },
+    "form_schema": [
+        {
+            "select": {
+                "label": {
+                    "en-US": "Cloud Provider",
+                    "zh-Hans": "云计算厂商"
+                },
+                "variable": "cloud_provider",
+                "required": true,
+                "options": [
+                    "腾讯云",
+                    "阿里云",
+                    "AWS"
+                ],
+                "default": "",
+                "placeholder": ""
+            }
+        },
+        {
+            "text-input": {
+                "label": {
+                    "en-US": "API Endpoint",
+                    "zh-Hans": "API Endpoint"
+                },
+                "variable": "api_endpoint",
+                "required": true,
+                "max_length": 100,
+                "default": "",
+                "placeholder": ""
+            }
+        },
+        {
+            "paragraph": {
+                "label": {
+                    "en-US": "API Key",
+                    "zh-Hans": "API Key"
+                },
+                "variable": "api_keys",
+                "required": true,
+                "default": "",
+                "placeholder": ""
+            }
+        }
+    ]
+}

api/core/moderation/keywords/keywords.py

@@ -0,0 +1,13 @@
+from core.moderation.base import BaseModeration
+
+class KeywordsModeration(BaseModeration):
+    register_name = "keywords"
+
+    @classmethod
+    def validate_config(self, config):
+        keywords = config.get("keywords")
+        if not keywords:
+            raise ValueError("keywords is required")
+
+        self._validate_inputs_and_outputs_config(config, True)
+

api/core/moderation/openai/openai.py

@@ -0,0 +1,8 @@
+from core.moderation.base import BaseModeration
+
+class OpenAIModeration(BaseModeration):
+    register_name = "openai"
+
+    @classmethod
+    def validate_config(self, config: dict):
+        self._validate_inputs_and_outputs_config(config, True)

api/services/app_model_config_service.py

@@ -7,6 +7,7 @@
 from core.model_providers.models.entity.model_params import ModelType, ModelMode
 from models.account import Account
 from services.dataset_service import DatasetService
+from core.moderation.base import BaseModeration
 
 
 SUPPORT_TOOLS = ["dataset", "google_search", "web_reader", "wikipedia", "current_datetime"]
@@ -153,33 +154,6 @@ def validate_configuration(tenant_id: str, account: Account, config: dict, mode:
         if not isinstance(config["more_like_this"]["enabled"], bool):
             raise ValueError("enabled in more_like_this must be of boolean type")
 
-        # sensitive_word_avoidance
-        if 'sensitive_word_avoidance' not in config or not config["sensitive_word_avoidance"]:
-            config["sensitive_word_avoidance"] = {
-                "enabled": False
-            }
-
-        if not isinstance(config["sensitive_word_avoidance"], dict):
-            raise ValueError("sensitive_word_avoidance must be of dict type")
-
-        if "enabled" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["enabled"]:
-            config["sensitive_word_avoidance"]["enabled"] = False
-
-        if not isinstance(config["sensitive_word_avoidance"]["enabled"], bool):
-            raise ValueError("enabled in sensitive_word_avoidance must be of boolean type")
-
-        if "words" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["words"]:
-            config["sensitive_word_avoidance"]["words"] = ""
-
-        if not isinstance(config["sensitive_word_avoidance"]["words"], str):
-            raise ValueError("words in sensitive_word_avoidance must be of string type")
-
-        if "canned_response" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["canned_response"]:
-            config["sensitive_word_avoidance"]["canned_response"] = ""
-
-        if not isinstance(config["sensitive_word_avoidance"]["canned_response"], str):
-            raise ValueError("canned_response in sensitive_word_avoidance must be of string type")
-
         # model
         if 'model' not in config:
             raise ValueError("model is required")
@@ -339,6 +313,9 @@ def validate_configuration(tenant_id: str, account: Account, config: dict, mode:
         # advanced prompt validation
         AppModelConfigService.is_advanced_prompt_valid(config, mode)
 
+        # moderation validation
+        AppModelConfigService.is_moderation_valid(config)
+
         # Filter out extra parameters
         filtered_config = {
             "opening_statement": config["opening_statement"],
@@ -365,6 +342,27 @@ def validate_configuration(tenant_id: str, account: Account, config: dict, mode:
         }
 
         return filtered_config
+
+    @staticmethod
+    def is_moderation_valid(config):
+        if 'sensitive_word_avoidance' not in config or not config["sensitive_word_avoidance"]:
+            config["sensitive_word_avoidance"] = {
+                "enabled": False
+            }
+
+        if not isinstance(config["sensitive_word_avoidance"], dict):
+            raise ValueError("sensitive_word_avoidance must be of dict type")
+
+        if "enabled" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["enabled"]:
+            config["sensitive_word_avoidance"]["enabled"] = False
+
+        if not config["sensitive_word_avoidance"]["enabled"]:
+            return
+
+        if "type" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["type"]:
+            raise ValueError("sensitive_word_avoidance.type is required")
+
+        BaseModeration.create_instance(type).validate_config(config["sensitive_word_avoidance"]["configs"])
 
     @staticmethod
     def is_dataset_query_variable_valid(config: dict, mode: str) -> None:

api/services/extension_service.py

@@ -0,0 +1,7 @@
+from core.helper.extensible import Extensible
+
+class ExtensionService:
+
+    @classmethod
+    def get_code_based_extensions(cls, module: str) -> list[dict]:
+        return Extensible.get_extensions().get(module, [])