From 30eb7c20690acdb43b86f3e9859137c448b0e609 Mon Sep 17 00:00:00 2001 From: Garfield Dai Date: Tue, 24 Oct 2023 17:35:21 +0800 Subject: [PATCH] add form schemas. --- {.vscode => api/.vscode}/launch.json | 2 +- api/controllers/console/__init__.py | 2 +- api/controllers/console/extension.py | 23 ++++++++ api/core/__init__.py | 1 + api/core/helper/auto_register.py | 17 ++++++ api/core/helper/extensible.py | 34 ++++++++++++ api/core/moderation/__init__.py | 4 ++ api/core/moderation/api_based/__init__.py | 0 api/core/moderation/api_based/api_based.py | 13 +++++ api/core/moderation/base.py | 46 ++++++++++++++++ api/core/moderation/cloud_service/__init__.py | 0 .../moderation/cloud_service/cloud_service.py | 5 ++ api/core/moderation/cloud_service/schema.json | 51 ++++++++++++++++++ api/core/moderation/keywords/__init__.py | 0 api/core/moderation/keywords/keywords.py | 13 +++++ api/core/moderation/openai/__init__.py | 0 api/core/moderation/openai/openai.py | 8 +++ api/services/app_model_config_service.py | 52 +++++++++---------- api/services/extension_service.py | 7 +++ 19 files changed, 249 insertions(+), 29 deletions(-) rename {.vscode => api/.vscode}/launch.json (94%) create mode 100644 api/controllers/console/extension.py create mode 100644 api/core/helper/auto_register.py create mode 100644 api/core/helper/extensible.py create mode 100644 api/core/moderation/__init__.py create mode 100644 api/core/moderation/api_based/__init__.py create mode 100644 api/core/moderation/api_based/api_based.py create mode 100644 api/core/moderation/base.py create mode 100644 api/core/moderation/cloud_service/__init__.py create mode 100644 api/core/moderation/cloud_service/cloud_service.py create mode 100644 api/core/moderation/cloud_service/schema.json create mode 100644 api/core/moderation/keywords/__init__.py create mode 100644 api/core/moderation/keywords/keywords.py create mode 100644 api/core/moderation/openai/__init__.py create mode 100644 api/core/moderation/openai/openai.py create mode 100644 api/services/extension_service.py diff --git a/.vscode/launch.json b/api/.vscode/launch.json similarity index 94% rename from .vscode/launch.json rename to api/.vscode/launch.json index 515deb4c0cf85b..e3c1f797c61601 100644 --- a/.vscode/launch.json +++ b/api/.vscode/launch.json @@ -10,7 +10,7 @@ "request": "launch", "module": "flask", "env": { - "FLASK_APP": "api/app.py", + "FLASK_APP": "app.py", "FLASK_DEBUG": "1", "GEVENT_SUPPORT": "True" }, diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.py index 2476d918870c6f..ac881dc126c0d0 100644 --- a/api/controllers/console/__init__.py +++ b/api/controllers/console/__init__.py @@ -6,7 +6,7 @@ api = ExternalApi(bp) # Import other controllers -from . import setup, version, apikey, admin +from . import extension, setup, version, apikey, admin # Import app controllers from .app import advanced_prompt_template, app, site, completion, model_config, statistic, conversation, message, generator, audio diff --git a/api/controllers/console/extension.py b/api/controllers/console/extension.py new file mode 100644 index 00000000000000..0b5ead214bf508 --- /dev/null +++ b/api/controllers/console/extension.py @@ -0,0 +1,23 @@ +from flask_restful import Resource, reqparse + +from controllers.console import api +from controllers.console.setup import setup_required +from controllers.console.wraps import account_initialization_required +from libs.login import login_required +from services.extension_service import ExtensionService + + +class CodeBasedExtension(Resource): + + @setup_required + @login_required + @account_initialization_required + def get(self): + parser = reqparse.RequestParser() + parser.add_argument('module', type=str, required=True, location='args') + args = parser.parse_args() + + return ExtensionService.get_code_based_extensions(args['module']) + + +api.add_resource(CodeBasedExtension, '/code-based-extensions') \ No newline at end of file diff --git a/api/core/__init__.py b/api/core/__init__.py index e69de29bb2d1d6..8c986fc8bd8afa 100644 --- a/api/core/__init__.py +++ b/api/core/__init__.py @@ -0,0 +1 @@ +import core.moderation.base \ No newline at end of file diff --git a/api/core/helper/auto_register.py b/api/core/helper/auto_register.py new file mode 100644 index 00000000000000..a908e8073655e3 --- /dev/null +++ b/api/core/helper/auto_register.py @@ -0,0 +1,17 @@ +# Desc: Metaclass for auto-registering subclasses of a class. + +class AutoRegisterMeta(type): + def __init__(cls, name, bases, attrs): + super(AutoRegisterMeta, cls).__init__(name, bases, attrs) + if not hasattr(cls, 'subclasses'): + cls.subclasses = {} + else: + register_name = getattr(cls, 'register_name', name) + cls.subclasses[register_name] = cls + +class AutoRegisterBase(metaclass=AutoRegisterMeta): + @classmethod + def create_instance(cls, subclass_name, *args, **kwargs): + if subclass_name not in cls.subclasses: + raise ValueError(f"No register_name with name '{subclass_name}' found") + return cls.subclasses[subclass_name](*args, **kwargs) diff --git a/api/core/helper/extensible.py b/api/core/helper/extensible.py new file mode 100644 index 00000000000000..7c3220cf0fe0b5 --- /dev/null +++ b/api/core/helper/extensible.py @@ -0,0 +1,34 @@ +import json +import os +import copy + +class Extensible: + __extensions = {} + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls.register() + + @classmethod + def register(cls): + subclass_path = os.path.abspath(cls.__module__.replace(".", os.path.sep) + '.py') + subclass_dir_path = os.path.dirname(subclass_path) + parent_folder_name = os.path.basename(os.path.dirname(subclass_dir_path)) + + json_path = os.path.join(subclass_dir_path, 'schema.json') + json_data = {} + if os.path.exists(json_path): + with open(json_path, 'r') as f: + json_data = json.load(f) + + if parent_folder_name not in cls.__extensions: + cls.__extensions[parent_folder_name] = { + "module": parent_folder_name, + "data": [] + } + + cls.__extensions[parent_folder_name]["data"].append(json_data) + + @classmethod + def get_extensions(cls) -> dict: + return copy.deepcopy(cls.__extensions) \ No newline at end of file diff --git a/api/core/moderation/__init__.py b/api/core/moderation/__init__.py new file mode 100644 index 00000000000000..727a8a1e99897c --- /dev/null +++ b/api/core/moderation/__init__.py @@ -0,0 +1,4 @@ +from core.moderation.openai.openai import OpenAIModeration +from core.moderation.keywords.keywords import KeywordsModeration +from core.moderation.api_based.api_based import ApiBasedModeration +from core.moderation.cloud_service.cloud_service import CloudServiceModeration \ No newline at end of file diff --git a/api/core/moderation/api_based/__init__.py b/api/core/moderation/api_based/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/core/moderation/api_based/api_based.py b/api/core/moderation/api_based/api_based.py new file mode 100644 index 00000000000000..c5cae1c85da7fa --- /dev/null +++ b/api/core/moderation/api_based/api_based.py @@ -0,0 +1,13 @@ +from core.moderation.base import BaseModeration + + +class ApiBasedModeration(BaseModeration): + register_name = "api_based" + + @classmethod + def validate_config(self, config: dict) -> None: + api_based_extension_id = config.get("api_based_extension_id") + if not api_based_extension_id: + raise ValueError("api_based_extension_id is required") + + self._validate_inputs_and_outputs_config(config, False) \ No newline at end of file diff --git a/api/core/moderation/base.py b/api/core/moderation/base.py new file mode 100644 index 00000000000000..ecf9d257fe5cb9 --- /dev/null +++ b/api/core/moderation/base.py @@ -0,0 +1,46 @@ +from abc import abstractclassmethod +from core.helper.auto_register import AutoRegisterBase + + +class BaseModeration(AutoRegisterBase): + + @abstractclassmethod + def validate_config(self, config: dict) -> None: + pass + + @abstractclassmethod + def moderation_for_inputs(self, config: dict): + pass + + @abstractclassmethod + def moderation_for_outputs(self, config: dict): + pass + + @classmethod + def _validate_inputs_and_outputs_config(self, config: dict, is_preset_response_required: bool) -> None: + # inputs_configs + inputs_configs = config.get("inputs_configs") + if not isinstance(inputs_configs, dict): + raise ValueError("inputs_configs must be a dict") + + # outputs_configs + outputs_configs = config.get("outputs_configs") + if not isinstance(outputs_configs, dict): + raise ValueError("outputs_configs must be a dict") + + inputs_configs_enabled = inputs_configs.get("enabled") + outputs_configs_enabled = outputs_configs.get("enabled") + if not inputs_configs_enabled and not outputs_configs_enabled: + raise ValueError("At least one of inputs_configs or outputs_configs must be enabled") + + # preset_response + if not is_preset_response_required: + return + + if inputs_configs_enabled and not inputs_configs.get("preset_response"): + raise ValueError("inputs_configs.preset_response is required") + + if outputs_configs_enabled and not outputs_configs.get("preset_response"): + raise ValueError("outputs_configs.preset_response is required") + + \ No newline at end of file diff --git a/api/core/moderation/cloud_service/__init__.py b/api/core/moderation/cloud_service/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/core/moderation/cloud_service/cloud_service.py b/api/core/moderation/cloud_service/cloud_service.py new file mode 100644 index 00000000000000..eaaa9c1eeca8a6 --- /dev/null +++ b/api/core/moderation/cloud_service/cloud_service.py @@ -0,0 +1,5 @@ +from core.moderation.base import BaseModeration +from core.helper.extensible import Extensible + +class CloudServiceModeration(BaseModeration, Extensible): + register_name = "cloud_service" \ No newline at end of file diff --git a/api/core/moderation/cloud_service/schema.json b/api/core/moderation/cloud_service/schema.json new file mode 100644 index 00000000000000..b9491de91dedc0 --- /dev/null +++ b/api/core/moderation/cloud_service/schema.json @@ -0,0 +1,51 @@ +{ + "name": "cloud_service", + "label": { + "en-US": "Cloud Service", + "zh-Hans": "云服务" + }, + "form_schema": [ + { + "select": { + "label": { + "en-US": "Cloud Provider", + "zh-Hans": "云计算厂商" + }, + "variable": "cloud_provider", + "required": true, + "options": [ + "腾讯云", + "阿里云", + "AWS" + ], + "default": "", + "placeholder": "" + } + }, + { + "text-input": { + "label": { + "en-US": "API Endpoint", + "zh-Hans": "API Endpoint" + }, + "variable": "api_endpoint", + "required": true, + "max_length": 100, + "default": "", + "placeholder": "" + } + }, + { + "paragraph": { + "label": { + "en-US": "API Key", + "zh-Hans": "API Key" + }, + "variable": "api_keys", + "required": true, + "default": "", + "placeholder": "" + } + } + ] +} \ No newline at end of file diff --git a/api/core/moderation/keywords/__init__.py b/api/core/moderation/keywords/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/core/moderation/keywords/keywords.py b/api/core/moderation/keywords/keywords.py new file mode 100644 index 00000000000000..4d58c1f5f62a86 --- /dev/null +++ b/api/core/moderation/keywords/keywords.py @@ -0,0 +1,13 @@ +from core.moderation.base import BaseModeration + +class KeywordsModeration(BaseModeration): + register_name = "keywords" + + @classmethod + def validate_config(self, config): + keywords = config.get("keywords") + if not keywords: + raise ValueError("keywords is required") + + self._validate_inputs_and_outputs_config(config, True) + \ No newline at end of file diff --git a/api/core/moderation/openai/__init__.py b/api/core/moderation/openai/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/core/moderation/openai/openai.py b/api/core/moderation/openai/openai.py new file mode 100644 index 00000000000000..b65748265da70d --- /dev/null +++ b/api/core/moderation/openai/openai.py @@ -0,0 +1,8 @@ +from core.moderation.base import BaseModeration + +class OpenAIModeration(BaseModeration): + register_name = "openai" + + @classmethod + def validate_config(self, config: dict): + self._validate_inputs_and_outputs_config(config, True) \ No newline at end of file diff --git a/api/services/app_model_config_service.py b/api/services/app_model_config_service.py index 79c1ed0ad6f663..a9c6db8d43746a 100644 --- a/api/services/app_model_config_service.py +++ b/api/services/app_model_config_service.py @@ -7,6 +7,7 @@ from core.model_providers.models.entity.model_params import ModelType, ModelMode from models.account import Account from services.dataset_service import DatasetService +from core.moderation.base import BaseModeration SUPPORT_TOOLS = ["dataset", "google_search", "web_reader", "wikipedia", "current_datetime"] @@ -153,33 +154,6 @@ def validate_configuration(tenant_id: str, account: Account, config: dict, mode: if not isinstance(config["more_like_this"]["enabled"], bool): raise ValueError("enabled in more_like_this must be of boolean type") - # sensitive_word_avoidance - if 'sensitive_word_avoidance' not in config or not config["sensitive_word_avoidance"]: - config["sensitive_word_avoidance"] = { - "enabled": False - } - - if not isinstance(config["sensitive_word_avoidance"], dict): - raise ValueError("sensitive_word_avoidance must be of dict type") - - if "enabled" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["enabled"]: - config["sensitive_word_avoidance"]["enabled"] = False - - if not isinstance(config["sensitive_word_avoidance"]["enabled"], bool): - raise ValueError("enabled in sensitive_word_avoidance must be of boolean type") - - if "words" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["words"]: - config["sensitive_word_avoidance"]["words"] = "" - - if not isinstance(config["sensitive_word_avoidance"]["words"], str): - raise ValueError("words in sensitive_word_avoidance must be of string type") - - if "canned_response" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["canned_response"]: - config["sensitive_word_avoidance"]["canned_response"] = "" - - if not isinstance(config["sensitive_word_avoidance"]["canned_response"], str): - raise ValueError("canned_response in sensitive_word_avoidance must be of string type") - # model if 'model' not in config: raise ValueError("model is required") @@ -339,6 +313,9 @@ def validate_configuration(tenant_id: str, account: Account, config: dict, mode: # advanced prompt validation AppModelConfigService.is_advanced_prompt_valid(config, mode) + # moderation validation + AppModelConfigService.is_moderation_valid(config) + # Filter out extra parameters filtered_config = { "opening_statement": config["opening_statement"], @@ -365,6 +342,27 @@ def validate_configuration(tenant_id: str, account: Account, config: dict, mode: } return filtered_config + + @staticmethod + def is_moderation_valid(config): + if 'sensitive_word_avoidance' not in config or not config["sensitive_word_avoidance"]: + config["sensitive_word_avoidance"] = { + "enabled": False + } + + if not isinstance(config["sensitive_word_avoidance"], dict): + raise ValueError("sensitive_word_avoidance must be of dict type") + + if "enabled" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["enabled"]: + config["sensitive_word_avoidance"]["enabled"] = False + + if not config["sensitive_word_avoidance"]["enabled"]: + return + + if "type" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["type"]: + raise ValueError("sensitive_word_avoidance.type is required") + + BaseModeration.create_instance(type).validate_config(config["sensitive_word_avoidance"]["configs"]) @staticmethod def is_dataset_query_variable_valid(config: dict, mode: str) -> None: diff --git a/api/services/extension_service.py b/api/services/extension_service.py new file mode 100644 index 00000000000000..08ecf50efc84a2 --- /dev/null +++ b/api/services/extension_service.py @@ -0,0 +1,7 @@ +from core.helper.extensible import Extensible + +class ExtensionService: + + @classmethod + def get_code_based_extensions(cls, module: str) -> list[dict]: + return Extensible.get_extensions().get(module, []) \ No newline at end of file