From 23e5eeec00c5c085b8d64351db06cb9d4e92f4ee Mon Sep 17 00:00:00 2001 From: Xiao Ley Date: Thu, 18 Jul 2024 08:43:14 +0800 Subject: [PATCH] feat: added custom secure_ascii to the json_process tool (#6401) --- .../builtin/json_process/tools/delete.py | 23 ++++----- .../builtin/json_process/tools/delete.yaml | 12 +++++ .../builtin/json_process/tools/insert.py | 26 +++++----- .../builtin/json_process/tools/insert.yaml | 12 +++++ .../builtin/json_process/tools/parse.py | 15 +++--- .../builtin/json_process/tools/parse.yaml | 12 +++++ .../builtin/json_process/tools/replace.py | 47 ++++++++++--------- .../builtin/json_process/tools/replace.yaml | 12 +++++ 8 files changed, 105 insertions(+), 54 deletions(-) diff --git a/api/core/tools/provider/builtin/json_process/tools/delete.py b/api/core/tools/provider/builtin/json_process/tools/delete.py index b09e4948813f5b..1b49cfe2f300f8 100644 --- a/api/core/tools/provider/builtin/json_process/tools/delete.py +++ b/api/core/tools/provider/builtin/json_process/tools/delete.py @@ -19,28 +19,29 @@ def _invoke(self, content = tool_parameters.get('content', '') if not content: return self.create_text_message('Invalid parameter content') - + # Get query query = tool_parameters.get('query', '') if not query: return self.create_text_message('Invalid parameter query') - + + ensure_ascii = tool_parameters.get('ensure_ascii', True) try: - result = self._delete(content, query) + result = self._delete(content, query, ensure_ascii) return self.create_text_message(str(result)) except Exception as e: return self.create_text_message(f'Failed to delete JSON content: {str(e)}') - def _delete(self, origin_json: str, query: str) -> str: + def _delete(self, origin_json: str, query: str, ensure_ascii: bool) -> str: try: input_data = json.loads(origin_json) expr = parse('$.' + query.lstrip('$.')) # Ensure query path starts with $ - + matches = expr.find(input_data) - + if not matches: - return json.dumps(input_data, ensure_ascii=True) # No changes if no matches found - + return json.dumps(input_data, ensure_ascii=ensure_ascii) # No changes if no matches found + for match in matches: if isinstance(match.context.value, dict): # Delete key from dictionary @@ -53,7 +54,7 @@ def _delete(self, origin_json: str, query: str) -> str: parent = match.context.parent if parent: del parent.value[match.path.fields[-1]] - - return json.dumps(input_data, ensure_ascii=True) + + return json.dumps(input_data, ensure_ascii=ensure_ascii) except Exception as e: - raise Exception(f"Delete operation failed: {str(e)}") \ No newline at end of file + raise Exception(f"Delete operation failed: {str(e)}") diff --git a/api/core/tools/provider/builtin/json_process/tools/delete.yaml b/api/core/tools/provider/builtin/json_process/tools/delete.yaml index 4cfa90b861d48f..4d390e40d17232 100644 --- a/api/core/tools/provider/builtin/json_process/tools/delete.yaml +++ b/api/core/tools/provider/builtin/json_process/tools/delete.yaml @@ -38,3 +38,15 @@ parameters: pt_BR: JSONPath query to locate the element to delete llm_description: JSONPath query to locate the element to delete form: llm + - name: ensure_ascii + type: boolean + default: true + label: + en_US: Ensure ASCII + zh_Hans: 确保 ASCII + pt_BR: Ensure ASCII + human_description: + en_US: Ensure the JSON output is ASCII encoded + zh_Hans: 确保输出的 JSON 是 ASCII 编码 + pt_BR: Ensure the JSON output is ASCII encoded + form: form diff --git a/api/core/tools/provider/builtin/json_process/tools/insert.py b/api/core/tools/provider/builtin/json_process/tools/insert.py index aa5986e2b45860..27e34f1ff3fda9 100644 --- a/api/core/tools/provider/builtin/json_process/tools/insert.py +++ b/api/core/tools/provider/builtin/json_process/tools/insert.py @@ -19,31 +19,31 @@ def _invoke(self, content = tool_parameters.get('content', '') if not content: return self.create_text_message('Invalid parameter content') - + # get query query = tool_parameters.get('query', '') if not query: return self.create_text_message('Invalid parameter query') - + # get new value new_value = tool_parameters.get('new_value', '') if not new_value: return self.create_text_message('Invalid parameter new_value') - + # get insert position index = tool_parameters.get('index') - + # get create path create_path = tool_parameters.get('create_path', False) - + + ensure_ascii = tool_parameters.get('ensure_ascii', True) try: - result = self._insert(content, query, new_value, index, create_path) + result = self._insert(content, query, new_value, ensure_ascii, index, create_path) return self.create_text_message(str(result)) except Exception: return self.create_text_message('Failed to insert JSON content') - - def _insert(self, origin_json, query, new_value, index=None, create_path=False): + def _insert(self, origin_json, query, new_value, ensure_ascii: bool, index=None, create_path=False): try: input_data = json.loads(origin_json) expr = parse(query) @@ -51,9 +51,9 @@ def _insert(self, origin_json, query, new_value, index=None, create_path=False): new_value = json.loads(new_value) except json.JSONDecodeError: new_value = new_value - + matches = expr.find(input_data) - + if not matches and create_path: # create new path path_parts = query.strip('$').strip('.').split('.') @@ -91,7 +91,7 @@ def _insert(self, origin_json, query, new_value, index=None, create_path=False): else: # replace old value with new value match.full_path.update(input_data, new_value) - - return json.dumps(input_data, ensure_ascii=True) + + return json.dumps(input_data, ensure_ascii=ensure_ascii) except Exception as e: - return str(e) \ No newline at end of file + return str(e) diff --git a/api/core/tools/provider/builtin/json_process/tools/insert.yaml b/api/core/tools/provider/builtin/json_process/tools/insert.yaml index 66a6ff99291cdb..63e7816455cb68 100644 --- a/api/core/tools/provider/builtin/json_process/tools/insert.yaml +++ b/api/core/tools/provider/builtin/json_process/tools/insert.yaml @@ -75,3 +75,15 @@ parameters: zh_Hans: 否 pt_BR: "No" form: form + - name: ensure_ascii + type: boolean + default: true + label: + en_US: Ensure ASCII + zh_Hans: 确保 ASCII + pt_BR: Ensure ASCII + human_description: + en_US: Ensure the JSON output is ASCII encoded + zh_Hans: 确保输出的 JSON 是 ASCII 编码 + pt_BR: Ensure the JSON output is ASCII encoded + form: form diff --git a/api/core/tools/provider/builtin/json_process/tools/parse.py b/api/core/tools/provider/builtin/json_process/tools/parse.py index b246afc07e8b12..ecd39113ae5498 100644 --- a/api/core/tools/provider/builtin/json_process/tools/parse.py +++ b/api/core/tools/provider/builtin/json_process/tools/parse.py @@ -19,33 +19,34 @@ def _invoke(self, content = tool_parameters.get('content', '') if not content: return self.create_text_message('Invalid parameter content') - + # get json filter json_filter = tool_parameters.get('json_filter', '') if not json_filter: return self.create_text_message('Invalid parameter json_filter') + ensure_ascii = tool_parameters.get('ensure_ascii', True) try: - result = self._extract(content, json_filter) + result = self._extract(content, json_filter, ensure_ascii) return self.create_text_message(str(result)) except Exception: return self.create_text_message('Failed to extract JSON content') # Extract data from JSON content - def _extract(self, content: str, json_filter: str) -> str: + def _extract(self, content: str, json_filter: str, ensure_ascii: bool) -> str: try: input_data = json.loads(content) expr = parse(json_filter) result = [match.value for match in expr.find(input_data)] - + if len(result) == 1: result = result[0] - + if isinstance(result, dict | list): - return json.dumps(result, ensure_ascii=True) + return json.dumps(result, ensure_ascii=ensure_ascii) elif isinstance(result, str | int | float | bool) or result is None: return str(result) else: return repr(result) except Exception as e: - return str(e) \ No newline at end of file + return str(e) diff --git a/api/core/tools/provider/builtin/json_process/tools/parse.yaml b/api/core/tools/provider/builtin/json_process/tools/parse.yaml index b619dcde94171d..c35f4eac0775ad 100644 --- a/api/core/tools/provider/builtin/json_process/tools/parse.yaml +++ b/api/core/tools/provider/builtin/json_process/tools/parse.yaml @@ -38,3 +38,15 @@ parameters: pt_BR: JSON fields to be parsed llm_description: JSON fields to be parsed form: llm + - name: ensure_ascii + type: boolean + default: true + label: + en_US: Ensure ASCII + zh_Hans: 确保 ASCII + pt_BR: Ensure ASCII + human_description: + en_US: Ensure the JSON output is ASCII encoded + zh_Hans: 确保输出的 JSON 是 ASCII 编码 + pt_BR: Ensure the JSON output is ASCII encoded + form: form diff --git a/api/core/tools/provider/builtin/json_process/tools/replace.py b/api/core/tools/provider/builtin/json_process/tools/replace.py index 9f127b9d067319..be696bce0e0a2c 100644 --- a/api/core/tools/provider/builtin/json_process/tools/replace.py +++ b/api/core/tools/provider/builtin/json_process/tools/replace.py @@ -19,61 +19,62 @@ def _invoke(self, content = tool_parameters.get('content', '') if not content: return self.create_text_message('Invalid parameter content') - + # get query query = tool_parameters.get('query', '') if not query: return self.create_text_message('Invalid parameter query') - + # get replace value replace_value = tool_parameters.get('replace_value', '') if not replace_value: return self.create_text_message('Invalid parameter replace_value') - + # get replace model replace_model = tool_parameters.get('replace_model', '') if not replace_model: return self.create_text_message('Invalid parameter replace_model') + ensure_ascii = tool_parameters.get('ensure_ascii', True) try: if replace_model == 'pattern': # get replace pattern replace_pattern = tool_parameters.get('replace_pattern', '') if not replace_pattern: return self.create_text_message('Invalid parameter replace_pattern') - result = self._replace_pattern(content, query, replace_pattern, replace_value) + result = self._replace_pattern(content, query, replace_pattern, replace_value, ensure_ascii) elif replace_model == 'key': - result = self._replace_key(content, query, replace_value) + result = self._replace_key(content, query, replace_value, ensure_ascii) elif replace_model == 'value': - result = self._replace_value(content, query, replace_value) + result = self._replace_value(content, query, replace_value, ensure_ascii) return self.create_text_message(str(result)) except Exception: return self.create_text_message('Failed to replace JSON content') # Replace pattern - def _replace_pattern(self, content: str, query: str, replace_pattern: str, replace_value: str) -> str: + def _replace_pattern(self, content: str, query: str, replace_pattern: str, replace_value: str, ensure_ascii: bool) -> str: try: input_data = json.loads(content) expr = parse(query) - + matches = expr.find(input_data) - + for match in matches: new_value = match.value.replace(replace_pattern, replace_value) match.full_path.update(input_data, new_value) - - return json.dumps(input_data, ensure_ascii=True) + + return json.dumps(input_data, ensure_ascii=ensure_ascii) except Exception as e: return str(e) - + # Replace key - def _replace_key(self, content: str, query: str, replace_value: str) -> str: + def _replace_key(self, content: str, query: str, replace_value: str, ensure_ascii: bool) -> str: try: input_data = json.loads(content) expr = parse(query) - + matches = expr.find(input_data) - + for match in matches: parent = match.context.value if isinstance(parent, dict): @@ -86,21 +87,21 @@ def _replace_key(self, content: str, query: str, replace_value: str) -> str: if isinstance(item, dict) and old_key in item: value = item.pop(old_key) item[replace_value] = value - return json.dumps(input_data, ensure_ascii=True) + return json.dumps(input_data, ensure_ascii=ensure_ascii) except Exception as e: return str(e) - + # Replace value - def _replace_value(self, content: str, query: str, replace_value: str) -> str: + def _replace_value(self, content: str, query: str, replace_value: str, ensure_ascii: bool) -> str: try: input_data = json.loads(content) expr = parse(query) - + matches = expr.find(input_data) - + for match in matches: match.full_path.update(input_data, replace_value) - - return json.dumps(input_data, ensure_ascii=True) + + return json.dumps(input_data, ensure_ascii=ensure_ascii) except Exception as e: - return str(e) \ No newline at end of file + return str(e) diff --git a/api/core/tools/provider/builtin/json_process/tools/replace.yaml b/api/core/tools/provider/builtin/json_process/tools/replace.yaml index 556be5e8b2bf66..cf4b1dc63f45b8 100644 --- a/api/core/tools/provider/builtin/json_process/tools/replace.yaml +++ b/api/core/tools/provider/builtin/json_process/tools/replace.yaml @@ -93,3 +93,15 @@ parameters: zh_Hans: 字符串替换 pt_BR: replace string form: form + - name: ensure_ascii + type: boolean + default: true + label: + en_US: Ensure ASCII + zh_Hans: 确保 ASCII + pt_BR: Ensure ASCII + human_description: + en_US: Ensure the JSON output is ASCII encoded + zh_Hans: 确保输出的 JSON 是 ASCII 编码 + pt_BR: Ensure the JSON output is ASCII encoded + form: form