From 0de224b15345f0cdc46e75338bf5dcce15f266fe Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Tue, 16 Jul 2024 19:09:04 +0800 Subject: [PATCH 01/62] fix wrong using of RetrievalMethod Enum (#6345) --- api/controllers/console/datasets/datasets.py | 16 ++++++++-------- api/core/rag/datasource/retrieval_service.py | 8 ++++---- api/core/rag/retrieval/dataset_retrieval.py | 4 ++-- api/core/rag/retrieval/retrival_methods.py | 6 +++--- .../dataset_multi_retriever_tool.py | 2 +- .../dataset_retriever/dataset_retriever_tool.py | 2 +- .../knowledge_retrieval_node.py | 2 +- api/models/dataset.py | 2 +- api/services/dataset_service.py | 4 ++-- api/services/hit_testing_service.py | 2 +- 10 files changed, 24 insertions(+), 24 deletions(-) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 70c506bb0e8383..9166372df50517 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -545,15 +545,15 @@ def get(self): case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE: return { 'retrieval_method': [ - RetrievalMethod.SEMANTIC_SEARCH + RetrievalMethod.SEMANTIC_SEARCH.value ] } case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH | VectorType.ANALYTICDB | VectorType.MYSCALE: return { 'retrieval_method': [ - RetrievalMethod.SEMANTIC_SEARCH, - RetrievalMethod.FULL_TEXT_SEARCH, - RetrievalMethod.HYBRID_SEARCH, + RetrievalMethod.SEMANTIC_SEARCH.value, + RetrievalMethod.FULL_TEXT_SEARCH.value, + RetrievalMethod.HYBRID_SEARCH.value, ] } case _: @@ -569,15 +569,15 @@ def get(self, vector_type): case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE: return { 'retrieval_method': [ - RetrievalMethod.SEMANTIC_SEARCH + RetrievalMethod.SEMANTIC_SEARCH.value ] } case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH| VectorType.ANALYTICDB | VectorType.MYSCALE: return { 'retrieval_method': [ - RetrievalMethod.SEMANTIC_SEARCH, - RetrievalMethod.FULL_TEXT_SEARCH, - RetrievalMethod.HYBRID_SEARCH, + RetrievalMethod.SEMANTIC_SEARCH.value, + RetrievalMethod.FULL_TEXT_SEARCH.value, + RetrievalMethod.HYBRID_SEARCH.value, ] } case _: diff --git a/api/core/rag/datasource/retrieval_service.py b/api/core/rag/datasource/retrieval_service.py index 623b7a31230cba..8814c614332c69 100644 --- a/api/core/rag/datasource/retrieval_service.py +++ b/api/core/rag/datasource/retrieval_service.py @@ -11,7 +11,7 @@ from models.dataset import Dataset default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', @@ -86,7 +86,7 @@ def retrieve(cls, retrival_method: str, dataset_id: str, query: str, exception_message = ';\n'.join(exceptions) raise Exception(exception_message) - if retrival_method == RetrievalMethod.HYBRID_SEARCH: + if retrival_method == RetrievalMethod.HYBRID_SEARCH.value: data_post_processor = DataPostProcessor(str(dataset.tenant_id), reranking_model, False) all_documents = data_post_processor.invoke( query=query, @@ -142,7 +142,7 @@ def embedding_search(cls, flask_app: Flask, dataset_id: str, query: str, ) if documents: - if reranking_model and retrival_method == RetrievalMethod.SEMANTIC_SEARCH: + if reranking_model and retrival_method == RetrievalMethod.SEMANTIC_SEARCH.value: data_post_processor = DataPostProcessor(str(dataset.tenant_id), reranking_model, False) all_documents.extend(data_post_processor.invoke( query=query, @@ -174,7 +174,7 @@ def full_text_index_search(cls, flask_app: Flask, dataset_id: str, query: str, top_k=top_k ) if documents: - if reranking_model and retrival_method == RetrievalMethod.FULL_TEXT_SEARCH: + if reranking_model and retrival_method == RetrievalMethod.FULL_TEXT_SEARCH.value: data_post_processor = DataPostProcessor(str(dataset.tenant_id), reranking_model, False) all_documents.extend(data_post_processor.invoke( query=query, diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py index ea2a194a687a4e..c1f5e0820c6dc4 100644 --- a/api/core/rag/retrieval/dataset_retrieval.py +++ b/api/core/rag/retrieval/dataset_retrieval.py @@ -28,7 +28,7 @@ from models.dataset import Document as DatasetDocument default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', @@ -464,7 +464,7 @@ def to_dataset_retriever_tool(self, tenant_id: str, if retrieve_config.retrieve_strategy == DatasetRetrieveConfigEntity.RetrieveStrategy.SINGLE: # get retrieval model config default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', diff --git a/api/core/rag/retrieval/retrival_methods.py b/api/core/rag/retrieval/retrival_methods.py index 9b7907013d2dab..12aa28a51c1d98 100644 --- a/api/core/rag/retrieval/retrival_methods.py +++ b/api/core/rag/retrieval/retrival_methods.py @@ -1,15 +1,15 @@ from enum import Enum -class RetrievalMethod(str, Enum): +class RetrievalMethod(Enum): SEMANTIC_SEARCH = 'semantic_search' FULL_TEXT_SEARCH = 'full_text_search' HYBRID_SEARCH = 'hybrid_search' @staticmethod def is_support_semantic_search(retrieval_method: str) -> bool: - return retrieval_method in {RetrievalMethod.SEMANTIC_SEARCH, RetrievalMethod.HYBRID_SEARCH} + return retrieval_method in {RetrievalMethod.SEMANTIC_SEARCH.value, RetrievalMethod.HYBRID_SEARCH.value} @staticmethod def is_support_fulltext_search(retrieval_method: str) -> bool: - return retrieval_method in {RetrievalMethod.FULL_TEXT_SEARCH, RetrievalMethod.HYBRID_SEARCH} + return retrieval_method in {RetrievalMethod.FULL_TEXT_SEARCH.value, RetrievalMethod.HYBRID_SEARCH.value} diff --git a/api/core/tools/tool/dataset_retriever/dataset_multi_retriever_tool.py b/api/core/tools/tool/dataset_retriever/dataset_multi_retriever_tool.py index 5b053678f36e25..eaf58ed5bd616a 100644 --- a/api/core/tools/tool/dataset_retriever/dataset_multi_retriever_tool.py +++ b/api/core/tools/tool/dataset_retriever/dataset_multi_retriever_tool.py @@ -14,7 +14,7 @@ from models.dataset import Dataset, Document, DocumentSegment default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', diff --git a/api/core/tools/tool/dataset_retriever/dataset_retriever_tool.py b/api/core/tools/tool/dataset_retriever/dataset_retriever_tool.py index de2ce5858a6602..b1e541b8dba977 100644 --- a/api/core/tools/tool/dataset_retriever/dataset_retriever_tool.py +++ b/api/core/tools/tool/dataset_retriever/dataset_retriever_tool.py @@ -8,7 +8,7 @@ from models.dataset import Dataset, Document, DocumentSegment default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index 9e29bd9ea1f580..12fe4dfa84b1a6 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -22,7 +22,7 @@ from models.workflow import WorkflowNodeExecutionStatus default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', diff --git a/api/models/dataset.py b/api/models/dataset.py index 02d49380bdad01..af840d26d688c0 100644 --- a/api/models/dataset.py +++ b/api/models/dataset.py @@ -117,7 +117,7 @@ def doc_form(self): @property def retrieval_model_dict(self): default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 3d9f1851b7ad38..84049712d951e7 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -688,7 +688,7 @@ def save_document_with_dataset_id( dataset.collection_binding_id = dataset_collection_binding.id if not dataset.retrieval_model: default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', @@ -1059,7 +1059,7 @@ def save_document_without_dataset_id(tenant_id: str, document_data: dict, accoun retrieval_model = document_data['retrieval_model'] else: default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', diff --git a/api/services/hit_testing_service.py b/api/services/hit_testing_service.py index 9bcf8287127c2c..b83e1d8cb74489 100644 --- a/api/services/hit_testing_service.py +++ b/api/services/hit_testing_service.py @@ -9,7 +9,7 @@ from models.dataset import Dataset, DatasetQuery, DocumentSegment default_retrieval_model = { - 'search_method': RetrievalMethod.SEMANTIC_SEARCH, + 'search_method': RetrievalMethod.SEMANTIC_SEARCH.value, 'reranking_enable': False, 'reranking_model': { 'reranking_provider_name': '', From 06fcc0c6505d6a8dfb6cd5540c7f4015d2095efd Mon Sep 17 00:00:00 2001 From: "Charlie.Wei" Date: Tue, 16 Jul 2024 21:53:57 +0800 Subject: [PATCH 02/62] Fix tts api err (#6349) Co-authored-by: luowei Co-authored-by: crazywoola <427733928@qq.com> Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> --- api/controllers/console/explore/audio.py | 7 +++++-- api/controllers/service_api/app/audio.py | 10 ++++++---- api/controllers/web/audio.py | 7 +++++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/api/controllers/console/explore/audio.py b/api/controllers/console/explore/audio.py index 920b1d8383d52b..27cc83042a7822 100644 --- a/api/controllers/console/explore/audio.py +++ b/api/controllers/console/explore/audio.py @@ -78,10 +78,12 @@ def post(self, installed_app): parser = reqparse.RequestParser() parser.add_argument('message_id', type=str, required=False, location='json') parser.add_argument('voice', type=str, location='json') + parser.add_argument('text', type=str, location='json') parser.add_argument('streaming', type=bool, location='json') args = parser.parse_args() - message_id = args.get('message_id') + message_id = args.get('message_id', None) + text = args.get('text', None) if (app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value] and app_model.workflow and app_model.workflow.features_dict): @@ -95,7 +97,8 @@ def post(self, installed_app): response = AudioService.transcript_tts( app_model=app_model, message_id=message_id, - voice=voice + voice=voice, + text=text ) return response except services.errors.app_model_config.AppModelConfigBrokenError: diff --git a/api/controllers/service_api/app/audio.py b/api/controllers/service_api/app/audio.py index 607d71598f7eed..3c009af343582d 100644 --- a/api/controllers/service_api/app/audio.py +++ b/api/controllers/service_api/app/audio.py @@ -76,10 +76,12 @@ def post(self, app_model: App, end_user: EndUser): parser = reqparse.RequestParser() parser.add_argument('message_id', type=str, required=False, location='json') parser.add_argument('voice', type=str, location='json') + parser.add_argument('text', type=str, location='json') parser.add_argument('streaming', type=bool, location='json') args = parser.parse_args() - message_id = args.get('message_id') + message_id = args.get('message_id', None) + text = args.get('text', None) if (app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value] and app_model.workflow and app_model.workflow.features_dict): @@ -87,15 +89,15 @@ def post(self, app_model: App, end_user: EndUser): voice = args.get('voice') if args.get('voice') else text_to_speech.get('voice') else: try: - voice = args.get('voice') if args.get('voice') else app_model.app_model_config.text_to_speech_dict.get( - 'voice') + voice = args.get('voice') if args.get('voice') else app_model.app_model_config.text_to_speech_dict.get('voice') except Exception: voice = None response = AudioService.transcript_tts( app_model=app_model, message_id=message_id, end_user=end_user.external_user_id, - voice=voice + voice=voice, + text=text ) return response diff --git a/api/controllers/web/audio.py b/api/controllers/web/audio.py index 8be872f5f99a73..0e905f905a7a2f 100644 --- a/api/controllers/web/audio.py +++ b/api/controllers/web/audio.py @@ -74,10 +74,12 @@ def post(self, app_model: App, end_user): parser = reqparse.RequestParser() parser.add_argument('message_id', type=str, required=False, location='json') parser.add_argument('voice', type=str, location='json') + parser.add_argument('text', type=str, location='json') parser.add_argument('streaming', type=bool, location='json') args = parser.parse_args() - message_id = args.get('message_id') + message_id = args.get('message_id', None) + text = args.get('text', None) if (app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value] and app_model.workflow and app_model.workflow.features_dict): @@ -94,7 +96,8 @@ def post(self, app_model: App, end_user): app_model=app_model, message_id=message_id, end_user=end_user.external_user_id, - voice=voice + voice=voice, + text=text ) return response From 7c397f5722a4d4c04573a195cce3610aa01550f1 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Wed, 17 Jul 2024 02:31:30 +0800 Subject: [PATCH 03/62] update celery beat scheduler time to env (#6352) --- api/.env.example | 4 ++++ api/configs/feature/__init__.py | 9 +++++++++ api/extensions/ext_celery.py | 6 +++--- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/api/.env.example b/api/.env.example index 228218be0de57b..474798cef7b1e6 100644 --- a/api/.env.example +++ b/api/.env.example @@ -256,3 +256,7 @@ WORKFLOW_CALL_MAX_DEPTH=5 # App configuration APP_MAX_EXECUTION_TIME=1200 APP_MAX_ACTIVE_REQUESTS=0 + + +# Celery beat configuration +CELERY_BEAT_SCHEDULER_TIME=1 \ No newline at end of file diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index c000c3a0f259f8..c22a89b158f360 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -23,6 +23,7 @@ class SecurityConfig(BaseSettings): default=24, ) + class AppExecutionConfig(BaseSettings): """ App Execution configs @@ -435,6 +436,13 @@ class ImageFormatConfig(BaseSettings): ) +class CeleryBeatConfig(BaseSettings): + CELERY_BEAT_SCHEDULER_TIME: int = Field( + description='the time of the celery scheduler, default to 1 day', + default=1, + ) + + class FeatureConfig( # place the configs in alphabet order AppExecutionConfig, @@ -462,5 +470,6 @@ class FeatureConfig( # hosted services config HostedServiceConfig, + CeleryBeatConfig, ): pass diff --git a/api/extensions/ext_celery.py b/api/extensions/ext_celery.py index 8302f91a438d12..ae9a07534084e7 100644 --- a/api/extensions/ext_celery.py +++ b/api/extensions/ext_celery.py @@ -43,15 +43,15 @@ def __call__(self, *args: object, **kwargs: object) -> object: "schedule.clean_embedding_cache_task", "schedule.clean_unused_datasets_task", ] - + day = app.config["CELERY_BEAT_SCHEDULER_TIME"] beat_schedule = { 'clean_embedding_cache_task': { 'task': 'schedule.clean_embedding_cache_task.clean_embedding_cache_task', - 'schedule': timedelta(days=1), + 'schedule': timedelta(days=day), }, 'clean_unused_datasets_task': { 'task': 'schedule.clean_unused_datasets_task.clean_unused_datasets_task', - 'schedule': timedelta(minutes=3), + 'schedule': timedelta(days=day), } } celery_app.conf.update( From 7943f7f697bd788b8158e40cc0079bc12e64fc19 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Wed, 17 Jul 2024 13:54:35 +0800 Subject: [PATCH 04/62] chore: fix legacy API usages of Query.get() by Session.get() in SqlAlchemy 2 (#6340) --- .../console/datasets/datasets_segments.py | 2 +- api/controllers/console/workspace/members.py | 2 +- api/models/dataset.py | 6 ++--- api/models/workflow.py | 18 ++++++------- api/poetry.lock | 26 +++++++++---------- api/pyproject.toml | 2 +- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.py index a189aac3f13761..3dcade61528707 100644 --- a/api/controllers/console/datasets/datasets_segments.py +++ b/api/controllers/console/datasets/datasets_segments.py @@ -75,7 +75,7 @@ def get(self, dataset_id, document_id): ) if last_id is not None: - last_segment = DocumentSegment.query.get(str(last_id)) + last_segment = db.session.get(DocumentSegment, str(last_id)) if last_segment: query = query.filter( DocumentSegment.position > last_segment.position) diff --git a/api/controllers/console/workspace/members.py b/api/controllers/console/workspace/members.py index e8c88850a42aa7..0e756778ab0e85 100644 --- a/api/controllers/console/workspace/members.py +++ b/api/controllers/console/workspace/members.py @@ -117,7 +117,7 @@ def put(self, member_id): if not TenantAccountRole.is_valid_role(new_role): return {'code': 'invalid-role', 'message': 'Invalid role'}, 400 - member = Account.query.get(str(member_id)) + member = db.session.get(Account, str(member_id)) if not member: abort(404) diff --git a/api/models/dataset.py b/api/models/dataset.py index af840d26d688c0..b0e3702dd7d175 100644 --- a/api/models/dataset.py +++ b/api/models/dataset.py @@ -68,7 +68,7 @@ def index_struct_dict(self): @property def created_by_account(self): - return Account.query.get(self.created_by) + return db.session.get(Account, self.created_by) @property def latest_process_rule(self): @@ -336,7 +336,7 @@ def average_segment_length(self): @property def dataset_process_rule(self): if self.dataset_process_rule_id: - return DatasetProcessRule.query.get(self.dataset_process_rule_id) + return db.session.get(DatasetProcessRule, self.dataset_process_rule_id) return None @property @@ -560,7 +560,7 @@ class AppDatasetJoin(db.Model): @property def app(self): - return App.query.get(self.app_id) + return db.session.get(App, self.app_id) class DatasetQuery(db.Model): diff --git a/api/models/workflow.py b/api/models/workflow.py index 2d6491032b9665..16e9d88ca15897 100644 --- a/api/models/workflow.py +++ b/api/models/workflow.py @@ -115,11 +115,11 @@ class Workflow(db.Model): @property def created_by_account(self): - return Account.query.get(self.created_by) + return db.session.get(Account, self.created_by) @property def updated_by_account(self): - return Account.query.get(self.updated_by) if self.updated_by else None + return db.session.get(Account, self.updated_by) if self.updated_by else None @property def graph_dict(self): @@ -290,14 +290,14 @@ class WorkflowRun(db.Model): @property def created_by_account(self): created_by_role = CreatedByRole.value_of(self.created_by_role) - return Account.query.get(self.created_by) \ + return db.session.get(Account, self.created_by) \ if created_by_role == CreatedByRole.ACCOUNT else None @property def created_by_end_user(self): from models.model import EndUser created_by_role = CreatedByRole.value_of(self.created_by_role) - return EndUser.query.get(self.created_by) \ + return db.session.get(EndUser, self.created_by) \ if created_by_role == CreatedByRole.END_USER else None @property @@ -500,14 +500,14 @@ class WorkflowNodeExecution(db.Model): @property def created_by_account(self): created_by_role = CreatedByRole.value_of(self.created_by_role) - return Account.query.get(self.created_by) \ + return db.session.get(Account, self.created_by) \ if created_by_role == CreatedByRole.ACCOUNT else None @property def created_by_end_user(self): from models.model import EndUser created_by_role = CreatedByRole.value_of(self.created_by_role) - return EndUser.query.get(self.created_by) \ + return db.session.get(EndUser, self.created_by) \ if created_by_role == CreatedByRole.END_USER else None @property @@ -612,17 +612,17 @@ class WorkflowAppLog(db.Model): @property def workflow_run(self): - return WorkflowRun.query.get(self.workflow_run_id) + return db.session.get(WorkflowRun, self.workflow_run_id) @property def created_by_account(self): created_by_role = CreatedByRole.value_of(self.created_by_role) - return Account.query.get(self.created_by) \ + return db.session.get(Account, self.created_by) \ if created_by_role == CreatedByRole.ACCOUNT else None @property def created_by_end_user(self): from models.model import EndUser created_by_role = CreatedByRole.value_of(self.created_by_role) - return EndUser.query.get(self.created_by) \ + return db.session.get(EndUser, self.created_by) \ if created_by_role == CreatedByRole.END_USER else None diff --git a/api/poetry.lock b/api/poetry.lock index ca967c57cd39d6..1b0d41e72d49b0 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -2486,18 +2486,18 @@ docs = ["sphinx"] [[package]] name = "flask-sqlalchemy" -version = "3.0.5" +version = "3.1.1" description = "Add SQLAlchemy support to your Flask application." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "flask_sqlalchemy-3.0.5-py3-none-any.whl", hash = "sha256:cabb6600ddd819a9f859f36515bb1bd8e7dbf30206cc679d2b081dff9e383283"}, - {file = "flask_sqlalchemy-3.0.5.tar.gz", hash = "sha256:c5765e58ca145401b52106c0f46178569243c5da25556be2c231ecc60867c5b1"}, + {file = "flask_sqlalchemy-3.1.1-py3-none-any.whl", hash = "sha256:4ba4be7f419dc72f4efd8802d69974803c37259dd42f3913b0dcf75c9447e0a0"}, + {file = "flask_sqlalchemy-3.1.1.tar.gz", hash = "sha256:e4b68bb881802dda1a7d878b2fc84c06d1ee57fb40b874d3dc97dabfa36b8312"}, ] [package.dependencies] flask = ">=2.2.5" -sqlalchemy = ">=1.4.18" +sqlalchemy = ">=2.0.16" [[package]] name = "flatbuffers" @@ -2794,8 +2794,8 @@ files = [ [package.dependencies] cffi = {version = ">=1.12.2", markers = "platform_python_implementation == \"CPython\" and sys_platform == \"win32\""} greenlet = [ - {version = ">=3.0rc3", markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\""}, {version = ">=2.0.0", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.11\""}, + {version = ">=3.0rc3", markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\""}, ] "zope.event" = "*" "zope.interface" = "*" @@ -2899,12 +2899,12 @@ files = [ google-auth = ">=2.14.1,<3.0.dev0" googleapis-common-protos = ">=1.56.2,<2.0.dev0" grpcio = [ - {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, + {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, ] grpcio-status = [ - {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, + {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, ] proto-plus = ">=1.22.3,<2.0.0dev" protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" @@ -4221,8 +4221,8 @@ files = [ [package.dependencies] orjson = ">=3.9.14,<4.0.0" pydantic = [ - {version = ">=2.7.4,<3.0.0", markers = "python_full_version >= \"3.12.4\""}, {version = ">=1,<3", markers = "python_full_version < \"3.12.4\""}, + {version = ">=2.7.4,<3.0.0", markers = "python_full_version >= \"3.12.4\""}, ] requests = ">=2,<3" @@ -5633,9 +5633,9 @@ bottleneck = {version = ">=1.3.6", optional = true, markers = "extra == \"perfor numba = {version = ">=0.56.4", optional = true, markers = "extra == \"performance\""} numexpr = {version = ">=2.8.4", optional = true, markers = "extra == \"performance\""} numpy = [ - {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, {version = ">=1.22.4", markers = "python_version < \"3.11\""}, {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] odfpy = {version = ">=1.4.1", optional = true, markers = "extra == \"excel\""} openpyxl = {version = ">=3.1.0", optional = true, markers = "extra == \"excel\""} @@ -6194,8 +6194,8 @@ files = [ annotated-types = ">=0.4.0" pydantic-core = "2.20.1" typing-extensions = [ - {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, {version = ">=4.6.1", markers = "python_version < \"3.13\""}, + {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, ] [package.extras] @@ -6964,8 +6964,8 @@ grpcio = ">=1.41.0" grpcio-tools = ">=1.41.0" httpx = {version = ">=0.14.0", extras = ["http2"]} numpy = [ - {version = ">=1.26", markers = "python_version >= \"3.12\""}, {version = ">=1.21", markers = "python_version >= \"3.8\" and python_version < \"3.12\""}, + {version = ">=1.26", markers = "python_version >= \"3.12\""}, ] portalocker = ">=2.7.0,<3.0.0" pydantic = ">=1.10.8" @@ -9408,4 +9408,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "5c30434ef3021083e74389544da4176c49aae15f530f30647793e240823f3fef" +content-hash = "8d2a12543340f6f4fa6dcb27f93d8b3f5380e7a3e7eb5e399e76e6b8588b4611" diff --git a/api/pyproject.toml b/api/pyproject.toml index b5d66184bea064..7868ea4ab67b0a 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -121,7 +121,7 @@ flask-cors = "~4.0.0" flask-login = "~0.6.3" flask-migrate = "~4.0.5" flask-restful = "~0.3.10" -flask-sqlalchemy = "~3.0.5" +Flask-SQLAlchemy = "~3.1.1" gevent = "~23.9.1" gmpy2 = "~2.1.5" google-ai-generativelanguage = "0.6.1" From 4e2fba404d5570cd5f23b374b17f449afbc864a5 Mon Sep 17 00:00:00 2001 From: Weaxs <459312872@qq.com> Date: Wed, 17 Jul 2024 14:13:57 +0800 Subject: [PATCH 05/62] WebscraperTool bypass cloudflare site by cloudscraper (#6337) --- api/core/tools/utils/web_reader_tool.py | 44 ++++++++++++++----------- api/poetry.lock | 32 +++++++++++++++++- api/pyproject.toml | 1 + 3 files changed, 57 insertions(+), 20 deletions(-) diff --git a/api/core/tools/utils/web_reader_tool.py b/api/core/tools/utils/web_reader_tool.py index 1e7eb129a75916..e52082541aa562 100644 --- a/api/core/tools/utils/web_reader_tool.py +++ b/api/core/tools/utils/web_reader_tool.py @@ -10,6 +10,7 @@ from contextlib import contextmanager from urllib.parse import unquote +import cloudscraper import requests from bs4 import BeautifulSoup, CData, Comment, NavigableString from newspaper import Article @@ -46,29 +47,34 @@ def get_url(url: str, user_agent: str = None) -> str: supported_content_types = extract_processor.SUPPORT_URL_CONTENT_TYPES + ["text/html"] response = requests.head(url, headers=headers, allow_redirects=True, timeout=(5, 10)) - if response.status_code != 200: - return "URL returned status code {}.".format(response.status_code) + if response.status_code == 200: + # check content-type + content_type = response.headers.get('Content-Type') + if content_type: + main_content_type = response.headers.get('Content-Type').split(';')[0].strip() + else: + content_disposition = response.headers.get('Content-Disposition', '') + filename_match = re.search(r'filename="([^"]+)"', content_disposition) + if filename_match: + filename = unquote(filename_match.group(1)) + extension = re.search(r'\.(\w+)$', filename) + if extension: + main_content_type = mimetypes.guess_type(filename)[0] - # check content-type - content_type = response.headers.get('Content-Type') - if content_type: - main_content_type = response.headers.get('Content-Type').split(';')[0].strip() - else: - content_disposition = response.headers.get('Content-Disposition', '') - filename_match = re.search(r'filename="([^"]+)"', content_disposition) - if filename_match: - filename = unquote(filename_match.group(1)) - extension = re.search(r'\.(\w+)$', filename) - if extension: - main_content_type = mimetypes.guess_type(filename)[0] + if main_content_type not in supported_content_types: + return "Unsupported content-type [{}] of URL.".format(main_content_type) - if main_content_type not in supported_content_types: - return "Unsupported content-type [{}] of URL.".format(main_content_type) + if main_content_type in extract_processor.SUPPORT_URL_CONTENT_TYPES: + return ExtractProcessor.load_from_url(url, return_text=True) - if main_content_type in extract_processor.SUPPORT_URL_CONTENT_TYPES: - return ExtractProcessor.load_from_url(url, return_text=True) + response = requests.get(url, headers=headers, allow_redirects=True, timeout=(120, 300)) + elif response.status_code == 403: + scraper = cloudscraper.create_scraper() + response = scraper.get(url, headers=headers, allow_redirects=True, timeout=(120, 300)) + + if response.status_code != 200: + return "URL returned status code {}.".format(response.status_code) - response = requests.get(url, headers=headers, allow_redirects=True, timeout=(120, 300)) a = extract_using_readabilipy(response.text) if not a['plain_text'] or not a['plain_text'].strip(): diff --git a/api/poetry.lock b/api/poetry.lock index 1b0d41e72d49b0..4b90b63e9fa6b8 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1610,6 +1610,22 @@ lz4 = ["clickhouse-cityhash (>=1.0.2.1)", "lz4", "lz4 (<=3.0.1)"] numpy = ["numpy (>=1.12.0)", "pandas (>=0.24.0)"] zstd = ["clickhouse-cityhash (>=1.0.2.1)", "zstd"] +[[package]] +name = "cloudscraper" +version = "1.2.71" +description = "A Python module to bypass Cloudflare's anti-bot page." +optional = false +python-versions = "*" +files = [ + {file = "cloudscraper-1.2.71-py2.py3-none-any.whl", hash = "sha256:76f50ca529ed2279e220837befdec892626f9511708e200d48d5bb76ded679b0"}, + {file = "cloudscraper-1.2.71.tar.gz", hash = "sha256:429c6e8aa6916d5bad5c8a5eac50f3ea53c9ac22616f6cb21b18dcc71517d0d3"}, +] + +[package.dependencies] +pyparsing = ">=2.4.7" +requests = ">=2.9.2" +requests-toolbelt = ">=0.9.1" + [[package]] name = "cohere" version = "5.2.6" @@ -7304,6 +7320,20 @@ requests = ">=2.0.0" [package.extras] rsa = ["oauthlib[signedtoken] (>=3.0.0)"] +[[package]] +name = "requests-toolbelt" +version = "1.0.0" +description = "A utility belt for advanced users of python-requests" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"}, + {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, +] + +[package.dependencies] +requests = ">=2.0.1,<3.0.0" + [[package]] name = "resend" version = "0.7.2" @@ -9408,4 +9438,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "8d2a12543340f6f4fa6dcb27f93d8b3f5380e7a3e7eb5e399e76e6b8588b4611" +content-hash = "9b1821b6e5d6d44947cc011c2d635a366557582b4540b99e0ff53a3078a989e5" diff --git a/api/pyproject.toml b/api/pyproject.toml index 7868ea4ab67b0a..d37d4c21f072cb 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -193,6 +193,7 @@ twilio = "~9.0.4" vanna = { version = "0.5.5", extras = ["postgres", "mysql", "clickhouse", "duckdb"] } wikipedia = "1.4.0" yfinance = "~0.2.40" +cloudscraper = "1.2.71" ############################################################ # VDB dependencies required by vector store clients From 20f73cb756e33d17fb51e0736e5e1f8a30fa3c04 Mon Sep 17 00:00:00 2001 From: FamousMai <906631095@qq.com> Date: Wed, 17 Jul 2024 14:14:12 +0800 Subject: [PATCH 06/62] =?UTF-8?q?fix:=20default=20model=20set=20wrong?= =?UTF-8?q?=EF=BC=88#6327=EF=BC=89=20(#6332)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: maiyouming --- api/services/app_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/services/app_service.py b/api/services/app_service.py index 36efde7825b6df..069f766793e704 100644 --- a/api/services/app_service.py +++ b/api/services/app_service.py @@ -98,7 +98,7 @@ def create_app(self, tenant_id: str, args: dict, account: Account) -> App: model_instance = None if model_instance: - if model_instance.model == default_model_config['model']['name']: + if model_instance.model == default_model_config['model']['name'] and model_instance.provider == default_model_config['model']['provider']: default_model_dict = default_model_config['model'] else: llm_model = cast(LargeLanguageModel, model_instance.model_type_instance) From ca69e1a2f53b7156100ea20ec7341428567c7b80 Mon Sep 17 00:00:00 2001 From: chenxu9741 Date: Wed, 17 Jul 2024 14:41:29 +0800 Subject: [PATCH 07/62] Add multilingual support for TTS (Text-to-Speech) functionality. (#6369) --- web/app/components/base/audio-btn/audio.ts | 15 --------------- web/i18n/es-ES/app-debug.ts | 3 +++ web/i18n/fr-FR/app-debug.ts | 3 +++ web/i18n/hi-IN/app-debug.ts | 3 +++ web/i18n/ko-KR/app-debug.ts | 3 +++ web/i18n/pl-PL/app-debug.ts | 3 +++ web/i18n/pt-BR/app-debug.ts | 3 +++ web/i18n/ro-RO/app-debug.ts | 3 +++ web/i18n/uk-UA/app-debug.ts | 3 +++ web/i18n/vi-VN/app-debug.ts | 3 +++ 10 files changed, 27 insertions(+), 15 deletions(-) diff --git a/web/app/components/base/audio-btn/audio.ts b/web/app/components/base/audio-btn/audio.ts index 239dfe0342b198..638626bf8ac69e 100644 --- a/web/app/components/base/audio-btn/audio.ts +++ b/web/app/components/base/audio-btn/audio.ts @@ -60,21 +60,6 @@ export default class AudioPlayer { return this.sourceBuffer = this.mediaSource?.addSourceBuffer(contentType) - // this.sourceBuffer?.addEventListener('update', () => { - // if (this.cacheBuffers.length && !this.sourceBuffer?.updating) { - // const cacheBuffer = this.cacheBuffers.shift()! - // this.sourceBuffer?.appendBuffer(cacheBuffer) - // } - // // this.pauseAudio() - // }) - // - // this.sourceBuffer?.addEventListener('updateend', () => { - // if (this.cacheBuffers.length && !this.sourceBuffer?.updating) { - // const cacheBuffer = this.cacheBuffers.shift()! - // this.sourceBuffer?.appendBuffer(cacheBuffer) - // } - // // this.pauseAudio() - // }) }) } diff --git a/web/i18n/es-ES/app-debug.ts b/web/i18n/es-ES/app-debug.ts index ecc4805059b44f..9e309a7d62b616 100644 --- a/web/i18n/es-ES/app-debug.ts +++ b/web/i18n/es-ES/app-debug.ts @@ -317,6 +317,9 @@ const translation = { language: 'Idioma', resolutionTooltip: 'Soporte de idioma para voz a texto.', voice: 'Voz', + autoPlay: 'Auto-reproducción', + autoPlayEnabled: 'Abierto', + autoPlayDisabled: 'Cierre', }, }, openingStatement: { diff --git a/web/i18n/fr-FR/app-debug.ts b/web/i18n/fr-FR/app-debug.ts index 5e5e8f9bada900..14a4b9cb14bf88 100644 --- a/web/i18n/fr-FR/app-debug.ts +++ b/web/i18n/fr-FR/app-debug.ts @@ -322,6 +322,9 @@ const translation = { language: 'Langue', resolutionTooltip: 'Support de la langue pour la voix de synthèse de texte.', voice: 'Voix', + autoPlay: 'Lecture Automatique', + autoPlayEnabled: 'Allumer', + autoPlayDisabled: 'Fermeture', }, }, openingStatement: { diff --git a/web/i18n/hi-IN/app-debug.ts b/web/i18n/hi-IN/app-debug.ts index 3950cf02adaa5f..872a1b9fe89544 100644 --- a/web/i18n/hi-IN/app-debug.ts +++ b/web/i18n/hi-IN/app-debug.ts @@ -349,6 +349,9 @@ const translation = { language: 'भाषा', resolutionTooltip: 'टेक्स्ट-टू-स्पीच वॉयस सपोर्ट भाषा।', voice: 'वॉयस', + autoPlay: 'ऑटो प्ले', + autoPlayEnabled: 'चालू करणे', + autoPlayDisabled: 'सोडा', }, }, openingStatement: { diff --git a/web/i18n/ko-KR/app-debug.ts b/web/i18n/ko-KR/app-debug.ts index 65d77a75fde2e4..77eac2503d6c7e 100644 --- a/web/i18n/ko-KR/app-debug.ts +++ b/web/i18n/ko-KR/app-debug.ts @@ -316,6 +316,9 @@ const translation = { language: '언어', resolutionTooltip: '텍스트 읽기 음성 언어를 지원합니다.', voice: '음성', + autoPlay: '자동 재생', + autoPlayEnabled: '켜다', + autoPlayDisabled: '폐쇄', }, }, openingStatement: { diff --git a/web/i18n/pl-PL/app-debug.ts b/web/i18n/pl-PL/app-debug.ts index 9d51b9ee46857e..960209c04589f1 100644 --- a/web/i18n/pl-PL/app-debug.ts +++ b/web/i18n/pl-PL/app-debug.ts @@ -347,6 +347,9 @@ const translation = { language: 'Język', resolutionTooltip: 'Wsparcie językowe głosu tekstu na mowę.', voice: 'Głos', + autoPlay: 'Automatyczne odtwarzanie', + autoPlayEnabled: 'włączyć coś', + autoPlayDisabled: 'zamknięcie', }, }, openingStatement: { diff --git a/web/i18n/pt-BR/app-debug.ts b/web/i18n/pt-BR/app-debug.ts index 1fae07a22321fc..91730d44b38861 100644 --- a/web/i18n/pt-BR/app-debug.ts +++ b/web/i18n/pt-BR/app-debug.ts @@ -322,6 +322,9 @@ const translation = { language: 'línguas', resolutionTooltip: 'Texto para voz timbre suporta idiomas.', voice: 'voz', + autoPlay: 'Reprodução automática', + autoPlayEnabled: 'ligar', + autoPlayDisabled: 'fecho', }, }, openingStatement: { diff --git a/web/i18n/ro-RO/app-debug.ts b/web/i18n/ro-RO/app-debug.ts index adb747b5140528..b4e9442de8a684 100644 --- a/web/i18n/ro-RO/app-debug.ts +++ b/web/i18n/ro-RO/app-debug.ts @@ -322,6 +322,9 @@ const translation = { language: 'Limbă', resolutionTooltip: 'Suport pentru limba de voce text-to-speech.', voice: 'Voce', + autoPlay: 'Redare automata', + autoPlayEnabled: 'Deschis', + autoPlayDisabled: 'închidere', }, }, openingStatement: { diff --git a/web/i18n/uk-UA/app-debug.ts b/web/i18n/uk-UA/app-debug.ts index 78798e7d53c43b..c64444c871b8f9 100644 --- a/web/i18n/uk-UA/app-debug.ts +++ b/web/i18n/uk-UA/app-debug.ts @@ -316,6 +316,9 @@ const translation = { language: 'Мова', // Language resolutionTooltip: 'Мовна підтримка для синтезу мовлення.', // Text-to-speech voice support language。 voice: 'Голос', // Voice + autoPlay: 'Автоматичне відтворення', + autoPlayEnabled: 'ВІДЧИНЕНО', + autoPlayDisabled: 'закриття', }, }, openingStatement: { diff --git a/web/i18n/vi-VN/app-debug.ts b/web/i18n/vi-VN/app-debug.ts index 19fbbae1fd274e..ac8fe4b9f781ff 100644 --- a/web/i18n/vi-VN/app-debug.ts +++ b/web/i18n/vi-VN/app-debug.ts @@ -322,6 +322,9 @@ const translation = { language: 'Ngôn ngữ', resolutionTooltip: 'Giọng nói văn bản hỗ trợ ngôn ngữ。', voice: 'Giọng', + autoPlay: 'chạy tự động', + autoPlayEnabled: 'bật', + autoPlayDisabled: 'Khép kín', }, }, openingStatement: { From 4ed14765314d77447e3d93d045b0ffa32aeb287d Mon Sep 17 00:00:00 2001 From: Lion Date: Wed, 17 Jul 2024 15:52:51 +0800 Subject: [PATCH 08/62] fix: incorrect config key name (#6371) Co-authored-by: LionYuYu --- api/configs/feature/hosted_service/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/configs/feature/hosted_service/__init__.py b/api/configs/feature/hosted_service/__init__.py index 209d46bb760a72..88fe188587e7a1 100644 --- a/api/configs/feature/hosted_service/__init__.py +++ b/api/configs/feature/hosted_service/__init__.py @@ -79,7 +79,7 @@ class HostedAzureOpenAiConfig(BaseSettings): default=False, ) - HOSTED_OPENAI_API_KEY: Optional[str] = Field( + HOSTED_AZURE_OPENAI_API_KEY: Optional[str] = Field( description='', default=None, ) From 984658f5e9896d3496e88976585ec070b0b7afa1 Mon Sep 17 00:00:00 2001 From: zxhlyh Date: Wed, 17 Jul 2024 16:51:48 +0800 Subject: [PATCH 09/62] fix: workflow sync before export (#6380) --- .../hooks/use-workflow-interactions.ts | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/web/app/components/workflow/hooks/use-workflow-interactions.ts b/web/app/components/workflow/hooks/use-workflow-interactions.ts index 109771dc87d509..88299af712be80 100644 --- a/web/app/components/workflow/hooks/use-workflow-interactions.ts +++ b/web/app/components/workflow/hooks/use-workflow-interactions.ts @@ -1,4 +1,7 @@ -import { useCallback } from 'react' +import { + useCallback, + useState, +} from 'react' import { useTranslation } from 'react-i18next' import { useReactFlow } from 'reactflow' import { useWorkflowStore } from '../store' @@ -10,6 +13,7 @@ import { } from '../utils' import { useEdgesInteractions } from './use-edges-interactions' import { useNodesInteractions } from './use-nodes-interactions' +import { useNodesSyncDraft } from './use-nodes-sync-draft' import { useEventEmitterContextContext } from '@/context/event-emitter' import { fetchWorkflowDraft } from '@/service/workflow' import { exportAppConfig } from '@/service/apps' @@ -79,12 +83,21 @@ export const useWorkflowUpdate = () => { export const useDSL = () => { const { t } = useTranslation() const { notify } = useToastContext() + const [exporting, setExporting] = useState(false) + const { doSyncWorkflowDraft } = useNodesSyncDraft() + const appDetail = useAppStore(s => s.appDetail) const handleExportDSL = useCallback(async () => { if (!appDetail) return + + if (exporting) + return + try { + setExporting(true) + await doSyncWorkflowDraft() const { data } = await exportAppConfig(appDetail.id) const a = document.createElement('a') const file = new Blob([data], { type: 'application/yaml' }) @@ -95,7 +108,10 @@ export const useDSL = () => { catch (e) { notify({ type: 'error', message: t('app.exportFailed') }) } - }, [appDetail, notify, t]) + finally { + setExporting(false) + } + }, [appDetail, notify, t, doSyncWorkflowDraft, exporting]) return { handleExportDSL, From fc37887a21021ddcc70c21aa2124012b22ed3473 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Wed, 17 Jul 2024 16:52:14 +0800 Subject: [PATCH 10/62] refactor(api/core/workflow/nodes/http_request): Remove `mask_authorization_header` because its alwary true. (#6379) --- .../workflow/nodes/http_request/entities.py | 1 - .../nodes/http_request/http_executor.py | 21 +++++++++---------- .../nodes/http_request/http_request_node.py | 8 ++----- .../utils/variable_template_parser.py | 7 ++++++- .../workflow/nodes/test_http.py | 16 -------------- 5 files changed, 18 insertions(+), 35 deletions(-) diff --git a/api/core/workflow/nodes/http_request/entities.py b/api/core/workflow/nodes/http_request/entities.py index 65451452c84ebe..90d644e0e22d3c 100644 --- a/api/core/workflow/nodes/http_request/entities.py +++ b/api/core/workflow/nodes/http_request/entities.py @@ -58,4 +58,3 @@ class HttpRequestNodeData(BaseNodeData): params: str body: Optional[HttpRequestNodeBody] = None timeout: Optional[HttpRequestNodeTimeout] = None - mask_authorization_header: Optional[bool] = True diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index 3736c67fb74109..fc32ee7a0cc025 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -283,7 +283,7 @@ def invoke(self) -> HttpExecutorResponse: # validate response return self._validate_and_parse_response(response) - def to_raw_request(self, mask_authorization_header: Optional[bool] = True) -> str: + def to_raw_request(self) -> str: """ convert to raw request """ @@ -295,16 +295,15 @@ def to_raw_request(self, mask_authorization_header: Optional[bool] = True) -> st headers = self._assembling_headers() for k, v in headers.items(): - if mask_authorization_header: - # get authorization header - if self.authorization.type == 'api-key': - authorization_header = 'Authorization' - if self.authorization.config and self.authorization.config.header: - authorization_header = self.authorization.config.header - - if k.lower() == authorization_header.lower(): - raw_request += f'{k}: {"*" * len(v)}\n' - continue + # get authorization header + if self.authorization.type == 'api-key': + authorization_header = 'Authorization' + if self.authorization.config and self.authorization.config.header: + authorization_header = self.authorization.config.header + + if k.lower() == authorization_header.lower(): + raw_request += f'{k}: {"*" * len(v)}\n' + continue raw_request += f'{k}: {v}\n' diff --git a/api/core/workflow/nodes/http_request/http_request_node.py b/api/core/workflow/nodes/http_request/http_request_node.py index 24acf984f2869f..86b17c9b33fbeb 100644 --- a/api/core/workflow/nodes/http_request/http_request_node.py +++ b/api/core/workflow/nodes/http_request/http_request_node.py @@ -65,9 +65,7 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: process_data = {} if http_executor: process_data = { - 'request': http_executor.to_raw_request( - mask_authorization_header=node_data.mask_authorization_header - ), + 'request': http_executor.to_raw_request(), } return NodeRunResult( status=WorkflowNodeExecutionStatus.FAILED, @@ -86,9 +84,7 @@ def _run(self, variable_pool: VariablePool) -> NodeRunResult: 'files': files, }, process_data={ - 'request': http_executor.to_raw_request( - mask_authorization_header=node_data.mask_authorization_header, - ), + 'request': http_executor.to_raw_request(), }, ) diff --git a/api/core/workflow/utils/variable_template_parser.py b/api/core/workflow/utils/variable_template_parser.py index 925c31a6aa6dbf..076a370da25386 100644 --- a/api/core/workflow/utils/variable_template_parser.py +++ b/api/core/workflow/utils/variable_template_parser.py @@ -1,4 +1,6 @@ import re +from collections.abc import Mapping +from typing import Any from core.workflow.entities.variable_entities import VariableSelector @@ -77,7 +79,7 @@ def extract_variable_selectors(self) -> list[VariableSelector]: return variable_selectors - def format(self, inputs: dict, remove_template_variables: bool = True) -> str: + def format(self, inputs: Mapping[str, Any], remove_template_variables: bool = True) -> str: """ Formats the template string by replacing the template variables with their corresponding values. @@ -91,6 +93,9 @@ def format(self, inputs: dict, remove_template_variables: bool = True) -> str: def replacer(match): key = match.group(1) value = inputs.get(key, match.group(0)) # return original matched string if key not found + + if value is None: + value = '' # convert the value to string if isinstance(value, list | dict | bool | int | float): value = str(value) diff --git a/api/tests/integration_tests/workflow/nodes/test_http.py b/api/tests/integration_tests/workflow/nodes/test_http.py index eaed24e56cbe12..2ab2eb7ecf7036 100644 --- a/api/tests/integration_tests/workflow/nodes/test_http.py +++ b/api/tests/integration_tests/workflow/nodes/test_http.py @@ -43,7 +43,6 @@ def test_get(setup_http_mock): 'headers': 'X-Header:123', 'params': 'A:b', 'body': None, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -52,7 +51,6 @@ def test_get(setup_http_mock): data = result.process_data.get('request', '') assert '?A=b' in data - assert 'api-key: Basic ak-xxx' in data assert 'X-Header: 123' in data @@ -103,7 +101,6 @@ def test_custom_authorization_header(setup_http_mock): 'headers': 'X-Header:123', 'params': 'A:b', 'body': None, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -113,7 +110,6 @@ def test_custom_authorization_header(setup_http_mock): assert '?A=b' in data assert 'X-Header: 123' in data - assert 'X-Auth: Auth' in data @pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True) @@ -136,7 +132,6 @@ def test_template(setup_http_mock): 'headers': 'X-Header:123\nX-Header2:{{#a.b123.args2#}}', 'params': 'A:b\nTemplate:{{#a.b123.args2#}}', 'body': None, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -145,7 +140,6 @@ def test_template(setup_http_mock): assert '?A=b' in data assert 'Template=2' in data - assert 'api-key: Basic ak-xxx' in data assert 'X-Header: 123' in data assert 'X-Header2: 2' in data @@ -173,7 +167,6 @@ def test_json(setup_http_mock): 'type': 'json', 'data': '{"a": "{{#a.b123.args1#}}"}' }, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -181,7 +174,6 @@ def test_json(setup_http_mock): data = result.process_data.get('request', '') assert '{"a": "1"}' in data - assert 'api-key: Basic ak-xxx' in data assert 'X-Header: 123' in data @@ -207,7 +199,6 @@ def test_x_www_form_urlencoded(setup_http_mock): 'type': 'x-www-form-urlencoded', 'data': 'a:{{#a.b123.args1#}}\nb:{{#a.b123.args2#}}' }, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -215,7 +206,6 @@ def test_x_www_form_urlencoded(setup_http_mock): data = result.process_data.get('request', '') assert 'a=1&b=2' in data - assert 'api-key: Basic ak-xxx' in data assert 'X-Header: 123' in data @@ -241,7 +231,6 @@ def test_form_data(setup_http_mock): 'type': 'form-data', 'data': 'a:{{#a.b123.args1#}}\nb:{{#a.b123.args2#}}' }, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -252,7 +241,6 @@ def test_form_data(setup_http_mock): assert '1' in data assert 'form-data; name="b"' in data assert '2' in data - assert 'api-key: Basic ak-xxx' in data assert 'X-Header: 123' in data @@ -278,14 +266,12 @@ def test_none_data(setup_http_mock): 'type': 'none', 'data': '123123123' }, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) result = node.run(pool) data = result.process_data.get('request', '') - assert 'api-key: Basic ak-xxx' in data assert 'X-Header: 123' in data assert '123123123' not in data @@ -305,7 +291,6 @@ def test_mock_404(setup_http_mock): 'body': None, 'params': '', 'headers': 'X-Header:123', - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -334,7 +319,6 @@ def test_multi_colons_parse(setup_http_mock): 'type': 'form-data', 'data': 'Referer:http://example5.com\nRedirect:http://example6.com' }, - 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) From 1bc90b992be15b2da1c312d489496724013519b6 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Wed, 17 Jul 2024 17:36:11 +0800 Subject: [PATCH 11/62] Feat/optimize clean dataset logic (#6384) --- api/schedule/clean_unused_datasets_task.py | 85 +++++++++++++++------- 1 file changed, 57 insertions(+), 28 deletions(-) diff --git a/api/schedule/clean_unused_datasets_task.py b/api/schedule/clean_unused_datasets_task.py index 2033791acee6e6..9cbee6e81e4b8e 100644 --- a/api/schedule/clean_unused_datasets_task.py +++ b/api/schedule/clean_unused_datasets_task.py @@ -2,6 +2,7 @@ import time import click +from sqlalchemy import func from werkzeug.exceptions import NotFound import app @@ -20,10 +21,46 @@ def clean_unused_datasets_task(): page = 1 while True: try: - datasets = db.session.query(Dataset).filter(Dataset.created_at < thirty_days_ago) \ - .order_by(Dataset.created_at.desc()).paginate(page=page, per_page=50) + # Subquery for counting new documents + document_subquery_new = db.session.query( + Document.dataset_id, + func.count(Document.id).label('document_count') + ).filter( + Document.indexing_status == 'completed', + Document.enabled == True, + Document.archived == False, + Document.updated_at > thirty_days_ago + ).group_by(Document.dataset_id).subquery() + + # Subquery for counting old documents + document_subquery_old = db.session.query( + Document.dataset_id, + func.count(Document.id).label('document_count') + ).filter( + Document.indexing_status == 'completed', + Document.enabled == True, + Document.archived == False, + Document.updated_at < thirty_days_ago + ).group_by(Document.dataset_id).subquery() + + # Main query with join and filter + datasets = (db.session.query(Dataset) + .outerjoin( + document_subquery_new, Dataset.id == document_subquery_new.c.dataset_id + ).outerjoin( + document_subquery_old, Dataset.id == document_subquery_old.c.dataset_id + ).filter( + Dataset.created_at < thirty_days_ago, + func.coalesce(document_subquery_new.c.document_count, 0) == 0, + func.coalesce(document_subquery_old.c.document_count, 0) > 0 + ).order_by( + Dataset.created_at.desc() + ).paginate(page=page, per_page=50)) + except NotFound: break + if datasets.items is None or len(datasets.items) == 0: + break page += 1 for dataset in datasets: dataset_query = db.session.query(DatasetQuery).filter( @@ -31,31 +68,23 @@ def clean_unused_datasets_task(): DatasetQuery.dataset_id == dataset.id ).all() if not dataset_query or len(dataset_query) == 0: - documents = db.session.query(Document).filter( - Document.dataset_id == dataset.id, - Document.indexing_status == 'completed', - Document.enabled == True, - Document.archived == False, - Document.updated_at > thirty_days_ago - ).all() - if not documents or len(documents) == 0: - try: - # remove index - index_processor = IndexProcessorFactory(dataset.doc_form).init_index_processor() - index_processor.clean(dataset, None) - - # update document - update_params = { - Document.enabled: False - } - - Document.query.filter_by(dataset_id=dataset.id).update(update_params) - db.session.commit() - click.echo(click.style('Cleaned unused dataset {} from db success!'.format(dataset.id), - fg='green')) - except Exception as e: - click.echo( - click.style('clean dataset index error: {} {}'.format(e.__class__.__name__, str(e)), - fg='red')) + try: + # remove index + index_processor = IndexProcessorFactory(dataset.doc_form).init_index_processor() + index_processor.clean(dataset, None) + + # update document + update_params = { + Document.enabled: False + } + + Document.query.filter_by(dataset_id=dataset.id).update(update_params) + db.session.commit() + click.echo(click.style('Cleaned unused dataset {} from db success!'.format(dataset.id), + fg='green')) + except Exception as e: + click.echo( + click.style('clean dataset index error: {} {}'.format(e.__class__.__name__, str(e)), + fg='red')) end_at = time.perf_counter() click.echo(click.style('Cleaned unused dataset from db success latency: {}'.format(end_at - start_at), fg='green')) From f3f052ba36471077a3ea68e6c0f7db5104b92583 Mon Sep 17 00:00:00 2001 From: xielong Date: Wed, 17 Jul 2024 19:07:47 +0800 Subject: [PATCH 12/62] fix: rename model from ernie-4.0-8k-Latest to ernie-4.0-8k-latest (#6383) --- .../model_providers/wenxin/llm/ernie-4.0-8k-latest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-8k-latest.yaml b/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-8k-latest.yaml index 50c82564f1d0f3..d23ae0dc48c12f 100644 --- a/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-8k-latest.yaml +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-8k-latest.yaml @@ -1,4 +1,4 @@ -model: ernie-4.0-8k-Latest +model: ernie-4.0-8k-latest label: en_US: Ernie-4.0-8K-Latest model_type: llm From a6dbd26f7593cabbcbdea6b015128299dfaa50a1 Mon Sep 17 00:00:00 2001 From: chenxu9741 Date: Wed, 17 Jul 2024 19:44:16 +0800 Subject: [PATCH 13/62] Add the API documentation for streaming TTS (Text-to-Speech) (#6382) --- .../develop/template/template.en.mdx | 24 ++++++++++++----- .../develop/template/template.zh.mdx | 25 +++++++++++++----- .../template/template_advanced_chat.en.mdx | 21 ++++++++++++--- .../template/template_advanced_chat.zh.mdx | 26 ++++++++++++++----- .../develop/template/template_chat.en.mdx | 26 ++++++++++++++----- .../develop/template/template_chat.zh.mdx | 26 ++++++++++++++----- .../develop/template/template_workflow.en.mdx | 12 +++++++++ .../develop/template/template_workflow.zh.mdx | 12 +++++++++ 8 files changed, 137 insertions(+), 35 deletions(-) diff --git a/web/app/components/develop/template/template.en.mdx b/web/app/components/develop/template/template.en.mdx index cedb26264dabd2..a324bbefa899c0 100755 --- a/web/app/components/develop/template/template.en.mdx +++ b/web/app/components/develop/template/template.en.mdx @@ -103,6 +103,16 @@ The text generation application offers non-session support and is ideal for tran - `metadata` (object) Metadata - `usage` (Usage) Model usage information - `retriever_resources` (array[RetrieverResource]) Citation and Attribution List + - `event: tts_message` TTS audio stream event, that is, speech synthesis output. The content is an audio block in Mp3 format, encoded as a base64 string. When playing, simply decode the base64 and feed it into the player. (This message is available only when auto-play is enabled) + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The audio after speech synthesis, encoded in base64 text content, when playing, simply decode the base64 and feed it into the player + - `created_at` (int) Creation timestamp, e.g.: 1705395332 + - `event: tts_message_end` TTS audio stream end event, receiving this event indicates the end of the audio stream. + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The end event has no audio, so this is an empty string + - `created_at` (int) Creation timestamp, e.g.: 1705395332 - `event: message_replace` Message content replacement event. When output content moderation is enabled, if the content is flagged, then the message content will be replaced with a preset reply through this event. - `task_id` (string) Task ID, used for request tracking and the below Stop Generate API @@ -185,6 +195,8 @@ The text generation application offers non-session support and is ideal for tran data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "answer": " meet", "created_at": 1679586595} data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "answer": " you", "created_at": 1679586595} data: {"event": "message_end", "id": "5e52ce04-874b-4d27-9045-b3bc80def685", "metadata": {"usage": {"prompt_tokens": 1033, "prompt_unit_price": "0.001", "prompt_price_unit": "0.001", "prompt_price": "0.0010330", "completion_tokens": 135, "completion_unit_price": "0.002", "completion_price_unit": "0.001", "completion_price": "0.0002700", "total_tokens": 1168, "total_price": "0.0013030", "currency": "USD", "latency": 1.381760165997548}}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` @@ -495,29 +507,29 @@ The text generation application offers non-session support and is ideal for tran ### Request Body + + For text messages generated by Dify, simply pass the generated message-id directly. The backend will use the message-id to look up the corresponding content and synthesize the voice information directly. If both message_id and text are provided simultaneously, the message_id is given priority. + Speech generated content。 The user identifier, defined by the developer, must ensure uniqueness within the app. - - Whether to enable streaming output, true、false。 - - + ```bash {{ title: 'cURL' }} curl -o text-to-audio.mp3 -X POST '${props.appDetail.api_base_url}/text-to-audio' \ --header 'Authorization: Bearer {api_key}' \ --header 'Content-Type: application/json' \ --data-raw '{ + "message_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "text": "Hello Dify", - "user": "abc-123", - "streaming": false + "user": "abc-123" }' ``` diff --git a/web/app/components/develop/template/template.zh.mdx b/web/app/components/develop/template/template.zh.mdx index c996ed647c88f8..e593f013c27f6e 100755 --- a/web/app/components/develop/template/template.zh.mdx +++ b/web/app/components/develop/template/template.zh.mdx @@ -97,12 +97,22 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' - `message_id` (string) 消息唯一 ID - `answer` (string) LLM 返回文本块内容 - `created_at` (int) 创建时间戳，如：1705395332 - - `event: message_end` 消息结束事件，收到此事件则代表流式返回结束。 + - `event: message_end` 消息结束事件，收到此事件则代表文本流式返回结束。 - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 - `message_id` (string) 消息唯一 ID - `metadata` (object) 元数据 - `usage` (Usage) 模型用量信息 - `retriever_resources` (array[RetrieverResource]) 引用和归属分段列表 + - `event: tts_message` TTS 音频流事件，即：语音合成输出。内容是Mp3格式的音频块，使用 base64 编码后的字符串，播放的时候直接解码即可。(开启自动播放才有此消息) + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 语音合成之后的音频块使用 Base64 编码之后的文本内容，播放的时候直接 base64 解码送入播放器即可 + - `created_at` (int) 创建时间戳，如：1705395332 + - `event: tts_message_end` TTS 音频流结束事件，收到这个事件表示音频流返回结束。 + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 结束事件是没有音频的，所以这里是空字符串 + - `created_at` (int) 创建时间戳，如：1705395332 - `event: message_replace` 消息内容替换事件。 开启内容审查和审查输出内容时，若命中了审查条件，则会通过此事件替换消息内容为预设回复。 - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 @@ -162,6 +172,8 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' ```streaming {{ title: 'Response' }} data: {"id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "answer": " I", "created_at": 1679586595} data: {"id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "answer": " I", "created_at": 1679586595} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` @@ -456,26 +468,27 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' ### Request Body + + Dify 生成的文本消息，那么直接传递生成的message-id 即可，后台会通过 message_id 查找相应的内容直接合成语音信息。如果同时传 message_id 和 text，优先使用 message_id。 + - 语音生成内容。 + 语音生成内容。如果没有传 message-id的话，则会使用这个字段的内容 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - 是否启用流式输出true、false。 - - + ```bash {{ title: 'cURL' }} curl -o text-to-audio.mp3 -X POST '${props.appDetail.api_base_url}/text-to-audio' \ --header 'Authorization: Bearer {api_key}' \ --header 'Content-Type: application/json' \ --data-raw '{ + "message_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "text": "你好Dify", "user": "abc-123", "streaming": false diff --git a/web/app/components/develop/template/template_advanced_chat.en.mdx b/web/app/components/develop/template/template_advanced_chat.en.mdx index c59f08f5d5148e..8481b4db43e3c2 100644 --- a/web/app/components/develop/template/template_advanced_chat.en.mdx +++ b/web/app/components/develop/template/template_advanced_chat.en.mdx @@ -118,6 +118,16 @@ Chat applications support session persistence, allowing previous chat history to - `metadata` (object) Metadata - `usage` (Usage) Model usage information - `retriever_resources` (array[RetrieverResource]) Citation and Attribution List + - `event: tts_message` TTS audio stream event, that is, speech synthesis output. The content is an audio block in Mp3 format, encoded as a base64 string. When playing, simply decode the base64 and feed it into the player. (This message is available only when auto-play is enabled) + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The audio after speech synthesis, encoded in base64 text content, when playing, simply decode the base64 and feed it into the player + - `created_at` (int) Creation timestamp, e.g.: 1705395332 + - `event: tts_message_end` TTS audio stream end event, receiving this event indicates the end of the audio stream. + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The end event has no audio, so this is an empty string + - `created_at` (int) Creation timestamp, e.g.: 1705395332 - `event: message_replace` Message content replacement event. When output content moderation is enabled, if the content is flagged, then the message content will be replaced with a preset reply through this event. - `task_id` (string) Task ID, used for request tracking and the below Stop Generate API @@ -276,6 +286,8 @@ Chat applications support session persistence, allowing previous chat history to data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " meet", "created_at": 1679586595} data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " you", "created_at": 1679586595} data: {"event": "message_end", "id": "5e52ce04-874b-4d27-9045-b3bc80def685", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "metadata": {"usage": {"prompt_tokens": 1033, "prompt_unit_price": "0.001", "prompt_price_unit": "0.001", "prompt_price": "0.0010330", "completion_tokens": 135, "completion_unit_price": "0.002", "completion_price_unit": "0.001", "completion_price": "0.0002700", "total_tokens": 1168, "total_price": "0.0013030", "currency": "USD", "latency": 1.381760165997548, "retriever_resources": [{"position": 1, "dataset_id": "101b4c97-fc2e-463c-90b1-5261a4cdcafb", "dataset_name": "iPhone", "document_id": "8dd1ad74-0b5f-4175-b735-7d98bbbb4e00", "document_name": "iPhone List", "segment_id": "ed599c7f-2766-4294-9d1d-e5235a61270a", "score": 0.98457545, "content": "\"Model\",\"Release Date\",\"Display Size\",\"Resolution\",\"Processor\",\"RAM\",\"Storage\",\"Camera\",\"Battery\",\"Operating System\"\n\"iPhone 13 Pro Max\",\"September 24, 2021\",\"6.7 inch\",\"1284 x 2778\",\"Hexa-core (2x3.23 GHz Avalanche + 4x1.82 GHz Blizzard)\",\"6 GB\",\"128, 256, 512 GB, 1TB\",\"12 MP\",\"4352 mAh\",\"iOS 15\""}]}}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` @@ -867,26 +879,27 @@ Chat applications support session persistence, allowing previous chat history to ### Request Body + + For text messages generated by Dify, simply pass the generated message-id directly. The backend will use the message-id to look up the corresponding content and synthesize the voice information directly. If both message_id and text are provided simultaneously, the message_id is given priority. + Speech generated content。 The user identifier, defined by the developer, must ensure uniqueness within the app. - - Whether to enable streaming output, true、false。 - - + ```bash {{ title: 'cURL' }} curl -o text-to-audio.mp3 -X POST '${props.appDetail.api_base_url}/text-to-audio' \ --header 'Authorization: Bearer {api_key}' \ --header 'Content-Type: application/json' \ --data-raw '{ + "message_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "text": "Hello Dify", "user": "abc-123", "streaming": false diff --git a/web/app/components/develop/template/template_advanced_chat.zh.mdx b/web/app/components/develop/template/template_advanced_chat.zh.mdx index a07096ebd15eb3..97a2df01a62bfc 100755 --- a/web/app/components/develop/template/template_advanced_chat.zh.mdx +++ b/web/app/components/develop/template/template_advanced_chat.zh.mdx @@ -120,6 +120,16 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' - `metadata` (object) 元数据 - `usage` (Usage) 模型用量信息 - `retriever_resources` (array[RetrieverResource]) 引用和归属分段列表 + - `event: tts_message` TTS 音频流事件，即：语音合成输出。内容是Mp3格式的音频块，使用 base64 编码后的字符串，播放的时候直接解码即可。(开启自动播放才有此消息) + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 语音合成之后的音频块使用 Base64 编码之后的文本内容，播放的时候直接 base64 解码送入播放器即可 + - `created_at` (int) 创建时间戳，如：1705395332 + - `event: tts_message_end` TTS 音频流结束事件，收到这个事件表示音频流返回结束。 + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 结束事件是没有音频的，所以这里是空字符串 + - `created_at` (int) 创建时间戳，如：1705395332 - `event: message_replace` 消息内容替换事件。 开启内容审查和审查输出内容时，若命中了审查条件，则会通过此事件替换消息内容为预设回复。 - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 @@ -287,6 +297,8 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " meet", "created_at": 1679586595} data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " you", "created_at": 1679586595} data: {"event": "message_end", "id": "5e52ce04-874b-4d27-9045-b3bc80def685", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "metadata": {"usage": {"prompt_tokens": 1033, "prompt_unit_price": "0.001", "prompt_price_unit": "0.001", "prompt_price": "0.0010330", "completion_tokens": 135, "completion_unit_price": "0.002", "completion_price_unit": "0.001", "completion_price": "0.0002700", "total_tokens": 1168, "total_price": "0.0013030", "currency": "USD", "latency": 1.381760165997548, "retriever_resources": [{"position": 1, "dataset_id": "101b4c97-fc2e-463c-90b1-5261a4cdcafb", "dataset_name": "iPhone", "document_id": "8dd1ad74-0b5f-4175-b735-7d98bbbb4e00", "document_name": "iPhone List", "segment_id": "ed599c7f-2766-4294-9d1d-e5235a61270a", "score": 0.98457545, "content": "\"Model\",\"Release Date\",\"Display Size\",\"Resolution\",\"Processor\",\"RAM\",\"Storage\",\"Camera\",\"Battery\",\"Operating System\"\n\"iPhone 13 Pro Max\",\"September 24, 2021\",\"6.7 inch\",\"1284 x 2778\",\"Hexa-core (2x3.23 GHz Avalanche + 4x1.82 GHz Blizzard)\",\"6 GB\",\"128, 256, 512 GB, 1TB\",\"12 MP\",\"4352 mAh\",\"iOS 15\""}]}}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` @@ -898,29 +910,29 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' ### Request Body + + Dify 生成的文本消息，那么直接传递生成的message-id 即可，后台会通过 message_id 查找相应的内容直接合成语音信息。如果同时传 message_id 和 text，优先使用 message_id。 + - 语音生成内容。 + 语音生成内容。如果没有传 message-id的话，则会使用这个字段的内容 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - 是否启用流式输出true、false。 - - + ```bash {{ title: 'cURL' }} curl -o text-to-audio.mp3 -X POST '${props.appDetail.api_base_url}/text-to-audio' \ --header 'Authorization: Bearer {api_key}' \ --header 'Content-Type: application/json' \ --data-raw '{ + "message_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "text": "你好Dify", - "user": "abc-123", - "streaming": false + "user": "abc-123" }' ``` diff --git a/web/app/components/develop/template/template_chat.en.mdx b/web/app/components/develop/template/template_chat.en.mdx index 9febd1ca6b7a75..00c02936a63626 100644 --- a/web/app/components/develop/template/template_chat.en.mdx +++ b/web/app/components/develop/template/template_chat.en.mdx @@ -112,6 +112,16 @@ Chat applications support session persistence, allowing previous chat history to - `conversation_id` (string) Conversation ID - `answer` (string) LLM returned text chunk content - `created_at` (int) Creation timestamp, e.g., 1705395332 + - `event: tts_message` TTS audio stream event, that is, speech synthesis output. The content is an audio block in Mp3 format, encoded as a base64 string. When playing, simply decode the base64 and feed it into the player. (This message is available only when auto-play is enabled) + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The audio after speech synthesis, encoded in base64 text content, when playing, simply decode the base64 and feed it into the player + - `created_at` (int) Creation timestamp, e.g.: 1705395332 + - `event: tts_message_end` TTS audio stream end event, receiving this event indicates the end of the audio stream. + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The end event has no audio, so this is an empty string + - `created_at` (int) Creation timestamp, e.g.: 1705395332 - `event: agent_thought` thought of Agent, contains the thought of LLM, input and output of tool calls (Only supported in Agent mode) - `id` (string) Agent thought ID, every iteration has a unique agent thought ID - `task_id` (string) (string) Task ID, used for request tracking and the below Stop Generate API @@ -233,6 +243,8 @@ Chat applications support session persistence, allowing previous chat history to data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " meet", "created_at": 1679586595} data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " you", "created_at": 1679586595} data: {"event": "message_end", "id": "5e52ce04-874b-4d27-9045-b3bc80def685", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "metadata": {"usage": {"prompt_tokens": 1033, "prompt_unit_price": "0.001", "prompt_price_unit": "0.001", "prompt_price": "0.0010330", "completion_tokens": 135, "completion_unit_price": "0.002", "completion_price_unit": "0.001", "completion_price": "0.0002700", "total_tokens": 1168, "total_price": "0.0013030", "currency": "USD", "latency": 1.381760165997548, "retriever_resources": [{"position": 1, "dataset_id": "101b4c97-fc2e-463c-90b1-5261a4cdcafb", "dataset_name": "iPhone", "document_id": "8dd1ad74-0b5f-4175-b735-7d98bbbb4e00", "document_name": "iPhone List", "segment_id": "ed599c7f-2766-4294-9d1d-e5235a61270a", "score": 0.98457545, "content": "\"Model\",\"Release Date\",\"Display Size\",\"Resolution\",\"Processor\",\"RAM\",\"Storage\",\"Camera\",\"Battery\",\"Operating System\"\n\"iPhone 13 Pro Max\",\"September 24, 2021\",\"6.7 inch\",\"1284 x 2778\",\"Hexa-core (2x3.23 GHz Avalanche + 4x1.82 GHz Blizzard)\",\"6 GB\",\"128, 256, 512 GB, 1TB\",\"12 MP\",\"4352 mAh\",\"iOS 15\""}]}}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` ### Response Example(Agent Assistant) @@ -245,7 +257,9 @@ Chat applications support session persistence, allowing previous chat history to data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " meet", "created_at": 1679586595} data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " you", "created_at": 1679586595} data: {"event": "message_end", "id": "5e52ce04-874b-4d27-9045-b3bc80def685", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "metadata": {"usage": {"prompt_tokens": 1033, "prompt_unit_price": "0.001", "prompt_price_unit": "0.001", "prompt_price": "0.0010330", "completion_tokens": 135, "completion_unit_price": "0.002", "completion_price_unit": "0.001", "completion_price": "0.0002700", "total_tokens": 1168, "total_price": "0.0013030", "currency": "USD", "latency": 1.381760165997548, "retriever_resources": [{"position": 1, "dataset_id": "101b4c97-fc2e-463c-90b1-5261a4cdcafb", "dataset_name": "iPhone", "document_id": "8dd1ad74-0b5f-4175-b735-7d98bbbb4e00", "document_name": "iPhone List", "segment_id": "ed599c7f-2766-4294-9d1d-e5235a61270a", "score": 0.98457545, "content": "\"Model\",\"Release Date\",\"Display Size\",\"Resolution\",\"Processor\",\"RAM\",\"Storage\",\"Camera\",\"Battery\",\"Operating System\"\n\"iPhone 13 Pro Max\",\"September 24, 2021\",\"6.7 inch\",\"1284 x 2778\",\"Hexa-core (2x3.23 GHz Avalanche + 4x1.82 GHz Blizzard)\",\"6 GB\",\"128, 256, 512 GB, 1TB\",\"12 MP\",\"4352 mAh\",\"iOS 15\""}]}}} - ``` + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} + ``` @@ -905,25 +919,25 @@ Chat applications support session persistence, allowing previous chat history to ### Request Body + + For text messages generated by Dify, simply pass the generated message-id directly. The backend will use the message-id to look up the corresponding content and synthesize the voice information directly. If both message_id and text are provided simultaneously, the message_id is given priority. + Speech generated content。 The user identifier, defined by the developer, must ensure uniqueness within the app. - - Whether to enable streaming output, true、false。 - - + ```bash {{ title: 'cURL' }} curl --location --request POST '${props.appDetail.api_base_url}/text-to-audio' \ --header 'Authorization: Bearer ENTER-YOUR-SECRET-KEY' \ - --form 'file=Hello Dify;user=abc-123;streaming=false' + --form 'file=Hello Dify;user=abc-123;message_id=5ad4cb98-f0c7-4085-b384-88c403be6290' ``` diff --git a/web/app/components/develop/template/template_chat.zh.mdx b/web/app/components/develop/template/template_chat.zh.mdx index 36d47651b5271f..9323313d9fa097 100644 --- a/web/app/components/develop/template/template_chat.zh.mdx +++ b/web/app/components/develop/template/template_chat.zh.mdx @@ -140,6 +140,16 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' - `metadata` (object) 元数据 - `usage` (Usage) 模型用量信息 - `retriever_resources` (array[RetrieverResource]) 引用和归属分段列表 + - `event: tts_message` TTS 音频流事件，即：语音合成输出。内容是Mp3格式的音频块，使用 base64 编码后的字符串，播放的时候直接解码即可。(开启自动播放才有此消息) + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 语音合成之后的音频块使用 Base64 编码之后的文本内容，播放的时候直接 base64 解码送入播放器即可 + - `created_at` (int) 创建时间戳，如：1705395332 + - `event: tts_message_end` TTS 音频流结束事件，收到这个事件表示音频流返回结束。 + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 结束事件是没有音频的，所以这里是空字符串 + - `created_at` (int) 创建时间戳，如：1705395332 - `event: message_replace` 消息内容替换事件。 开启内容审查和审查输出内容时，若命中了审查条件，则会通过此事件替换消息内容为预设回复。 - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 @@ -246,6 +256,8 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " meet", "created_at": 1679586595} data: {"event": "message", "message_id": : "5ad4cb98-f0c7-4085-b384-88c403be6290", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "answer": " you", "created_at": 1679586595} data: {"event": "message_end", "id": "5e52ce04-874b-4d27-9045-b3bc80def685", "conversation_id": "45701982-8118-4bc5-8e9b-64562b4555f2", "metadata": {"usage": {"prompt_tokens": 1033, "prompt_unit_price": "0.001", "prompt_price_unit": "0.001", "prompt_price": "0.0010330", "completion_tokens": 135, "completion_unit_price": "0.002", "completion_price_unit": "0.001", "completion_price": "0.0002700", "total_tokens": 1168, "total_price": "0.0013030", "currency": "USD", "latency": 1.381760165997548, "retriever_resources": [{"position": 1, "dataset_id": "101b4c97-fc2e-463c-90b1-5261a4cdcafb", "dataset_name": "iPhone", "document_id": "8dd1ad74-0b5f-4175-b735-7d98bbbb4e00", "document_name": "iPhone List", "segment_id": "ed599c7f-2766-4294-9d1d-e5235a61270a", "score": 0.98457545, "content": "\"Model\",\"Release Date\",\"Display Size\",\"Resolution\",\"Processor\",\"RAM\",\"Storage\",\"Camera\",\"Battery\",\"Operating System\"\n\"iPhone 13 Pro Max\",\"September 24, 2021\",\"6.7 inch\",\"1284 x 2778\",\"Hexa-core (2x3.23 GHz Avalanche + 4x1.82 GHz Blizzard)\",\"6 GB\",\"128, 256, 512 GB, 1TB\",\"12 MP\",\"4352 mAh\",\"iOS 15\""}]}}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` @@ -263,6 +275,8 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' data: {"event": "agent_message", "id": "1fb10045-55fd-4040-99e6-d048d07cbad3", "task_id": "9cf1ddd7-f94b-459b-b942-b77b26c59e9b", "message_id": "1fb10045-55fd-4040-99e6-d048d07cbad3", "answer": " suit .", "created_at": 1705639511, "conversation_id": "c216c595-2d89-438c-b33c-aae5ddddd142"} data: {"event": "agent_thought", "id": "67a99dc1-4f82-42d3-b354-18d4594840c8", "task_id": "9cf1ddd7-f94b-459b-b942-b77b26c59e9b", "message_id": "1fb10045-55fd-4040-99e6-d048d07cbad3", "position": 2, "thought": "I have created an image of a cute Japanese anime girl with white hair and blue eyes wearing a bunny girl suit.", "observation": "", "tool": "", "tool_input": "", "created_at": 1705639511, "message_files": [], "conversation_id": "c216c595-2d89-438c-b33c-aae5ddddd142"} data: {"event": "message_end", "task_id": "9cf1ddd7-f94b-459b-b942-b77b26c59e9b", "id": "1fb10045-55fd-4040-99e6-d048d07cbad3", "message_id": "1fb10045-55fd-4040-99e6-d048d07cbad3", "conversation_id": "c216c595-2d89-438c-b33c-aae5ddddd142", "metadata": {"usage": {"prompt_tokens": 305, "prompt_unit_price": "0.001", "prompt_price_unit": "0.001", "prompt_price": "0.0003050", "completion_tokens": 97, "completion_unit_price": "0.002", "completion_price_unit": "0.001", "completion_price": "0.0001940", "total_tokens": 184, "total_price": "0.0002290", "currency": "USD", "latency": 1.771092874929309}}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` @@ -915,25 +929,25 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' ### Request Body + + Dify 生成的文本消息，那么直接传递生成的message-id 即可，后台会通过 message_id 查找相应的内容直接合成语音信息。如果同时传 message_id 和 text，优先使用 message_id。 + - 语音生成内容。 + 语音生成内容。如果没有传 message-id的话，则会使用这个字段的内容 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - 是否启用流式输出true、false。 - - + ```bash {{ title: 'cURL' }} curl --location --request POST '${props.appDetail.api_base_url}/text-to-audio' \ --header 'Authorization: Bearer ENTER-YOUR-SECRET-KEY' \ - --form 'file=你好Dify;user=abc-123;streaming=false' + --form 'file=你好Dify;user=abc-123;message_id=5ad4cb98-f0c7-4085-b384-88c403be6290' ``` diff --git a/web/app/components/develop/template/template_workflow.en.mdx b/web/app/components/develop/template/template_workflow.en.mdx index 9580c2aa0bca69..64a9092f591d59 100644 --- a/web/app/components/develop/template/template_workflow.en.mdx +++ b/web/app/components/develop/template/template_workflow.en.mdx @@ -147,6 +147,16 @@ Workflow applications offers non-session support and is ideal for translation, a - `total_steps` (int) default 0 - `created_at` (timestamp) start time - `finished_at` (timestamp) end time + - `event: tts_message` TTS audio stream event, that is, speech synthesis output. The content is an audio block in Mp3 format, encoded as a base64 string. When playing, simply decode the base64 and feed it into the player. (This message is available only when auto-play is enabled) + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The audio after speech synthesis, encoded in base64 text content, when playing, simply decode the base64 and feed it into the player + - `created_at` (int) Creation timestamp, e.g.: 1705395332 + - `event: tts_message_end` TTS audio stream end event, receiving this event indicates the end of the audio stream. + - `task_id` (string) Task ID, used for request tracking and the stop response interface below + - `message_id` (string) Unique message ID + - `audio` (string) The end event has no audio, so this is an empty string + - `created_at` (int) Creation timestamp, e.g.: 1705395332 - `event: ping` Ping event every 10 seconds to keep the connection alive. ### Errors @@ -204,6 +214,8 @@ Workflow applications offers non-session support and is ideal for translation, a data: {"event": "node_started", "task_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "workflow_run_id": "5ad498-f0c7-4085-b384-88cbe6290", "data": {"id": "5ad498-f0c7-4085-b384-88cbe6290", "node_id": "dfjasklfjdslag", "node_type": "start", "title": "Start", "index": 0, "predecessor_node_id": "fdljewklfklgejlglsd", "inputs": {}, "created_at": 1679586595}} data: {"event": "node_finished", "task_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "workflow_run_id": "5ad498-f0c7-4085-b384-88cbe6290", "data": {"id": "5ad498-f0c7-4085-b384-88cbe6290", "node_id": "dfjasklfjdslag", "node_type": "start", "title": "Start", "index": 0, "predecessor_node_id": "fdljewklfklgejlglsd", "inputs": {}, "outputs": {}, "status": "succeeded", "elapsed_time": 0.324, "execution_metadata": {"total_tokens": 63127864, "total_price": 2.378, "currency": "USD"}, "created_at": 1679586595}} data: {"event": "workflow_finished", "task_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "workflow_run_id": "5ad498-f0c7-4085-b384-88cbe6290", "data": {"id": "5ad498-f0c7-4085-b384-88cbe6290", "workflow_id": "dfjasklfjdslag", "outputs": {}, "status": "succeeded", "elapsed_time": 0.324, "total_tokens": 63127864, "total_steps": "1", "created_at": 1679586595, "finished_at": 1679976595}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` diff --git a/web/app/components/develop/template/template_workflow.zh.mdx b/web/app/components/develop/template/template_workflow.zh.mdx index 6264eb57e5ecd6..c891982873124c 100644 --- a/web/app/components/develop/template/template_workflow.zh.mdx +++ b/web/app/components/develop/template/template_workflow.zh.mdx @@ -143,6 +143,16 @@ Workflow 应用无会话支持，适合用于翻译/文章写作/总结 AI 等 - `total_steps` (int) 总步数（冗余），默认 0 - `created_at` (timestamp) 开始时间 - `finished_at` (timestamp) 结束时间 + - `event: tts_message` TTS 音频流事件，即：语音合成输出。内容是Mp3格式的音频块，使用 base64 编码后的字符串，播放的时候直接解码即可。(开启自动播放才有此消息) + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 语音合成之后的音频块使用 Base64 编码之后的文本内容，播放的时候直接 base64 解码送入播放器即可 + - `created_at` (int) 创建时间戳，如：1705395332 + - `event: tts_message_end` TTS 音频流结束事件，收到这个事件表示音频流返回结束。 + - `task_id` (string) 任务 ID，用于请求跟踪和下方的停止响应接口 + - `message_id` (string) 消息唯一 ID + - `audio` (string) 结束事件是没有音频的，所以这里是空字符串 + - `created_at` (int) 创建时间戳，如：1705395332 - `event: ping` 每 10s 一次的 ping 事件，保持连接存活。 ### Errors @@ -200,6 +210,8 @@ Workflow 应用无会话支持，适合用于翻译/文章写作/总结 AI 等 data: {"event": "node_started", "task_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "workflow_run_id": "5ad498-f0c7-4085-b384-88cbe6290", "data": {"id": "5ad498-f0c7-4085-b384-88cbe6290", "node_id": "dfjasklfjdslag", "node_type": "start", "title": "Start", "index": 0, "predecessor_node_id": "fdljewklfklgejlglsd", "inputs": {}, "created_at": 1679586595}} data: {"event": "node_finished", "task_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "workflow_run_id": "5ad498-f0c7-4085-b384-88cbe6290", "data": {"id": "5ad498-f0c7-4085-b384-88cbe6290", "node_id": "dfjasklfjdslag", "node_type": "start", "title": "Start", "index": 0, "predecessor_node_id": "fdljewklfklgejlglsd", "inputs": {}, "outputs": {}, "status": "succeeded", "elapsed_time": 0.324, "execution_metadata": {"total_tokens": 63127864, "total_price": 2.378, "currency": "USD"}, "created_at": 1679586595}} data: {"event": "workflow_finished", "task_id": "5ad4cb98-f0c7-4085-b384-88c403be6290", "workflow_run_id": "5ad498-f0c7-4085-b384-88cbe6290", "data": {"id": "5ad498-f0c7-4085-b384-88cbe6290", "workflow_id": "dfjasklfjdslag", "outputs": {}, "status": "succeeded", "elapsed_time": 0.324, "total_tokens": 63127864, "total_steps": "1", "created_at": 1679586595, "finished_at": 1679976595}} + data: {"event": "tts_message", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq"} + data: {"event": "tts_message_end", "conversation_id": "23dd85f3-1a41-4ea0-b7a9-062734ccfaf9", "message_id": "a8bdc41c-13b2-4c18-bfd9-054b9803038c", "created_at": 1721205487, "task_id": "3bf8a0bb-e73b-4690-9e66-4e429bad8ee7", "audio": ""} ``` From 65bc4e0fc091ed643f4ca49e484715c1059dbd35 Mon Sep 17 00:00:00 2001 From: faye1225 <56664035+faye1225@users.noreply.github.com> Date: Wed, 17 Jul 2024 20:24:31 +0800 Subject: [PATCH 14/62] Fix issues related to search apps, notification duration, and loading icon on the explore page (#6374) --- .../app/configuration/debug/index.tsx | 6 +- .../components/app/configuration/index.tsx | 10 +-- .../components/base/chat/chat/chat-input.tsx | 2 +- web/app/components/base/toast/index.tsx | 5 +- .../settings/permission-selector/index.tsx | 2 + web/app/components/explore/app-list/index.tsx | 62 ++++++++++++++----- 6 files changed, 59 insertions(+), 28 deletions(-) diff --git a/web/app/components/app/configuration/debug/index.tsx b/web/app/components/app/configuration/debug/index.tsx index ff476246fe5f45..45e2ae720b3611 100644 --- a/web/app/components/app/configuration/debug/index.tsx +++ b/web/app/components/app/configuration/debug/index.tsx @@ -136,7 +136,7 @@ const Debug: FC = ({ const { notify } = useContext(ToastContext) const logError = useCallback((message: string) => { - notify({ type: 'error', message, duration: 3000 }) + notify({ type: 'error', message }) }, [notify]) const [completionFiles, setCompletionFiles] = useState([]) @@ -144,11 +144,11 @@ const Debug: FC = ({ if (isAdvancedMode && mode !== AppType.completion) { if (modelModeType === ModelModeType.completion) { if (!hasSetBlockStatus.history) { - notify({ type: 'error', message: t('appDebug.otherError.historyNoBeEmpty'), duration: 3000 }) + notify({ type: 'error', message: t('appDebug.otherError.historyNoBeEmpty') }) return false } if (!hasSetBlockStatus.query) { - notify({ type: 'error', message: t('appDebug.otherError.queryNoBeEmpty'), duration: 3000 }) + notify({ type: 'error', message: t('appDebug.otherError.queryNoBeEmpty') }) return false } } diff --git a/web/app/components/app/configuration/index.tsx b/web/app/components/app/configuration/index.tsx index 3b61a0b7fe62bc..030d0ef5263da0 100644 --- a/web/app/components/app/configuration/index.tsx +++ b/web/app/components/app/configuration/index.tsx @@ -555,23 +555,23 @@ const Configuration: FC = () => { const promptVariables = modelConfig.configs.prompt_variables if (promptEmpty) { - notify({ type: 'error', message: t('appDebug.otherError.promptNoBeEmpty'), duration: 3000 }) + notify({ type: 'error', message: t('appDebug.otherError.promptNoBeEmpty') }) return } if (isAdvancedMode && mode !== AppType.completion) { if (modelModeType === ModelModeType.completion) { if (!hasSetBlockStatus.history) { - notify({ type: 'error', message: t('appDebug.otherError.historyNoBeEmpty'), duration: 3000 }) + notify({ type: 'error', message: t('appDebug.otherError.historyNoBeEmpty') }) return } if (!hasSetBlockStatus.query) { - notify({ type: 'error', message: t('appDebug.otherError.queryNoBeEmpty'), duration: 3000 }) + notify({ type: 'error', message: t('appDebug.otherError.queryNoBeEmpty') }) return } } } if (contextVarEmpty) { - notify({ type: 'error', message: t('appDebug.feature.dataSet.queryVariable.contextVarNotEmpty'), duration: 3000 }) + notify({ type: 'error', message: t('appDebug.feature.dataSet.queryVariable.contextVarNotEmpty') }) return } const postDatasets = dataSets.map(({ id }) => ({ @@ -638,7 +638,7 @@ const Configuration: FC = () => { modelConfig: newModelConfig, completionParams, }) - notify({ type: 'success', message: t('common.api.success'), duration: 3000 }) + notify({ type: 'success', message: t('common.api.success') }) setCanReturnToSimpleMode(false) return true diff --git a/web/app/components/base/chat/chat/chat-input.tsx b/web/app/components/base/chat/chat/chat-input.tsx index e5a99be065a60d..6d15010f85685c 100644 --- a/web/app/components/base/chat/chat/chat-input.tsx +++ b/web/app/components/base/chat/chat/chat-input.tsx @@ -106,7 +106,7 @@ const ChatInput: FC = ({ } const logError = (message: string) => { - notify({ type: 'error', message, duration: 3000 }) + notify({ type: 'error', message }) } const handleVoiceInputShow = () => { (Recorder as any).getPermission().then(() => { diff --git a/web/app/components/base/toast/index.tsx b/web/app/components/base/toast/index.tsx index 06069f57e8ddef..903b6d29a7d0c9 100644 --- a/web/app/components/base/toast/index.tsx +++ b/web/app/components/base/toast/index.tsx @@ -22,7 +22,6 @@ export type IToastProps = { type IToastContext = { notify: (props: IToastProps) => void } -const defaultDuring = 3000 export const ToastContext = createContext({} as IToastContext) export const useToastContext = () => useContext(ToastContext) @@ -89,10 +88,10 @@ export const ToastProvider = ({ const placeholder: IToastProps = { type: 'info', message: 'Toast message', - duration: 3000, + duration: 6000, } const [params, setParams] = React.useState(placeholder) - + const defaultDuring = params.type === 'success' ? 3000 : 6000 const [mounted, setMounted] = useState(false) useEffect(() => { diff --git a/web/app/components/datasets/settings/permission-selector/index.tsx b/web/app/components/datasets/settings/permission-selector/index.tsx index 2405f9512b7f5e..910dfc5347b3f2 100644 --- a/web/app/components/datasets/settings/permission-selector/index.tsx +++ b/web/app/components/datasets/settings/permission-selector/index.tsx @@ -51,9 +51,11 @@ const PermissionSelector = ({ disabled, permission, value, memberList, onChange, ...memberList.filter(member => member.id !== userProfile.id).filter(member => value.includes(member.id)), ].map(member => member.name).join(', ') }, [userProfile, value, memberList]) + const showMe = useMemo(() => { return userProfile.name.includes(searchKeywords) || userProfile.email.includes(searchKeywords) }, [searchKeywords, userProfile]) + const filteredMemberList = useMemo(() => { return memberList.filter(member => (member.name.includes(searchKeywords) || member.email.includes(searchKeywords)) && member.id !== userProfile.id && ['owner', 'admin', 'editor', 'dataset_operator'].includes(member.role)) }, [memberList, searchKeywords, userProfile]) diff --git a/web/app/components/explore/app-list/index.tsx b/web/app/components/explore/app-list/index.tsx index b4658934100a76..4f2926677d0d45 100644 --- a/web/app/components/explore/app-list/index.tsx +++ b/web/app/components/explore/app-list/index.tsx @@ -5,6 +5,7 @@ import { useRouter } from 'next/navigation' import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' import useSWR from 'swr' +import { useDebounceFn } from 'ahooks' import Toast from '../../base/toast' import s from './style.module.css' import cn from '@/utils/classnames' @@ -22,6 +23,7 @@ import Loading from '@/app/components/base/loading' import { NEED_REFRESH_APP_LIST_KEY } from '@/config' import { useAppContext } from '@/context/app-context' import { getRedirection } from '@/utils/app-redirection' +import SearchInput from '@/app/components/base/search-input' type AppsProps = { pageType?: PageType @@ -43,6 +45,18 @@ const Apps = ({ const { hasEditPermission } = useContext(ExploreContext) const allCategoriesEn = t('explore.apps.allCategories', { lng: 'en' }) + const [keywords, setKeywords] = useState('') + const [searchKeywords, setSearchKeywords] = useState('') + + const { run: handleSearch } = useDebounceFn(() => { + setSearchKeywords(keywords) + }, { wait: 500 }) + + const handleKeywordsChange = (value: string) => { + setKeywords(value) + handleSearch() + } + const [currentType, setCurrentType] = useState('') const [currCategory, setCurrCategory] = useTabSearchParams({ defaultTab: allCategoriesEn, @@ -89,6 +103,17 @@ const Apps = ({ } }, [currentType, currCategory, allCategoriesEn, allList]) + const searchFilteredList = useMemo(() => { + if (!searchKeywords || !filteredList || filteredList.length === 0) + return filteredList + + const lowerCaseSearchKeywords = searchKeywords.toLowerCase() + + return filteredList.filter(item => + item.app && item.app.name && item.app.name.toLowerCase().includes(lowerCaseSearchKeywords), + ) + }, [searchKeywords, filteredList]) + const [currApp, setCurrApp] = React.useState(null) const [isShowCreateModal, setIsShowCreateModal] = React.useState(false) const onCreate: CreateAppModalProps['onConfirm'] = async ({ @@ -123,7 +148,7 @@ const Apps = ({ } } - if (!categories) { + if (!categories || categories.length === 0) { return (
@@ -143,25 +168,30 @@ const Apps = ({
)}
- {pageType !== PageType.EXPLORE && ( - <> - -
- - )} - + <> + {pageType !== PageType.EXPLORE && ( + <> + +
+ + )} + + + +
+
From 35f4a264d6774bf43dd769a200fc2189e91bf6a7 Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Wed, 17 Jul 2024 21:19:04 +0800 Subject: [PATCH 18/62] fix: default duration (#6393) --- web/app/components/base/toast/index.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/app/components/base/toast/index.tsx b/web/app/components/base/toast/index.tsx index 903b6d29a7d0c9..3e13db5d7ff90a 100644 --- a/web/app/components/base/toast/index.tsx +++ b/web/app/components/base/toast/index.tsx @@ -27,7 +27,6 @@ export const ToastContext = createContext({} as IToastContext) export const useToastContext = () => useContext(ToastContext) const Toast = ({ type = 'info', - duration, message, children, className, @@ -91,7 +90,7 @@ export const ToastProvider = ({ duration: 6000, } const [params, setParams] = React.useState(placeholder) - const defaultDuring = params.type === 'success' ? 3000 : 6000 + const defaultDuring = (params.type === 'success' || params.type === 'info') ? 3000 : 6000 const [mounted, setMounted] = useState(false) useEffect(() => { @@ -100,7 +99,7 @@ export const ToastProvider = ({ setMounted(false) }, params.duration || defaultDuring) } - }, [mounted]) + }, [defaultDuring, mounted, params.duration]) return { @@ -119,6 +118,7 @@ Toast.notify = ({ duration, className, }: Pick) => { + const defaultDuring = (type === 'success' || type === 'info') ? 3000 : 6000 if (typeof window === 'object') { const holder = document.createElement('div') const root = createRoot(holder) From 8a80af39c916a3b71a0b8f99b3cf018258e3eb69 Mon Sep 17 00:00:00 2001 From: Poorandy <63747977+Poorandy@users.noreply.github.com> Date: Wed, 17 Jul 2024 22:26:18 +0800 Subject: [PATCH 19/62] refactor(models&tools): switch to dify_config in models and tools. (#6394) Co-authored-by: Poorandy --- api/core/tools/tool_file_manager.py | 121 ++++++++++++------- api/core/tools/tool_manager.py | 7 +- api/core/workflow/workflow_engine_manager.py | 43 ++++--- api/models/dataset.py | 4 +- api/models/model.py | 7 +- 5 files changed, 105 insertions(+), 77 deletions(-) diff --git a/api/core/tools/tool_file_manager.py b/api/core/tools/tool_file_manager.py index 207f009eed0966..f9f7c7d78a7f28 100644 --- a/api/core/tools/tool_file_manager.py +++ b/api/core/tools/tool_file_manager.py @@ -9,9 +9,9 @@ from typing import Optional, Union from uuid import uuid4 -from flask import current_app from httpx import get +from configs import dify_config from extensions.ext_database import db from extensions.ext_storage import storage from models.model import MessageFile @@ -26,25 +26,25 @@ def sign_file(tool_file_id: str, extension: str) -> str: """ sign file to get a temporary url """ - base_url = current_app.config.get('FILES_URL') + base_url = dify_config.FILES_URL file_preview_url = f'{base_url}/files/tools/{tool_file_id}{extension}' timestamp = str(int(time.time())) nonce = os.urandom(16).hex() - data_to_sign = f"file-preview|{tool_file_id}|{timestamp}|{nonce}" - secret_key = current_app.config['SECRET_KEY'].encode() + data_to_sign = f'file-preview|{tool_file_id}|{timestamp}|{nonce}' + secret_key = dify_config.SECRET_KEY.encode() if dify_config.SECRET_KEY else b'' sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest() encoded_sign = base64.urlsafe_b64encode(sign).decode() - return f"{file_preview_url}?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}" + return f'{file_preview_url}?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}' @staticmethod def verify_file(file_id: str, timestamp: str, nonce: str, sign: str) -> bool: """ verify signature """ - data_to_sign = f"file-preview|{file_id}|{timestamp}|{nonce}" - secret_key = current_app.config['SECRET_KEY'].encode() + data_to_sign = f'file-preview|{file_id}|{timestamp}|{nonce}' + secret_key = dify_config.SECRET_KEY.encode() if dify_config.SECRET_KEY else b'' recalculated_sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest() recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode() @@ -53,23 +53,23 @@ def verify_file(file_id: str, timestamp: str, nonce: str, sign: str) -> bool: return False current_time = int(time.time()) - return current_time - int(timestamp) <= current_app.config.get('FILES_ACCESS_TIMEOUT') + return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT @staticmethod - def create_file_by_raw(user_id: str, tenant_id: str, - conversation_id: Optional[str], file_binary: bytes, - mimetype: str - ) -> ToolFile: + def create_file_by_raw( + user_id: str, tenant_id: str, conversation_id: Optional[str], file_binary: bytes, mimetype: str + ) -> ToolFile: """ create file """ extension = guess_extension(mimetype) or '.bin' unique_name = uuid4().hex - filename = f"tools/{tenant_id}/{unique_name}{extension}" + filename = f'tools/{tenant_id}/{unique_name}{extension}' storage.save(filename, file_binary) - tool_file = ToolFile(user_id=user_id, tenant_id=tenant_id, - conversation_id=conversation_id, file_key=filename, mimetype=mimetype) + tool_file = ToolFile( + user_id=user_id, tenant_id=tenant_id, conversation_id=conversation_id, file_key=filename, mimetype=mimetype + ) db.session.add(tool_file) db.session.commit() @@ -77,9 +77,12 @@ def create_file_by_raw(user_id: str, tenant_id: str, return tool_file @staticmethod - def create_file_by_url(user_id: str, tenant_id: str, - conversation_id: str, file_url: str, - ) -> ToolFile: + def create_file_by_url( + user_id: str, + tenant_id: str, + conversation_id: str, + file_url: str, + ) -> ToolFile: """ create file """ @@ -90,12 +93,17 @@ def create_file_by_url(user_id: str, tenant_id: str, mimetype = guess_type(file_url)[0] or 'octet/stream' extension = guess_extension(mimetype) or '.bin' unique_name = uuid4().hex - filename = f"tools/{tenant_id}/{unique_name}{extension}" + filename = f'tools/{tenant_id}/{unique_name}{extension}' storage.save(filename, blob) - tool_file = ToolFile(user_id=user_id, tenant_id=tenant_id, - conversation_id=conversation_id, file_key=filename, - mimetype=mimetype, original_url=file_url) + tool_file = ToolFile( + user_id=user_id, + tenant_id=tenant_id, + conversation_id=conversation_id, + file_key=filename, + mimetype=mimetype, + original_url=file_url, + ) db.session.add(tool_file) db.session.commit() @@ -103,15 +111,15 @@ def create_file_by_url(user_id: str, tenant_id: str, return tool_file @staticmethod - def create_file_by_key(user_id: str, tenant_id: str, - conversation_id: str, file_key: str, - mimetype: str - ) -> ToolFile: + def create_file_by_key( + user_id: str, tenant_id: str, conversation_id: str, file_key: str, mimetype: str + ) -> ToolFile: """ create file """ - tool_file = ToolFile(user_id=user_id, tenant_id=tenant_id, - conversation_id=conversation_id, file_key=file_key, mimetype=mimetype) + tool_file = ToolFile( + user_id=user_id, tenant_id=tenant_id, conversation_id=conversation_id, file_key=file_key, mimetype=mimetype + ) return tool_file @staticmethod @@ -123,9 +131,13 @@ def get_file_binary(id: str) -> Union[tuple[bytes, str], None]: :return: the binary of the file, mime type """ - tool_file: ToolFile = db.session.query(ToolFile).filter( - ToolFile.id == id, - ).first() + tool_file: ToolFile = ( + db.session.query(ToolFile) + .filter( + ToolFile.id == id, + ) + .first() + ) if not tool_file: return None @@ -143,18 +155,31 @@ def get_file_binary_by_message_file_id(id: str) -> Union[tuple[bytes, str], None :return: the binary of the file, mime type """ - message_file: MessageFile = db.session.query(MessageFile).filter( - MessageFile.id == id, - ).first() - - # get tool file id - tool_file_id = message_file.url.split('/')[-1] - # trim extension - tool_file_id = tool_file_id.split('.')[0] - - tool_file: ToolFile = db.session.query(ToolFile).filter( - ToolFile.id == tool_file_id, - ).first() + message_file: MessageFile = ( + db.session.query(MessageFile) + .filter( + MessageFile.id == id, + ) + .first() + ) + + # Check if message_file is not None + if message_file is not None: + # get tool file id + tool_file_id = message_file.url.split('/')[-1] + # trim extension + tool_file_id = tool_file_id.split('.')[0] + else: + tool_file_id = None + + + tool_file: ToolFile = ( + db.session.query(ToolFile) + .filter( + ToolFile.id == tool_file_id, + ) + .first() + ) if not tool_file: return None @@ -172,9 +197,13 @@ def get_file_generator_by_tool_file_id(tool_file_id: str) -> Union[tuple[Generat :return: the binary of the file, mime type """ - tool_file: ToolFile = db.session.query(ToolFile).filter( - ToolFile.id == tool_file_id, - ).first() + tool_file: ToolFile = ( + db.session.query(ToolFile) + .filter( + ToolFile.id == tool_file_id, + ) + .first() + ) if not tool_file: return None diff --git a/api/core/tools/tool_manager.py b/api/core/tools/tool_manager.py index e30a905cbcf309..3342300eb4e480 100644 --- a/api/core/tools/tool_manager.py +++ b/api/core/tools/tool_manager.py @@ -6,8 +6,7 @@ from threading import Lock from typing import Any, Union -from flask import current_app - +from configs import dify_config from core.agent.entities import AgentToolEntity from core.app.entities.app_invoke_entities import InvokeFrom from core.helper.module_import_helper import load_single_subclass_from_source @@ -566,7 +565,7 @@ def get_tool_icon(cls, tenant_id: str, provider_type: str, provider_id: str) -> provider_type = provider_type provider_id = provider_id if provider_type == 'builtin': - return (current_app.config.get("CONSOLE_API_URL") + return (dify_config.CONSOLE_API_URL + "/console/api/workspaces/current/tool-provider/builtin/" + provider_id + "/icon") @@ -594,4 +593,4 @@ def get_tool_icon(cls, tenant_id: str, provider_type: str, provider_id: str) -> else: raise ValueError(f"provider type {provider_type} not found") -ToolManager.load_builtin_providers_cache() \ No newline at end of file +ToolManager.load_builtin_providers_cache() diff --git a/api/core/workflow/workflow_engine_manager.py b/api/core/workflow/workflow_engine_manager.py index 22deafb8a37997..e81bf684a96a14 100644 --- a/api/core/workflow/workflow_engine_manager.py +++ b/api/core/workflow/workflow_engine_manager.py @@ -2,8 +2,7 @@ import time from typing import Optional, cast -from flask import current_app - +from configs import dify_config from core.app.app_config.entities import FileExtraConfig from core.app.apps.base_app_queue_manager import GenerateTaskStoppedException from core.app.entities.app_invoke_entities import InvokeFrom @@ -118,7 +117,7 @@ def run_workflow(self, workflow: Workflow, if not isinstance(graph.get('edges'), list): raise ValueError('edges in workflow graph must be a list') - + # init variable pool if not variable_pool: variable_pool = VariablePool( @@ -126,7 +125,7 @@ def run_workflow(self, workflow: Workflow, user_inputs=user_inputs ) - workflow_call_max_depth = current_app.config.get("WORKFLOW_CALL_MAX_DEPTH") + workflow_call_max_depth = dify_config.WORKFLOW_CALL_MAX_DEPTH if call_depth > workflow_call_max_depth: raise ValueError('Max workflow call depth {} reached.'.format(workflow_call_max_depth)) @@ -177,8 +176,8 @@ def _run_workflow(self, workflow: Workflow, predecessor_node: BaseNode = None current_iteration_node: BaseIterationNode = None has_entry_node = False - max_execution_steps = current_app.config.get("WORKFLOW_MAX_EXECUTION_STEPS") - max_execution_time = current_app.config.get("WORKFLOW_MAX_EXECUTION_TIME") + max_execution_steps = dify_config.WORKFLOW_MAX_EXECUTION_STEPS + max_execution_time = dify_config.WORKFLOW_MAX_EXECUTION_TIME while True: # get next node, multiple target nodes in the future next_node = self._get_next_overall_node( @@ -237,7 +236,7 @@ def _run_workflow(self, workflow: Workflow, next_node_id = next_iteration # get next id next_node = self._get_node(workflow_run_state, graph, next_node_id, callbacks) - + if not next_node: break @@ -398,7 +397,7 @@ def single_step_run_workflow_node(self, workflow: Workflow, tenant_id=workflow.tenant_id, node_instance=node_instance ) - + # run node node_run_result = node_instance.run( variable_pool=variable_pool @@ -443,7 +442,7 @@ def single_step_run_iteration_workflow_node(self, workflow: Workflow, node_config = node else: raise ValueError('node id is not an iteration node') - + # init variable pool variable_pool = VariablePool( system_variables={}, @@ -452,7 +451,7 @@ def single_step_run_iteration_workflow_node(self, workflow: Workflow, # variable selector to variable mapping iteration_nested_nodes = [ - node for node in nodes + node for node in nodes if node.get('data', {}).get('iteration_id') == node_id or node.get('id') == node_id ] iteration_nested_node_ids = [node.get('id') for node in iteration_nested_nodes] @@ -475,13 +474,13 @@ def single_step_run_iteration_workflow_node(self, workflow: Workflow, # remove iteration variables variable_mapping = { - f'{node_config.get("id")}.{key}': value for key, value in variable_mapping.items() + f'{node_config.get("id")}.{key}': value for key, value in variable_mapping.items() if value[0] != node_id } # remove variable out from iteration variable_mapping = { - key: value for key, value in variable_mapping.items() + key: value for key, value in variable_mapping.items() if value[0] not in iteration_nested_node_ids } @@ -561,7 +560,7 @@ def _workflow_run_failed(self, error: str, error=error ) - def _workflow_iteration_started(self, graph: dict, + def _workflow_iteration_started(self, graph: dict, current_iteration_node: BaseIterationNode, workflow_run_state: WorkflowRunState, predecessor_node_id: Optional[str] = None, @@ -600,7 +599,7 @@ def _workflow_iteration_started(self, graph: dict, def _workflow_iteration_next(self, graph: dict, current_iteration_node: BaseIterationNode, - workflow_run_state: WorkflowRunState, + workflow_run_state: WorkflowRunState, callbacks: list[BaseWorkflowCallback] = None) -> None: """ Workflow iteration next @@ -629,9 +628,9 @@ def _workflow_iteration_next(self, graph: dict, for node in nodes: workflow_run_state.variable_pool.clear_node_variables(node_id=node.get('id')) - + def _workflow_iteration_completed(self, current_iteration_node: BaseIterationNode, - workflow_run_state: WorkflowRunState, + workflow_run_state: WorkflowRunState, callbacks: list[BaseWorkflowCallback] = None) -> None: if callbacks: if isinstance(workflow_run_state.current_iteration_state, IterationState): @@ -684,7 +683,7 @@ def _get_next_overall_node(self, workflow_run_state: WorkflowRunState, callbacks=callbacks, workflow_call_depth=workflow_run_state.workflow_call_depth ) - + else: edges = graph.get('edges') source_node_id = predecessor_node.node_id @@ -738,9 +737,9 @@ def _get_next_overall_node(self, workflow_run_state: WorkflowRunState, callbacks=callbacks, workflow_call_depth=workflow_run_state.workflow_call_depth ) - - def _get_node(self, workflow_run_state: WorkflowRunState, - graph: dict, + + def _get_node(self, workflow_run_state: WorkflowRunState, + graph: dict, node_id: str, callbacks: list[BaseWorkflowCallback]) -> Optional[BaseNode]: """ @@ -940,7 +939,7 @@ def handle_special_values(cls, value: Optional[dict]) -> Optional[dict]: return new_value - def _mapping_user_inputs_to_variable_pool(self, + def _mapping_user_inputs_to_variable_pool(self, variable_mapping: dict, user_inputs: dict, variable_pool: VariablePool, @@ -988,4 +987,4 @@ def _mapping_user_inputs_to_variable_pool(self, node_id=variable_node_id, variable_key_list=variable_key_list, value=value - ) \ No newline at end of file + ) diff --git a/api/models/dataset.py b/api/models/dataset.py index b0e3702dd7d175..d0be005a154878 100644 --- a/api/models/dataset.py +++ b/api/models/dataset.py @@ -9,10 +9,10 @@ import time from json import JSONDecodeError -from flask import current_app from sqlalchemy import func from sqlalchemy.dialects.postgresql import JSONB +from configs import dify_config from core.rag.retrieval.retrival_methods import RetrievalMethod from extensions.ext_database import db from extensions.ext_storage import storage @@ -528,7 +528,7 @@ def get_sign_content(self): nonce = os.urandom(16).hex() timestamp = str(int(time.time())) data_to_sign = f"image-preview|{upload_file_id}|{timestamp}|{nonce}" - secret_key = current_app.config['SECRET_KEY'].encode() + secret_key = dify_config.SECRET_KEY.encode() if dify_config.SECRET_KEY else b'' sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest() encoded_sign = base64.urlsafe_b64encode(sign).decode() diff --git a/api/models/model.py b/api/models/model.py index 4d67272c1a393f..331bb91c29819f 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -4,10 +4,11 @@ from enum import Enum from typing import Optional -from flask import current_app, request +from flask import request from flask_login import UserMixin from sqlalchemy import Float, func, text +from configs import dify_config from core.file.tool_file_parser import ToolFileParser from core.file.upload_file_parser import UploadFileParser from extensions.ext_database import db @@ -111,7 +112,7 @@ def workflow(self) -> Optional['Workflow']: @property def api_base_url(self): - return (current_app.config['SERVICE_API_URL'] if current_app.config['SERVICE_API_URL'] + return (dify_config.SERVICE_API_URL if dify_config.SERVICE_API_URL else request.host_url.rstrip('/')) + '/v1' @property @@ -1113,7 +1114,7 @@ def generate_code(n): @property def app_base_url(self): return ( - current_app.config['APP_WEB_URL'] if current_app.config['APP_WEB_URL'] else request.host_url.rstrip('/')) + dify_config.APP_WEB_URL if dify_config.APP_WEB_URL else request.host_url.rstrip('/')) class ApiToken(db.Model): From f55876bcc588211f3064c2232259c66020034964 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Thu, 18 Jul 2024 01:14:36 +0800 Subject: [PATCH 20/62] fix web import url is too long (#6402) --- api/services/dataset_service.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 84049712d951e7..e7af975009e2a7 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -845,13 +845,17 @@ def save_document_with_dataset_id( 'only_main_content': website_info.get('only_main_content', False), 'mode': 'crawl', } + if url.length > 255: + document_name = url[:200] + '...' + else: + document_name = url document = DocumentService.build_document( dataset, dataset_process_rule.id, document_data["data_source"]["type"], document_data["doc_form"], document_data["doc_language"], data_source_info, created_from, position, - account, url, batch + account, document_name, batch ) db.session.add(document) db.session.flush() From 4782fb50c46eb96d83888accc4b959988aeee990 Mon Sep 17 00:00:00 2001 From: Richards Tu <142148415+richards199999@users.noreply.github.com> Date: Thu, 18 Jul 2024 07:47:06 +0800 Subject: [PATCH 21/62] Support new Claude-3.5 Sonnet max token limit (#6335) --- .../anthropic/llm/claude-3-5-sonnet-20240620.yaml | 4 ++-- .../model_runtime/model_providers/anthropic/llm/llm.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/api/core/model_runtime/model_providers/anthropic/llm/claude-3-5-sonnet-20240620.yaml b/api/core/model_runtime/model_providers/anthropic/llm/claude-3-5-sonnet-20240620.yaml index 72d4d8545b9723..e02c5517fe1f3c 100644 --- a/api/core/model_runtime/model_providers/anthropic/llm/claude-3-5-sonnet-20240620.yaml +++ b/api/core/model_runtime/model_providers/anthropic/llm/claude-3-5-sonnet-20240620.yaml @@ -27,9 +27,9 @@ parameter_rules: - name: max_tokens use_template: max_tokens required: true - default: 4096 + default: 8192 min: 1 - max: 4096 + max: 8192 - name: response_format use_template: response_format pricing: diff --git a/api/core/model_runtime/model_providers/anthropic/llm/llm.py b/api/core/model_runtime/model_providers/anthropic/llm/llm.py index fbc0b722b19da3..787c3baaaa9992 100644 --- a/api/core/model_runtime/model_providers/anthropic/llm/llm.py +++ b/api/core/model_runtime/model_providers/anthropic/llm/llm.py @@ -113,6 +113,11 @@ def _chat_generate(self, model: str, credentials: dict, if system: extra_model_kwargs['system'] = system + # Add the new header for claude-3-5-sonnet-20240620 model + headers = {} + if model == "claude-3-5-sonnet-20240620": + headers["anthropic-beta"] = "max-tokens-3-5-sonnet-2024-07-15" + if tools: extra_model_kwargs['tools'] = [ self._transform_tool_prompt(tool) for tool in tools @@ -121,6 +126,7 @@ def _chat_generate(self, model: str, credentials: dict, model=model, messages=prompt_message_dicts, stream=stream, + headers=headers, **model_parameters, **extra_model_kwargs ) @@ -130,6 +136,7 @@ def _chat_generate(self, model: str, credentials: dict, model=model, messages=prompt_message_dicts, stream=stream, + headers=headers, **model_parameters, **extra_model_kwargs ) @@ -138,7 +145,7 @@ def _chat_generate(self, model: str, credentials: dict, return self._handle_chat_generate_stream_response(model, credentials, response, prompt_messages) return self._handle_chat_generate_response(model, credentials, response, prompt_messages) - + def _code_block_mode_wrapper(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], model_parameters: dict, tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, stream: bool = True, user: Optional[str] = None, From 3b5b548af39186e8a27347e5725fd699b1f857bb Mon Sep 17 00:00:00 2001 From: forrestlinfeng Date: Thu, 18 Jul 2024 07:47:18 +0800 Subject: [PATCH 22/62] Add Stepfun LLM Support (#6346) --- .../model_providers/stepfun/__init__.py | 0 .../stepfun/_assets/icon_l_en.png | Bin 0 -> 9176 bytes .../stepfun/_assets/icon_s_en.png | Bin 0 -> 1991 bytes .../model_providers/stepfun/llm/__init__.py | 0 .../stepfun/llm/_position.yaml | 6 + .../model_providers/stepfun/llm/llm.py | 328 ++++++++++++++++++ .../stepfun/llm/step-1-128k.yaml | 25 ++ .../stepfun/llm/step-1-256k.yaml | 25 ++ .../stepfun/llm/step-1-32k.yaml | 28 ++ .../stepfun/llm/step-1-8k.yaml | 28 ++ .../stepfun/llm/step-1v-32k.yaml | 25 ++ .../stepfun/llm/step-1v-8k.yaml | 25 ++ .../model_providers/stepfun/stepfun.py | 30 ++ .../model_providers/stepfun/stepfun.yaml | 81 +++++ .../model_runtime/stepfun/__init__.py | 0 .../model_runtime/stepfun/test_llm.py | 176 ++++++++++ 16 files changed, 777 insertions(+) create mode 100644 api/core/model_runtime/model_providers/stepfun/__init__.py create mode 100644 api/core/model_runtime/model_providers/stepfun/_assets/icon_l_en.png create mode 100644 api/core/model_runtime/model_providers/stepfun/_assets/icon_s_en.png create mode 100644 api/core/model_runtime/model_providers/stepfun/llm/__init__.py create mode 100644 api/core/model_runtime/model_providers/stepfun/llm/_position.yaml create mode 100644 api/core/model_runtime/model_providers/stepfun/llm/llm.py create mode 100644 api/core/model_runtime/model_providers/stepfun/llm/step-1-128k.yaml create mode 100644 api/core/model_runtime/model_providers/stepfun/llm/step-1-256k.yaml create mode 100644 api/core/model_runtime/model_providers/stepfun/llm/step-1-32k.yaml create mode 100644 api/core/model_runtime/model_providers/stepfun/llm/step-1-8k.yaml create mode 100644 api/core/model_runtime/model_providers/stepfun/llm/step-1v-32k.yaml create mode 100644 api/core/model_runtime/model_providers/stepfun/llm/step-1v-8k.yaml create mode 100644 api/core/model_runtime/model_providers/stepfun/stepfun.py create mode 100644 api/core/model_runtime/model_providers/stepfun/stepfun.yaml create mode 100644 api/tests/integration_tests/model_runtime/stepfun/__init__.py create mode 100644 api/tests/integration_tests/model_runtime/stepfun/test_llm.py diff --git a/api/core/model_runtime/model_providers/stepfun/__init__.py b/api/core/model_runtime/model_providers/stepfun/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/core/model_runtime/model_providers/stepfun/_assets/icon_l_en.png b/api/core/model_runtime/model_providers/stepfun/_assets/icon_l_en.png new file mode 100644 index 0000000000000000000000000000000000000000..c118ea09bd8a84d7d43f03f6f5bc15c9a0c67e95 GIT binary patch literal 9176 zcmd^l`9IX}`|k+Zq9T-uWG%8YBC;=oLAGd$EW=E^?7Ol?A<7y}H1_P&7=)CvFWHwF z`@T)G@8*p6`}6*M&i8T7=llcb^~20-xvuAZEzkR!`+i*$X{fKoNY6nJ0)ZHz+UiCi z5G6o9Z=t0mf9kRB-zI-u@zgN)GUY!KW~z}f*}gs`^rb8AH?khN~0 zOw2vaVS4hmE`X@@sf?&E;7Zm8ffST{U9D{$5uV&O2nUq2BF|b)Jr6g^PLb!n3``v6 zs)j(KwEf)?#{T*yw*HQ`a&|mQcexdO<;eg5!qb}D7jSa+koQ&O`3qN`e13XcjEDO# z5l=@&p1+MUhZ%CKxws>^);^|=x%2(Z=|mIui?m7iabb9Pgi*{F&`fvQ6DK$7k39S2{}1AF>y&T zNl6j1gouZqv!}JMh_eUpKM3jw4_kMXt0&6Enfnyc+Q!ApQ;~;k?B8AhuK$L0_V`z% z$T1W1wRRPg5EVc5>K{QE?Efta0RNWu@H9gF$KL-du!o7CD?-c&;o;)tZcEOcJ@4sI zuJUT`2y0IlcM}&Er+)%zh;;FE@j$w`a!X2!igVw9S=*wVPw(9PO9cj#hdO(BT07e! zpz4Y|WDrpl%1&NFT|!MlT24wzO&lU2p`ocFqb?&SBP)4ZUFP;}Nfp(9bk$vKy#R!> z=Rdl3|IwBHuezs%09?r?>>#9K=jah6si&m3MST`8M*ggC=!7kT0nz||I=g!O=;%nmo5bZx zO8PXPUEUgyx!Bn^hHIK_>z}P@nXpdyy|l8r_Gd%ZX9-)NJsYNluB7he@je`&(Moh30lkcz(apS2>*F8;BJB|IM?F`-C0 z0Rw8G8$Y_H3%|{)gdYieZe@P%JvcZVn_9AqpQ&r?9~@t8>Y0T`Y~#o0TUK{hR)164 zkfvvsjG|Tp(EUC56%psTn@)?NS)+qvi@SUKJG*-d&(_b{k!Y<+VL8*wtLuJfLTq+B zr|lf*0ZH0#Yh!a;KXSt&cC~kCg2Q2p--86QBbi3d#pI8=CH4hk#(WYxKnA4IiMi={ z;*+GQ#`dA{+21xvekA$CGYU*8-3p|*Z znrE~j4Nnq-KMW*&UJ1?r9g#mbIkOa-*H8aw8Jec43vy>WK0&DL8l8DP>;MZy>K~@RHs3^uijYC8_k=Sl95PWzrrkiuAFfYh-ll z%tI2N`$2W{RC?v?2kgkm#=&{SA;^*hf|EdxNT7!#kQGV)S)aJeOk{e~4dmFb;RQzZ6x@ul=Ag()$4A-h!UWUBW%W(J=v)y{F7bfh*Lsmk;|_D52U?xyk_s3 za9K?0rot3@b-{e*JmUqDz5`p@I?FrwyLj$}+d@V=)B&1|p8rmCg$vr4Lz3cXRSD(Nh@uSrJ?;P%%|wmbha&UPTJfn$UqL zXbvPga~^+u^o!~|kS^@d_&Ddm)vU?DJiurK_pDU$EwN!>1;-WVWH5Snh~&ti(%a$G zh|2t=Z55>`rHs2bu6AJTLgMTm%q=ljFuxK<*EdFU<(#rscTT{2@8$@f%BwInYY}eXP78>6R`WFCH{#y0D@Q{s1v+Oo)v8J)Qn4p3Yc3 zwOMBj3!2__s;csxJ<-s8jAqn-3tl``ql_alzvFwg7a-_JhXuW8h1&kf)K5 zZ~Q1+oFAr`SY351>FWzxQp7%;@uE?=aPG(CW!L54kKpls&c$4vQ`APm^>O2LYj5a`J*?}vgot@1f-_K)n zs?6znww><981fHCWqxC9og1gU4l2C z{()uVWtLZu_te;EM1e$rj!cUEMBKwv_&~-)-Pgcl;gw%mRTm#oCPM_}=*e^7DHMGC zZZ?gHQT=-XbfGsYe%GdOEP0LcE$;i@{trjT8!r*>@w5!UKD@qu@8JT?EkK{}@YK)i z05qQeP{^Nm{-krB;cd4->FPjmPVp`3Q(P^u=?$+B1PgP4N}Qj5W>u;)6&G-+@l!NU z?yp`|aO<+D(H+3DC8p?`fzdmNW|*9*rPAI_a_I-`9b}#99@wMjREY zRng1M^iwWtFhibDdbb5@^Dkr$kaMkmJlekICP5-{S0p@YH^z@@UT?q4 z3!I3}Km{-(rNn_W+H7LNAJb2UnVWGUI?eWmkS2dsG}fa&4?Cci{PL za?FW|TqihC;EvL7>}Jv9w$`FgUN6&*)-i0Fl?05w#a8oG$}>khuYXRGvul2pa>Qc* zqbFi!RftE6|;@zK2^jhfp z-#UX2eS?$!*_XoVF{u>Ah;Qp{ytb7~y}3Td%7HaSZ@>LPho!y_yr#b(*jm@HrB5fR!WqhvrdEYowPLs>f)XB_)-?PBnGdAQ z4r`TVYJppjP!X$xdpoKfVUFRfYfh4IgnVq8pD4K3X3`D2X4*I(tMmt>GuhNDD~`M4PuX;k7etigjLN5 zw5xhKs4AB_ix63}yKF))TW?cf_ah+Z;kdjIve~)_Vl4gXrU0?<1Bzv z@`qAUao%lXSAongysK+dws$M5s5fHKR?Ak%FRLz;_;VTrr$82UZ|CEq`=Mc3!=IPA z{9_Cw(Sj0{M=%12oCXhF_$_AbXyPzE8Ut?Czkk^8;MhO46X2P8+&DEYZ^cLs0Rhx|ABL-VCT^@oUPXr{y*);1dO*qB z+B*#sQe>oqmL%6x_+KjBOQTZynKaGcD_$IZfa&zXZp)8O(8wQ%DWlKeHF-H>LS+~~YU z7f-%XY%q>$AU5(fQC$OAbmOBv&Bl3OFk$x}Dx;-=ma-W6UU4V1i4bx`Zwh~{&L9|k zoHvwo7{?`CPr9N4brjH3-Eh-J*r!;ewu&b1J?*y1Qo40#P=J&1Xvh8<>iBnAueM{ms&bvo%@?be|M(f* zvZFFv5oaB3KNPjF9Gu|6jyx%DrJwD}=4T>j)>w^jhZg9kw8XR&jA0Yf#+MNT=?)AB zYjf8IqmjJNNBVm&?2P`Dk1YH|r-coAs7@Av1RffIvyeRel(n@C=&eN*tbynP-L z+lJ#KBlhWozra0>JVz`-;x6?zff>Gb>--tcDKUPq(G`U4yS}Bi)a;+(o={Igu7uA} z49#67k(Ld*EaORiNf6PUiRp^jXCLq2Yp3S)Pfk~kTR9^8x~skujXE5ci?OIn-Sc#S zY3tW+A2UqnF}*VNAcNE#CGcGXKB0K=ZuhrqXMo=*LVO6sdCL_8y9o%)>(z;hM}@D2 z8x}Ay{>neBfZ!j?=DQ(h>{H8|fIHo}uW++>_!)!vmF^b+Ez#Y_Yd2u#y<6D|7?Ez& zOjmCdhb*SkFVZ*tB(M#Cd%Ml>pE0Pe5 zdy^fbcK#qI7f&uBTxC-S6@Xx>vdMRjDe2YdF}xHFnPSCV5ZU8+v{}wnS*r;Z;6gd-*$YA ze^Q@%vrv}w3P2zJsdxA?AUn~HKwUBl@4R=qP)UJ!K8U}1#T;85b&u3yy31zUs z{2ch$sMmO)!f+_6#9^?ONO;4iqxmu(+xTq5uCB>@_8LTeEqyop>z9PwM4Zya824~- z;Gw8d){D6-$&ibiL96p6?9|V0dcSfQT-83y1ay9BkUg_T_276e0wC*6W!FCkBZ{B41UlW{V6lBe-Z z!(r9!CUKH4ku}#PTWrqz+H}ZS=>JXoq10rtm~S~ic?)+m2dsfheZiiJwRngh@H#Sc^rfOq7+ znt#Q-&|senCp4=k**PncJw}D_y4LJQ5j#@t7p~Tj`4)EjjXMkW{osS{AR(1VEx-wYKJXNs$yi_k{hioV@d`_bUdX zp7M>0i~-$3;q-@5R=kV9qgZgu(Yu@HR0&J0z?y0-C0^kjKlA~@1kc>m%-CS}HubJv zl7uZbm&Kw{A0{c;T@JXZHu3mbNqC5u=0t zGM3Hi&cacIYa$A^D89%E=K>fElCF<27L0P;z>b_OY|5HG@u)q)J}RDa5NZ(80uyAF zkt^<0`K(C-in;s_ygeExP2pDo+2rZX+Sz4ga<&u}D64Y-Nu^4??8$={wep&^EZkUA z3=OA?d?c4dLrMSRRy12@k49)1PTxZH)=Rn&RX96uH`|MIuG-(@`QNq&O_Au&;itQE zV-&hrdHHoxner=V^`AfXe)_V<41NnQM|vGZg-`5@h{)j)n;zQSbMPpur#*-7&h1Ra z27_Y)-K~rLb-z9zZIHZ)Me2&NBBmRz1w-brrWvst7KL_N%0FKPV*sO@%Y_vgC!d|L z+MSA~aCgceSjf_kaAp9-%NF^?m&Io$Look6F1F=7#Q27`(xQA$Z4>x8?jn-n!W3hD zCvrwpt3bwXLn6>Qk)-jf?K~AevqL)o zw&r{!UBfJLr(!*LTvgz~s3teP0RF(#?pQ5MIB0PYd7u>Gh~Z3l{f#|??0TCzt4hzF^l}AHv`f;UrdEo&FxeIN35nF zar~a!qw5!TJINFgQp1W$;SzYP&f>FC1a!=O=JH&kxn=R+v8!RWvy!?v0m4aUngoKBGv=n=ygqI1FbVMHf8tTTCs{*6l0%(SFs{5?WiO4m`eIH?p=!0$4P?7%neKj+Ui=+)xkvT0Jc0o0WnP6Ls{9V7Isot#;$ z_7`HH`tK8}9SpnKlCCc+s={-%^H5ntFPD_-7f=NDRPF%@gJ{F6-o0Vj2ifg@t)U+Ba=!OMAJTtnK^+ z&gKDU=C_ocFFefwn?bf_+9Z9uN42x?pv%^u%29-9*k_|;zXq{68+1{fTEA621STJyZ?oUh)UNSMq_S#L({>UPVwaZAFE z1VCUw5SHIXpVGWY_vTZIx!8X7tIQMw?tS&xikbPQhbxBZ&xDB)+<2u?mZ z`{R5#YL!X$&u~lp`ggX;O<>jT{3_V^-LhE^a$GXY*tFG*YmFD*0I7O(WMhAy0uVx~ zsh;%a#>PW?#NE7>PI5Y-JoR79sRr?imsLH>x3(abcYoRtcgEa#x{9j;wTouY7fkm< zA-B=7*lm0k=Dyz1Wu)?GU>A!4Ot#Mc~hIgE(mF!&r4;&aVALh$C_Dk$X>Lk2a0Z zdd6|8usiG4b<400xLP{S3K8v6A{R=Uzt`ge0@_fHLf0ZH7$Xy4kJ0F_Z`@Zb+7BR-Mx86-@R{SOyut0f=Fl^R(0o~Nt zT3W*m11rXFvI1Rgn^Y<_oGP;$Y$+^%uFY!8GOQ2%!IkxzFt=eFuZgg%Xh_C-ZHK44 zcIo07q9B8`yYbukc*zpeRJ9ML{8J%$@qNWNbyp6&W3Q~OB+l0aOuE&|6k#>OMS;3Z zjlx@g_4$n-LcuHUVi=eNfU0q>`QWJb4x(dWU(uU<6nelv{Uy1CxJD&x$+X*-H#kzI z%b2w1{Ib`qqW6}(qN<+D&8;c(`;&vm;-`yeo#$tF?S0(FGl^S+VKuUr-H(svIqCLu zd`+(>nk&sHXwNdtBK~|}hGkDJyPW305%$(-5Wv91@S1`cDdx>u zEqE+>$NkRBoTtX%V|~!O3lM^RPVN%ljXWJ=Ut~|RlxZz{wbtVo1H;|L^5qQOx#xj( z-MDe;n%GS)pdwZ1Anp=0z5MBMPen@Zg{7U5P#HN7u{oTau&a+li4Ws-d;ycZI$)ru zblq34$LQ%IgcI5K0(VhcCi~JgiQ;gGV}vSR8`pJ7qI&HeKLDi`H5YnTmd>1=_i#)# z0;e{TIYy7?`+{e&b$57zfw2JhyDyJFpQitaqQ->X@wj&hR(8mHc8x=kts0R;>qIkP zq2)^>=r+p1f%TG!q}ltGyco6HI3hg;!Ykwft(sfvxp)06bBaqxzr|Q&^={$txo>Yu);F(oA?x* zF7mvhN_BII0A7c$LqkG(>hkTa#>8D>rfvIv=sk8hbFa^dY&S;>^z2_A)n-3~kHQO? zT+|!f6%~s1^&3eGSAZ_7`q84zOh#~>f)?J)!_C2YgBR0jj4yNjF?+A82HP058fcHi zJ$^Rz^0fm#$B32g(csc6-|d}?)vuAi_Mb=gh(4{X@N<;H8lTcEXLoKg4!V0Uptt>YX&)p4xSU;X*ZZSu#y`e1 zaXErJSuAVpg;J{0n5xCkI(~zcg0OdxwjRZix_q0g^%T(`ArK;#Som zsJB_x;CEGbZ-^%oxGeIwm5q(s0D+GLNK|dMXGcw}GW;hy`x$_%BYt$|_+??taju_i ze)7)CfxfgV7h^bdXTGZ2k&vP{$8~#ftTu;or-e0zf8B}tvDL~dczcd+rZ<@#x;*en zX3-|hfqJuyf!!Yfqb0P$GxRGP?RmV-WO#g`H=f0l3lB*aPfh$-MbUXfltYHeBe|* z)W8=>Rj>{L)~G-L)vxy0J*%8nCeG^8?<`;2zdlohpV%udgjF75oQBVI$zn%nd9{BU zNcLFkKRM8qw8A|Nv8s2kK8aT|K~!_Z49Te8N)}s73`k)2^n0<6T5&5fK*!!(yIS(@ z2X*j9gwyUoLa*Sfjd}svAxv?_2ZJYl3lmdAmHk$Mjv-a695MI!@kCS(u}00sS7m|_ zV*sX|F!@hEi`*&utGg8Pci)V>`1yV^iZZ?vD3M!(g z9aA4|@c~t-Xr+LIFs*jVIHfk!_j77|j*lXWU{tVH@N|PfTL=5McV_qAd*=JT^WA&S z?3Se@O%0-l(h&p+QY9!f)a>#;0lw6Gb7^J`HH{^e>0~ONP3CI}43Qad9fqn*+8j)S zX$_7At=MD)@nsp)(#dpnqC}6II9e};V>ekSHiAr!v0JqI`51}nupFaV%IrGW%tVa_ zDRX9|ny0qNv0P(90fD6!B&F#K=Ig}%(P9oiny(i`XKQsIq5_^kz~za!{3y0SAc=^S z2>Iy1U{VPQL$*Yth#yEt?WD|HlC(&;-2D7}PJRRjCvv!au~^LI3Ah3Qn?kUy4l}8> zv(479K?Mb7)e}YwX~fN_S5d3OZKRY*nSFi=lVwQOY#q2M>dv@!t%b|y@VuuQ1gh2l z4KaUo`iCvX$me`<+w7%$P0 zaS`!6xrove4)5hT*YHv=BIQfHa49ofZ*Bik>%kD!K;Gq0^HQ6k_Q%Xr&l6O0?|oDr zgdjc(QWDdY$ZUAp3-9ECa1r1;K)n{)J@DNrKye712e&%GF9SZX!sT{2aSfzP;l@4C zZw8+^u>UyJAA|4(;GYFC%VFATV3DwTKa|$PdjyD}-QD4CPdyVmrr7hS zo|WL&xasrLGj8v(`_Ng}*(-_N*OvNETt!=5%)FC@w_hg@A4+sz5IWAZJ!u**XvGEP zmks*fo`hEzXlya!LsNtrgqqGKyQ^^LC8fu?flkl^7{}Nw*L08b2>rnf)Q~$lrG0Lk zEHnA?#80!QhRbTl0-hLBb!J3}FRiMl^UK(f^=NV;8ZkY#!q#DXyj1V(Y<+#xt|FE! z%OUi9Pj~&;w{}|R35QKw`AsDwcyHL2mb%7b#K>~h&P4ZSF5TKguNvu!ac;}|ssdZH zP?kBS``)sQ6RzzNxf?3}s0gk8S$?27xaEfD`e+ZM!6q4SJ|f`Xi&1{FNCpw+-oe(l z%3KdCkGTuFf0Mbo3vjO6FOcR#dyy~@sgaeQ!@Ae3%u^oe3kjZ375BAftjD?Bm)4TO z$U0oz(D3$aGWuug=HsWM$1S@_0)yYh2qzU0;Q=F7NuA=l*z=c0&T?$P8|QZUKaM-E zJlgRlx@7!wZ%qFOCoek_I>vf9$z6DUea Union[LLMResult, Generator]: + self._add_custom_parameters(credentials) + self._add_function_call(model, credentials) + user = user[:32] if user else None + return super()._invoke(model, credentials, prompt_messages, model_parameters, tools, stop, stream, user) + + def validate_credentials(self, model: str, credentials: dict) -> None: + self._add_custom_parameters(credentials) + super().validate_credentials(model, credentials) + + def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity | None: + return AIModelEntity( + model=model, + label=I18nObject(en_US=model, zh_Hans=model), + model_type=ModelType.LLM, + features=[ModelFeature.TOOL_CALL, ModelFeature.MULTI_TOOL_CALL, ModelFeature.STREAM_TOOL_CALL] + if credentials.get('function_calling_type') == 'tool_call' + else [], + fetch_from=FetchFrom.CUSTOMIZABLE_MODEL, + model_properties={ + ModelPropertyKey.CONTEXT_SIZE: int(credentials.get('context_size', 8000)), + ModelPropertyKey.MODE: LLMMode.CHAT.value, + }, + parameter_rules=[ + ParameterRule( + name='temperature', + use_template='temperature', + label=I18nObject(en_US='Temperature', zh_Hans='温度'), + type=ParameterType.FLOAT, + ), + ParameterRule( + name='max_tokens', + use_template='max_tokens', + default=512, + min=1, + max=int(credentials.get('max_tokens', 1024)), + label=I18nObject(en_US='Max Tokens', zh_Hans='最大标记'), + type=ParameterType.INT, + ), + ParameterRule( + name='top_p', + use_template='top_p', + label=I18nObject(en_US='Top P', zh_Hans='Top P'), + type=ParameterType.FLOAT, + ), + ] + ) + + def _add_custom_parameters(self, credentials: dict) -> None: + credentials['mode'] = 'chat' + credentials['endpoint_url'] = 'https://api.stepfun.com/v1' + + def _add_function_call(self, model: str, credentials: dict) -> None: + model_schema = self.get_model_schema(model, credentials) + if model_schema and { + ModelFeature.TOOL_CALL, ModelFeature.MULTI_TOOL_CALL + }.intersection(model_schema.features or []): + credentials['function_calling_type'] = 'tool_call' + + def _convert_prompt_message_to_dict(self, message: PromptMessage,credentials: Optional[dict] = None) -> dict: + """ + Convert PromptMessage to dict for OpenAI API format + """ + if isinstance(message, UserPromptMessage): + message = cast(UserPromptMessage, message) + if isinstance(message.content, str): + message_dict = {"role": "user", "content": message.content} + else: + sub_messages = [] + for message_content in message.content: + if message_content.type == PromptMessageContentType.TEXT: + message_content = cast(PromptMessageContent, message_content) + sub_message_dict = { + "type": "text", + "text": message_content.data + } + sub_messages.append(sub_message_dict) + elif message_content.type == PromptMessageContentType.IMAGE: + message_content = cast(ImagePromptMessageContent, message_content) + sub_message_dict = { + "type": "image_url", + "image_url": { + "url": message_content.data, + } + } + sub_messages.append(sub_message_dict) + message_dict = {"role": "user", "content": sub_messages} + elif isinstance(message, AssistantPromptMessage): + message = cast(AssistantPromptMessage, message) + message_dict = {"role": "assistant", "content": message.content} + if message.tool_calls: + message_dict["tool_calls"] = [] + for function_call in message.tool_calls: + message_dict["tool_calls"].append({ + "id": function_call.id, + "type": function_call.type, + "function": { + "name": function_call.function.name, + "arguments": function_call.function.arguments + } + }) + elif isinstance(message, ToolPromptMessage): + message = cast(ToolPromptMessage, message) + message_dict = {"role": "tool", "content": message.content, "tool_call_id": message.tool_call_id} + elif isinstance(message, SystemPromptMessage): + message = cast(SystemPromptMessage, message) + message_dict = {"role": "system", "content": message.content} + else: + raise ValueError(f"Got unknown type {message}") + + if message.name: + message_dict["name"] = message.name + + return message_dict + + def _extract_response_tool_calls(self, response_tool_calls: list[dict]) -> list[AssistantPromptMessage.ToolCall]: + """ + Extract tool calls from response + + :param response_tool_calls: response tool calls + :return: list of tool calls + """ + tool_calls = [] + if response_tool_calls: + for response_tool_call in response_tool_calls: + function = AssistantPromptMessage.ToolCall.ToolCallFunction( + name=response_tool_call["function"]["name"] if response_tool_call.get("function", {}).get("name") else "", + arguments=response_tool_call["function"]["arguments"] if response_tool_call.get("function", {}).get("arguments") else "" + ) + + tool_call = AssistantPromptMessage.ToolCall( + id=response_tool_call["id"] if response_tool_call.get("id") else "", + type=response_tool_call["type"] if response_tool_call.get("type") else "", + function=function + ) + tool_calls.append(tool_call) + + return tool_calls + + def _handle_generate_stream_response(self, model: str, credentials: dict, response: requests.Response, + prompt_messages: list[PromptMessage]) -> Generator: + """ + Handle llm stream response + + :param model: model name + :param credentials: model credentials + :param response: streamed response + :param prompt_messages: prompt messages + :return: llm response chunk generator + """ + full_assistant_content = '' + chunk_index = 0 + + def create_final_llm_result_chunk(index: int, message: AssistantPromptMessage, finish_reason: str) \ + -> LLMResultChunk: + # calculate num tokens + prompt_tokens = self._num_tokens_from_string(model, prompt_messages[0].content) + completion_tokens = self._num_tokens_from_string(model, full_assistant_content) + + # transform usage + usage = self._calc_response_usage(model, credentials, prompt_tokens, completion_tokens) + + return LLMResultChunk( + model=model, + prompt_messages=prompt_messages, + delta=LLMResultChunkDelta( + index=index, + message=message, + finish_reason=finish_reason, + usage=usage + ) + ) + + tools_calls: list[AssistantPromptMessage.ToolCall] = [] + finish_reason = "Unknown" + + def increase_tool_call(new_tool_calls: list[AssistantPromptMessage.ToolCall]): + def get_tool_call(tool_name: str): + if not tool_name: + return tools_calls[-1] + + tool_call = next((tool_call for tool_call in tools_calls if tool_call.function.name == tool_name), None) + if tool_call is None: + tool_call = AssistantPromptMessage.ToolCall( + id='', + type='', + function=AssistantPromptMessage.ToolCall.ToolCallFunction(name=tool_name, arguments="") + ) + tools_calls.append(tool_call) + + return tool_call + + for new_tool_call in new_tool_calls: + # get tool call + tool_call = get_tool_call(new_tool_call.function.name) + # update tool call + if new_tool_call.id: + tool_call.id = new_tool_call.id + if new_tool_call.type: + tool_call.type = new_tool_call.type + if new_tool_call.function.name: + tool_call.function.name = new_tool_call.function.name + if new_tool_call.function.arguments: + tool_call.function.arguments += new_tool_call.function.arguments + + for chunk in response.iter_lines(decode_unicode=True, delimiter="\n\n"): + if chunk: + # ignore sse comments + if chunk.startswith(':'): + continue + decoded_chunk = chunk.strip().lstrip('data: ').lstrip() + chunk_json = None + try: + chunk_json = json.loads(decoded_chunk) + # stream ended + except json.JSONDecodeError as e: + yield create_final_llm_result_chunk( + index=chunk_index + 1, + message=AssistantPromptMessage(content=""), + finish_reason="Non-JSON encountered." + ) + break + if not chunk_json or len(chunk_json['choices']) == 0: + continue + + choice = chunk_json['choices'][0] + finish_reason = chunk_json['choices'][0].get('finish_reason') + chunk_index += 1 + + if 'delta' in choice: + delta = choice['delta'] + delta_content = delta.get('content') + + assistant_message_tool_calls = delta.get('tool_calls', None) + # assistant_message_function_call = delta.delta.function_call + + # extract tool calls from response + if assistant_message_tool_calls: + tool_calls = self._extract_response_tool_calls(assistant_message_tool_calls) + increase_tool_call(tool_calls) + + if delta_content is None or delta_content == '': + continue + + # transform assistant message to prompt message + assistant_prompt_message = AssistantPromptMessage( + content=delta_content, + tool_calls=tool_calls if assistant_message_tool_calls else [] + ) + + full_assistant_content += delta_content + elif 'text' in choice: + choice_text = choice.get('text', '') + if choice_text == '': + continue + + # transform assistant message to prompt message + assistant_prompt_message = AssistantPromptMessage(content=choice_text) + full_assistant_content += choice_text + else: + continue + + # check payload indicator for completion + yield LLMResultChunk( + model=model, + prompt_messages=prompt_messages, + delta=LLMResultChunkDelta( + index=chunk_index, + message=assistant_prompt_message, + ) + ) + + chunk_index += 1 + + if tools_calls: + yield LLMResultChunk( + model=model, + prompt_messages=prompt_messages, + delta=LLMResultChunkDelta( + index=chunk_index, + message=AssistantPromptMessage( + tool_calls=tools_calls, + content="" + ), + ) + ) + + yield create_final_llm_result_chunk( + index=chunk_index, + message=AssistantPromptMessage(content=""), + finish_reason=finish_reason + ) \ No newline at end of file diff --git a/api/core/model_runtime/model_providers/stepfun/llm/step-1-128k.yaml b/api/core/model_runtime/model_providers/stepfun/llm/step-1-128k.yaml new file mode 100644 index 00000000000000..13f7b7fd264aa8 --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/llm/step-1-128k.yaml @@ -0,0 +1,25 @@ +model: step-1-128k +label: + zh_Hans: step-1-128k + en_US: step-1-128k +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 128000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 1 + max: 128000 +pricing: + input: '0.04' + output: '0.20' + unit: '0.001' + currency: RMB diff --git a/api/core/model_runtime/model_providers/stepfun/llm/step-1-256k.yaml b/api/core/model_runtime/model_providers/stepfun/llm/step-1-256k.yaml new file mode 100644 index 00000000000000..f80ec9851c451d --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/llm/step-1-256k.yaml @@ -0,0 +1,25 @@ +model: step-1-256k +label: + zh_Hans: step-1-256k + en_US: step-1-256k +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 256000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 1 + max: 256000 +pricing: + input: '0.095' + output: '0.300' + unit: '0.001' + currency: RMB diff --git a/api/core/model_runtime/model_providers/stepfun/llm/step-1-32k.yaml b/api/core/model_runtime/model_providers/stepfun/llm/step-1-32k.yaml new file mode 100644 index 00000000000000..96132d14a8251f --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/llm/step-1-32k.yaml @@ -0,0 +1,28 @@ +model: step-1-32k +label: + zh_Hans: step-1-32k + en_US: step-1-32k +model_type: llm +features: + - agent-thought + - tool-call + - multi-tool-call + - stream-tool-call +model_properties: + mode: chat + context_size: 32000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 1 + max: 32000 +pricing: + input: '0.015' + output: '0.070' + unit: '0.001' + currency: RMB diff --git a/api/core/model_runtime/model_providers/stepfun/llm/step-1-8k.yaml b/api/core/model_runtime/model_providers/stepfun/llm/step-1-8k.yaml new file mode 100644 index 00000000000000..4a4ba8d17825d3 --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/llm/step-1-8k.yaml @@ -0,0 +1,28 @@ +model: step-1-8k +label: + zh_Hans: step-1-8k + en_US: step-1-8k +model_type: llm +features: + - agent-thought + - tool-call + - multi-tool-call + - stream-tool-call +model_properties: + mode: chat + context_size: 8000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 512 + min: 1 + max: 8000 +pricing: + input: '0.005' + output: '0.020' + unit: '0.001' + currency: RMB diff --git a/api/core/model_runtime/model_providers/stepfun/llm/step-1v-32k.yaml b/api/core/model_runtime/model_providers/stepfun/llm/step-1v-32k.yaml new file mode 100644 index 00000000000000..f878ee3e56e1f9 --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/llm/step-1v-32k.yaml @@ -0,0 +1,25 @@ +model: step-1v-32k +label: + zh_Hans: step-1v-32k + en_US: step-1v-32k +model_type: llm +features: + - vision +model_properties: + mode: chat + context_size: 32000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 1 + max: 32000 +pricing: + input: '0.015' + output: '0.070' + unit: '0.001' + currency: RMB diff --git a/api/core/model_runtime/model_providers/stepfun/llm/step-1v-8k.yaml b/api/core/model_runtime/model_providers/stepfun/llm/step-1v-8k.yaml new file mode 100644 index 00000000000000..6c3cb61d2c6621 --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/llm/step-1v-8k.yaml @@ -0,0 +1,25 @@ +model: step-1v-8k +label: + zh_Hans: step-1v-8k + en_US: step-1v-8k +model_type: llm +features: + - vision +model_properties: + mode: chat + context_size: 8192 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 512 + min: 1 + max: 8192 +pricing: + input: '0.005' + output: '0.020' + unit: '0.001' + currency: RMB diff --git a/api/core/model_runtime/model_providers/stepfun/stepfun.py b/api/core/model_runtime/model_providers/stepfun/stepfun.py new file mode 100644 index 00000000000000..50b17392b54276 --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/stepfun.py @@ -0,0 +1,30 @@ +import logging + +from core.model_runtime.entities.model_entities import ModelType +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.__base.model_provider import ModelProvider + +logger = logging.getLogger(__name__) + + +class StepfunProvider(ModelProvider): + + def validate_provider_credentials(self, credentials: dict) -> None: + """ + Validate provider credentials + if validate failed, raise exception + + :param credentials: provider credentials, credentials form defined in `provider_credential_schema`. + """ + try: + model_instance = self.get_model_instance(ModelType.LLM) + + model_instance.validate_credentials( + model='step-1-8k', + credentials=credentials + ) + except CredentialsValidateFailedError as ex: + raise ex + except Exception as ex: + logger.exception(f'{self.get_provider_schema().provider} credentials validate failed') + raise ex diff --git a/api/core/model_runtime/model_providers/stepfun/stepfun.yaml b/api/core/model_runtime/model_providers/stepfun/stepfun.yaml new file mode 100644 index 00000000000000..ccc8455adcf3ca --- /dev/null +++ b/api/core/model_runtime/model_providers/stepfun/stepfun.yaml @@ -0,0 +1,81 @@ +provider: stepfun +label: + zh_Hans: 阶跃星辰 + en_US: Stepfun +description: + en_US: Models provided by stepfun, such as step-1-8k, step-1-32k、step-1v-8k、step-1v-32k, step-1-128k and step-1-256k + zh_Hans: 阶跃星辰提供的模型，例如 step-1-8k、step-1-32k、step-1v-8k、step-1v-32k、step-1-128k 和 step-1-256k。 +icon_small: + en_US: icon_s_en.png +icon_large: + en_US: icon_l_en.png +background: "#FFFFFF" +help: + title: + en_US: Get your API Key from stepfun + zh_Hans: 从 stepfun 获取 API Key + url: + en_US: https://platform.stepfun.com/interface-key +supported_model_types: + - llm +configurate_methods: + - predefined-model + - customizable-model +provider_credential_schema: + credential_form_schemas: + - variable: api_key + label: + en_US: API Key + type: secret-input + required: true + placeholder: + zh_Hans: 在此输入您的 API Key + en_US: Enter your API Key +model_credential_schema: + model: + label: + en_US: Model Name + zh_Hans: 模型名称 + placeholder: + en_US: Enter your model name + zh_Hans: 输入模型名称 + credential_form_schemas: + - variable: api_key + label: + en_US: API Key + type: secret-input + required: true + placeholder: + zh_Hans: 在此输入您的 API Key + en_US: Enter your API Key + - variable: context_size + label: + zh_Hans: 模型上下文长度 + en_US: Model context size + required: true + type: text-input + default: '8192' + placeholder: + zh_Hans: 在此输入您的模型上下文长度 + en_US: Enter your Model context size + - variable: max_tokens + label: + zh_Hans: 最大 token 上限 + en_US: Upper bound for max tokens + default: '8192' + type: text-input + - variable: function_calling_type + label: + en_US: Function calling + type: select + required: false + default: no_call + options: + - value: no_call + label: + en_US: Not supported + zh_Hans: 不支持 + - value: tool_call + label: + en_US: Tool Call + zh_Hans: Tool Call diff --git a/api/tests/integration_tests/model_runtime/stepfun/__init__.py b/api/tests/integration_tests/model_runtime/stepfun/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/tests/integration_tests/model_runtime/stepfun/test_llm.py b/api/tests/integration_tests/model_runtime/stepfun/test_llm.py new file mode 100644 index 00000000000000..d703147d638be3 --- /dev/null +++ b/api/tests/integration_tests/model_runtime/stepfun/test_llm.py @@ -0,0 +1,176 @@ +import os +from collections.abc import Generator + +import pytest + +from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta +from core.model_runtime.entities.message_entities import ( + AssistantPromptMessage, + ImagePromptMessageContent, + PromptMessageTool, + SystemPromptMessage, + TextPromptMessageContent, + UserPromptMessage, +) +from core.model_runtime.entities.model_entities import AIModelEntity, ModelType +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.stepfun.llm.llm import StepfunLargeLanguageModel + + +def test_validate_credentials(): + model = StepfunLargeLanguageModel() + + with pytest.raises(CredentialsValidateFailedError): + model.validate_credentials( + model='step-1-8k', + credentials={ + 'api_key': 'invalid_key' + } + ) + + model.validate_credentials( + model='step-1-8k', + credentials={ + 'api_key': os.environ.get('STEPFUN_API_KEY') + } + ) + +def test_invoke_model(): + model = StepfunLargeLanguageModel() + + response = model.invoke( + model='step-1-8k', + credentials={ + 'api_key': os.environ.get('STEPFUN_API_KEY') + }, + prompt_messages=[ + UserPromptMessage( + content='Hello World!' + ) + ], + model_parameters={ + 'temperature': 0.9, + 'top_p': 0.7 + }, + stop=['Hi'], + stream=False, + user="abc-123" + ) + + assert isinstance(response, LLMResult) + assert len(response.message.content) > 0 + + +def test_invoke_stream_model(): + model = StepfunLargeLanguageModel() + + response = model.invoke( + model='step-1-8k', + credentials={ + 'api_key': os.environ.get('STEPFUN_API_KEY') + }, + prompt_messages=[ + SystemPromptMessage( + content='You are a helpful AI assistant.', + ), + UserPromptMessage( + content='Hello World!' + ) + ], + model_parameters={ + 'temperature': 0.9, + 'top_p': 0.7 + }, + stream=True, + user="abc-123" + ) + + assert isinstance(response, Generator) + + for chunk in response: + assert isinstance(chunk, LLMResultChunk) + assert isinstance(chunk.delta, LLMResultChunkDelta) + assert isinstance(chunk.delta.message, AssistantPromptMessage) + assert len(chunk.delta.message.content) > 0 if chunk.delta.finish_reason is None else True + + +def test_get_customizable_model_schema(): + model = StepfunLargeLanguageModel() + + schema = model.get_customizable_model_schema( + model='step-1-8k', + credentials={ + 'api_key': os.environ.get('STEPFUN_API_KEY') + } + ) + assert isinstance(schema, AIModelEntity) + + +def test_invoke_chat_model_with_tools(): + model = StepfunLargeLanguageModel() + + result = model.invoke( + model='step-1-8k', + credentials={ + 'api_key': os.environ.get('STEPFUN_API_KEY') + }, + prompt_messages=[ + SystemPromptMessage( + content='You are a helpful AI assistant.', + ), + UserPromptMessage( + content="what's the weather today in Shanghai?", + ) + ], + model_parameters={ + 'temperature': 0.9, + 'max_tokens': 100 + }, + tools=[ + PromptMessageTool( + name='get_weather', + description='Determine weather in my location', + parameters={ + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": [ + "c", + "f" + ] + } + }, + "required": [ + "location" + ] + } + ), + PromptMessageTool( + name='get_stock_price', + description='Get the current stock price', + parameters={ + "type": "object", + "properties": { + "symbol": { + "type": "string", + "description": "The stock symbol" + } + }, + "required": [ + "symbol" + ] + } + ) + ], + stream=False, + user="abc-123" + ) + + assert isinstance(result, LLMResult) + assert isinstance(result.message, AssistantPromptMessage) + assert len(result.message.tool_calls) > 0 \ No newline at end of file From 5236cb1888abc676f4079b168cd19e1545294398 Mon Sep 17 00:00:00 2001 From: Masashi Tomooka Date: Thu, 18 Jul 2024 08:50:54 +0900 Subject: [PATCH 23/62] fix: kill signal is not passed to the main process (#6159) --- api/docker/entrypoint.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/docker/entrypoint.sh b/api/docker/entrypoint.sh index e74c6c2406d499..a53d84c6e9e433 100755 --- a/api/docker/entrypoint.sh +++ b/api/docker/entrypoint.sh @@ -8,15 +8,15 @@ if [[ "${MIGRATION_ENABLED}" == "true" ]]; then fi if [[ "${MODE}" == "worker" ]]; then - celery -A app.celery worker -P ${CELERY_WORKER_CLASS:-gevent} -c ${CELERY_WORKER_AMOUNT:-1} --loglevel INFO \ + exec celery -A app.celery worker -P ${CELERY_WORKER_CLASS:-gevent} -c ${CELERY_WORKER_AMOUNT:-1} --loglevel INFO \ -Q ${CELERY_QUEUES:-dataset,generation,mail,ops_trace,app_deletion} elif [[ "${MODE}" == "beat" ]]; then - celery -A app.celery beat --loglevel INFO + exec celery -A app.celery beat --loglevel INFO else if [[ "${DEBUG}" == "true" ]]; then - flask run --host=${DIFY_BIND_ADDRESS:-0.0.0.0} --port=${DIFY_PORT:-5001} --debug + exec flask run --host=${DIFY_BIND_ADDRESS:-0.0.0.0} --port=${DIFY_PORT:-5001} --debug else - gunicorn \ + exec gunicorn \ --bind "${DIFY_BIND_ADDRESS:-0.0.0.0}:${DIFY_PORT:-5001}" \ --workers ${SERVER_WORKER_AMOUNT:-1} \ --worker-class ${SERVER_WORKER_CLASS:-gevent} \ @@ -24,4 +24,4 @@ else --preload \ app:app fi -fi \ No newline at end of file +fi From 287b42997da3599f68804a2d1ea9e2aff75a44a2 Mon Sep 17 00:00:00 2001 From: Harry Wang Date: Wed, 17 Jul 2024 20:37:16 -0400 Subject: [PATCH 24/62] fix inconsistent label (#6404) --- web/i18n/en-US/app-debug.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/i18n/en-US/app-debug.ts b/web/i18n/en-US/app-debug.ts index 2f130c049af889..a148a5b39591c7 100644 --- a/web/i18n/en-US/app-debug.ts +++ b/web/i18n/en-US/app-debug.ts @@ -51,7 +51,7 @@ const translation = { title: 'Experience enhance', }, conversationOpener: { - title: 'Conversation remakers', + title: 'Conversation Opener', description: 'In a chat app, the first sentence that the AI actively speaks to the user is usually used as a welcome.', }, suggestedQuestionsAfterAnswer: { From 23e5eeec00c5c085b8d64351db06cb9d4e92f4ee Mon Sep 17 00:00:00 2001 From: Xiao Ley Date: Thu, 18 Jul 2024 08:43:14 +0800 Subject: [PATCH 25/62] feat: added custom secure_ascii to the json_process tool (#6401) --- .../builtin/json_process/tools/delete.py | 23 ++++----- .../builtin/json_process/tools/delete.yaml | 12 +++++ .../builtin/json_process/tools/insert.py | 26 +++++----- .../builtin/json_process/tools/insert.yaml | 12 +++++ .../builtin/json_process/tools/parse.py | 15 +++--- .../builtin/json_process/tools/parse.yaml | 12 +++++ .../builtin/json_process/tools/replace.py | 47 ++++++++++--------- .../builtin/json_process/tools/replace.yaml | 12 +++++ 8 files changed, 105 insertions(+), 54 deletions(-) diff --git a/api/core/tools/provider/builtin/json_process/tools/delete.py b/api/core/tools/provider/builtin/json_process/tools/delete.py index b09e4948813f5b..1b49cfe2f300f8 100644 --- a/api/core/tools/provider/builtin/json_process/tools/delete.py +++ b/api/core/tools/provider/builtin/json_process/tools/delete.py @@ -19,28 +19,29 @@ def _invoke(self, content = tool_parameters.get('content', '') if not content: return self.create_text_message('Invalid parameter content') - + # Get query query = tool_parameters.get('query', '') if not query: return self.create_text_message('Invalid parameter query') - + + ensure_ascii = tool_parameters.get('ensure_ascii', True) try: - result = self._delete(content, query) + result = self._delete(content, query, ensure_ascii) return self.create_text_message(str(result)) except Exception as e: return self.create_text_message(f'Failed to delete JSON content: {str(e)}') - def _delete(self, origin_json: str, query: str) -> str: + def _delete(self, origin_json: str, query: str, ensure_ascii: bool) -> str: try: input_data = json.loads(origin_json) expr = parse('$.' + query.lstrip('$.')) # Ensure query path starts with $ - + matches = expr.find(input_data) - + if not matches: - return json.dumps(input_data, ensure_ascii=True) # No changes if no matches found - + return json.dumps(input_data, ensure_ascii=ensure_ascii) # No changes if no matches found + for match in matches: if isinstance(match.context.value, dict): # Delete key from dictionary @@ -53,7 +54,7 @@ def _delete(self, origin_json: str, query: str) -> str: parent = match.context.parent if parent: del parent.value[match.path.fields[-1]] - - return json.dumps(input_data, ensure_ascii=True) + + return json.dumps(input_data, ensure_ascii=ensure_ascii) except Exception as e: - raise Exception(f"Delete operation failed: {str(e)}") \ No newline at end of file + raise Exception(f"Delete operation failed: {str(e)}") diff --git a/api/core/tools/provider/builtin/json_process/tools/delete.yaml b/api/core/tools/provider/builtin/json_process/tools/delete.yaml index 4cfa90b861d48f..4d390e40d17232 100644 --- a/api/core/tools/provider/builtin/json_process/tools/delete.yaml +++ b/api/core/tools/provider/builtin/json_process/tools/delete.yaml @@ -38,3 +38,15 @@ parameters: pt_BR: JSONPath query to locate the element to delete llm_description: JSONPath query to locate the element to delete form: llm + - name: ensure_ascii + type: boolean + default: true + label: + en_US: Ensure ASCII + zh_Hans: 确保 ASCII + pt_BR: Ensure ASCII + human_description: + en_US: Ensure the JSON output is ASCII encoded + zh_Hans: 确保输出的 JSON 是 ASCII 编码 + pt_BR: Ensure the JSON output is ASCII encoded + form: form diff --git a/api/core/tools/provider/builtin/json_process/tools/insert.py b/api/core/tools/provider/builtin/json_process/tools/insert.py index aa5986e2b45860..27e34f1ff3fda9 100644 --- a/api/core/tools/provider/builtin/json_process/tools/insert.py +++ b/api/core/tools/provider/builtin/json_process/tools/insert.py @@ -19,31 +19,31 @@ def _invoke(self, content = tool_parameters.get('content', '') if not content: return self.create_text_message('Invalid parameter content') - + # get query query = tool_parameters.get('query', '') if not query: return self.create_text_message('Invalid parameter query') - + # get new value new_value = tool_parameters.get('new_value', '') if not new_value: return self.create_text_message('Invalid parameter new_value') - + # get insert position index = tool_parameters.get('index') - + # get create path create_path = tool_parameters.get('create_path', False) - + + ensure_ascii = tool_parameters.get('ensure_ascii', True) try: - result = self._insert(content, query, new_value, index, create_path) + result = self._insert(content, query, new_value, ensure_ascii, index, create_path) return self.create_text_message(str(result)) except Exception: return self.create_text_message('Failed to insert JSON content') - - def _insert(self, origin_json, query, new_value, index=None, create_path=False): + def _insert(self, origin_json, query, new_value, ensure_ascii: bool, index=None, create_path=False): try: input_data = json.loads(origin_json) expr = parse(query) @@ -51,9 +51,9 @@ def _insert(self, origin_json, query, new_value, index=None, create_path=False): new_value = json.loads(new_value) except json.JSONDecodeError: new_value = new_value - + matches = expr.find(input_data) - + if not matches and create_path: # create new path path_parts = query.strip('$').strip('.').split('.') @@ -91,7 +91,7 @@ def _insert(self, origin_json, query, new_value, index=None, create_path=False): else: # replace old value with new value match.full_path.update(input_data, new_value) - - return json.dumps(input_data, ensure_ascii=True) + + return json.dumps(input_data, ensure_ascii=ensure_ascii) except Exception as e: - return str(e) \ No newline at end of file + return str(e) diff --git a/api/core/tools/provider/builtin/json_process/tools/insert.yaml b/api/core/tools/provider/builtin/json_process/tools/insert.yaml index 66a6ff99291cdb..63e7816455cb68 100644 --- a/api/core/tools/provider/builtin/json_process/tools/insert.yaml +++ b/api/core/tools/provider/builtin/json_process/tools/insert.yaml @@ -75,3 +75,15 @@ parameters: zh_Hans: 否 pt_BR: "No" form: form + - name: ensure_ascii + type: boolean + default: true + label: + en_US: Ensure ASCII + zh_Hans: 确保 ASCII + pt_BR: Ensure ASCII + human_description: + en_US: Ensure the JSON output is ASCII encoded + zh_Hans: 确保输出的 JSON 是 ASCII 编码 + pt_BR: Ensure the JSON output is ASCII encoded + form: form diff --git a/api/core/tools/provider/builtin/json_process/tools/parse.py b/api/core/tools/provider/builtin/json_process/tools/parse.py index b246afc07e8b12..ecd39113ae5498 100644 --- a/api/core/tools/provider/builtin/json_process/tools/parse.py +++ b/api/core/tools/provider/builtin/json_process/tools/parse.py @@ -19,33 +19,34 @@ def _invoke(self, content = tool_parameters.get('content', '') if not content: return self.create_text_message('Invalid parameter content') - + # get json filter json_filter = tool_parameters.get('json_filter', '') if not json_filter: return self.create_text_message('Invalid parameter json_filter') + ensure_ascii = tool_parameters.get('ensure_ascii', True) try: - result = self._extract(content, json_filter) + result = self._extract(content, json_filter, ensure_ascii) return self.create_text_message(str(result)) except Exception: return self.create_text_message('Failed to extract JSON content') # Extract data from JSON content - def _extract(self, content: str, json_filter: str) -> str: + def _extract(self, content: str, json_filter: str, ensure_ascii: bool) -> str: try: input_data = json.loads(content) expr = parse(json_filter) result = [match.value for match in expr.find(input_data)] - + if len(result) == 1: result = result[0] - + if isinstance(result, dict | list): - return json.dumps(result, ensure_ascii=True) + return json.dumps(result, ensure_ascii=ensure_ascii) elif isinstance(result, str | int | float | bool) or result is None: return str(result) else: return repr(result) except Exception as e: - return str(e) \ No newline at end of file + return str(e) diff --git a/api/core/tools/provider/builtin/json_process/tools/parse.yaml b/api/core/tools/provider/builtin/json_process/tools/parse.yaml index b619dcde94171d..c35f4eac0775ad 100644 --- a/api/core/tools/provider/builtin/json_process/tools/parse.yaml +++ b/api/core/tools/provider/builtin/json_process/tools/parse.yaml @@ -38,3 +38,15 @@ parameters: pt_BR: JSON fields to be parsed llm_description: JSON fields to be parsed form: llm + - name: ensure_ascii + type: boolean + default: true + label: + en_US: Ensure ASCII + zh_Hans: 确保 ASCII + pt_BR: Ensure ASCII + human_description: + en_US: Ensure the JSON output is ASCII encoded + zh_Hans: 确保输出的 JSON 是 ASCII 编码 + pt_BR: Ensure the JSON output is ASCII encoded + form: form diff --git a/api/core/tools/provider/builtin/json_process/tools/replace.py b/api/core/tools/provider/builtin/json_process/tools/replace.py index 9f127b9d067319..be696bce0e0a2c 100644 --- a/api/core/tools/provider/builtin/json_process/tools/replace.py +++ b/api/core/tools/provider/builtin/json_process/tools/replace.py @@ -19,61 +19,62 @@ def _invoke(self, content = tool_parameters.get('content', '') if not content: return self.create_text_message('Invalid parameter content') - + # get query query = tool_parameters.get('query', '') if not query: return self.create_text_message('Invalid parameter query') - + # get replace value replace_value = tool_parameters.get('replace_value', '') if not replace_value: return self.create_text_message('Invalid parameter replace_value') - + # get replace model replace_model = tool_parameters.get('replace_model', '') if not replace_model: return self.create_text_message('Invalid parameter replace_model') + ensure_ascii = tool_parameters.get('ensure_ascii', True) try: if replace_model == 'pattern': # get replace pattern replace_pattern = tool_parameters.get('replace_pattern', '') if not replace_pattern: return self.create_text_message('Invalid parameter replace_pattern') - result = self._replace_pattern(content, query, replace_pattern, replace_value) + result = self._replace_pattern(content, query, replace_pattern, replace_value, ensure_ascii) elif replace_model == 'key': - result = self._replace_key(content, query, replace_value) + result = self._replace_key(content, query, replace_value, ensure_ascii) elif replace_model == 'value': - result = self._replace_value(content, query, replace_value) + result = self._replace_value(content, query, replace_value, ensure_ascii) return self.create_text_message(str(result)) except Exception: return self.create_text_message('Failed to replace JSON content') # Replace pattern - def _replace_pattern(self, content: str, query: str, replace_pattern: str, replace_value: str) -> str: + def _replace_pattern(self, content: str, query: str, replace_pattern: str, replace_value: str, ensure_ascii: bool) -> str: try: input_data = json.loads(content) expr = parse(query) - + matches = expr.find(input_data) - + for match in matches: new_value = match.value.replace(replace_pattern, replace_value) match.full_path.update(input_data, new_value) - - return json.dumps(input_data, ensure_ascii=True) + + return json.dumps(input_data, ensure_ascii=ensure_ascii) except Exception as e: return str(e) - + # Replace key - def _replace_key(self, content: str, query: str, replace_value: str) -> str: + def _replace_key(self, content: str, query: str, replace_value: str, ensure_ascii: bool) -> str: try: input_data = json.loads(content) expr = parse(query) - + matches = expr.find(input_data) - + for match in matches: parent = match.context.value if isinstance(parent, dict): @@ -86,21 +87,21 @@ def _replace_key(self, content: str, query: str, replace_value: str) -> str: if isinstance(item, dict) and old_key in item: value = item.pop(old_key) item[replace_value] = value - return json.dumps(input_data, ensure_ascii=True) + return json.dumps(input_data, ensure_ascii=ensure_ascii) except Exception as e: return str(e) - + # Replace value - def _replace_value(self, content: str, query: str, replace_value: str) -> str: + def _replace_value(self, content: str, query: str, replace_value: str, ensure_ascii: bool) -> str: try: input_data = json.loads(content) expr = parse(query) - + matches = expr.find(input_data) - + for match in matches: match.full_path.update(input_data, replace_value) - - return json.dumps(input_data, ensure_ascii=True) + + return json.dumps(input_data, ensure_ascii=ensure_ascii) except Exception as e: - return str(e) \ No newline at end of file + return str(e) diff --git a/api/core/tools/provider/builtin/json_process/tools/replace.yaml b/api/core/tools/provider/builtin/json_process/tools/replace.yaml index 556be5e8b2bf66..cf4b1dc63f45b8 100644 --- a/api/core/tools/provider/builtin/json_process/tools/replace.yaml +++ b/api/core/tools/provider/builtin/json_process/tools/replace.yaml @@ -93,3 +93,15 @@ parameters: zh_Hans: 字符串替换 pt_BR: replace string form: form + - name: ensure_ascii + type: boolean + default: true + label: + en_US: Ensure ASCII + zh_Hans: 确保 ASCII + pt_BR: Ensure ASCII + human_description: + en_US: Ensure the JSON output is ASCII encoded + zh_Hans: 确保输出的 JSON 是 ASCII 编码 + pt_BR: Ensure the JSON output is ASCII encoded + form: form From d5dca46854c7db9246edad95077ad2ae3250b288 Mon Sep 17 00:00:00 2001 From: listeng Date: Thu, 18 Jul 2024 13:04:03 +0800 Subject: [PATCH 26/62] feat: add a Tianditu tool (#6320) Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> --- api/core/tools/provider/_position.yaml | 1 + .../builtin/tianditu/_assets/icon.svg | 21 +++++++++ .../provider/builtin/tianditu/tianditu.py | 21 +++++++++ .../provider/builtin/tianditu/tianditu.yaml | 32 +++++++++++++ .../builtin/tianditu/tools/geocoder.py | 33 ++++++++++++++ .../builtin/tianditu/tools/geocoder.yaml | 26 +++++++++++ .../builtin/tianditu/tools/poisearch.py | 45 +++++++++++++++++++ .../builtin/tianditu/tools/poisearch.yaml | 38 ++++++++++++++++ .../builtin/tianditu/tools/staticmap.py | 36 +++++++++++++++ .../builtin/tianditu/tools/staticmap.yaml | 26 +++++++++++ 10 files changed, 279 insertions(+) create mode 100644 api/core/tools/provider/builtin/tianditu/_assets/icon.svg create mode 100644 api/core/tools/provider/builtin/tianditu/tianditu.py create mode 100644 api/core/tools/provider/builtin/tianditu/tianditu.yaml create mode 100644 api/core/tools/provider/builtin/tianditu/tools/geocoder.py create mode 100644 api/core/tools/provider/builtin/tianditu/tools/geocoder.yaml create mode 100644 api/core/tools/provider/builtin/tianditu/tools/poisearch.py create mode 100644 api/core/tools/provider/builtin/tianditu/tools/poisearch.yaml create mode 100644 api/core/tools/provider/builtin/tianditu/tools/staticmap.py create mode 100644 api/core/tools/provider/builtin/tianditu/tools/staticmap.yaml diff --git a/api/core/tools/provider/_position.yaml b/api/core/tools/provider/_position.yaml index fa13629ef75f51..3a3ff644269d1c 100644 --- a/api/core/tools/provider/_position.yaml +++ b/api/core/tools/provider/_position.yaml @@ -30,3 +30,4 @@ - feishu - feishu_base - slack +- tianditu diff --git a/api/core/tools/provider/builtin/tianditu/_assets/icon.svg b/api/core/tools/provider/builtin/tianditu/_assets/icon.svg new file mode 100644 index 00000000000000..749d4bda265ab0 --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/_assets/icon.svg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/api/core/tools/provider/builtin/tianditu/tianditu.py b/api/core/tools/provider/builtin/tianditu/tianditu.py new file mode 100644 index 00000000000000..1f96be06b0200d --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tianditu.py @@ -0,0 +1,21 @@ +from typing import Any + +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.tianditu.tools.poisearch import PoiSearchTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + +class TiandituProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict[str, Any]) -> None: + try: + PoiSearchTool().fork_tool_runtime( + runtime={ + "credentials": credentials, + } + ).invoke(user_id='', + tool_parameters={ + 'content': '北京', + 'specify': '156110000', + }) + except Exception as e: + raise ToolProviderCredentialValidationError(str(e)) diff --git a/api/core/tools/provider/builtin/tianditu/tianditu.yaml b/api/core/tools/provider/builtin/tianditu/tianditu.yaml new file mode 100644 index 00000000000000..77af834bdc5893 --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tianditu.yaml @@ -0,0 +1,32 @@ +identity: + author: Listeng + name: tianditu + label: + en_US: Tianditu + zh_Hans: 天地图 + pt_BR: Tianditu + description: + en_US: The Tianditu tool provided the functions of place name search, geocoding, static maps generation, etc. in China region. + zh_Hans: 天地图工具可以调用天地图的接口，实现中国区域内的地名搜索、地理编码、静态地图等功能。 + pt_BR: The Tianditu tool provided the functions of place name search, geocoding, static maps generation, etc. in China region. + icon: icon.svg + tags: + - utilities + - travel +credentials_for_provider: + tianditu_api_key: + type: secret-input + required: true + label: + en_US: Tianditu API Key + zh_Hans: 天地图Key + pt_BR: Tianditu API key + placeholder: + en_US: Please input your Tianditu API key + zh_Hans: 请输入你的天地图Key + pt_BR: Please input your Tianditu API key + help: + en_US: Get your Tianditu API key from Tianditu + zh_Hans: 获取您的天地图Key + pt_BR: Get your Tianditu API key from Tianditu + url: http://lbs.tianditu.gov.cn/home.html diff --git a/api/core/tools/provider/builtin/tianditu/tools/geocoder.py b/api/core/tools/provider/builtin/tianditu/tools/geocoder.py new file mode 100644 index 00000000000000..484a3768c851df --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tools/geocoder.py @@ -0,0 +1,33 @@ +import json +from typing import Any, Union + +import requests + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class GeocoderTool(BuiltinTool): + + def _invoke(self, + user_id: str, + tool_parameters: dict[str, Any], + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + """ + invoke tools + """ + base_url = 'http://api.tianditu.gov.cn/geocoder' + + keyword = tool_parameters.get('keyword', '') + if not keyword: + return self.create_text_message('Invalid parameter keyword') + + tk = self.runtime.credentials['tianditu_api_key'] + + params = { + 'keyWord': keyword, + } + + result = requests.get(base_url + '?ds=' + json.dumps(params, ensure_ascii=False) + '&tk=' + tk).json() + + return self.create_json_message(result) diff --git a/api/core/tools/provider/builtin/tianditu/tools/geocoder.yaml b/api/core/tools/provider/builtin/tianditu/tools/geocoder.yaml new file mode 100644 index 00000000000000..d6a168f9502019 --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tools/geocoder.yaml @@ -0,0 +1,26 @@ +identity: + name: geocoder + author: Listeng + label: + en_US: Get coords converted from address name + zh_Hans: 地理编码 + pt_BR: Get coords converted from address name +description: + human: + en_US: Geocoder + zh_Hans: 中国区域地理编码查询 + pt_BR: Geocoder + llm: A tool for geocoder in China +parameters: + - name: keyword + type: string + required: true + label: + en_US: keyword + zh_Hans: 搜索的关键字 + pt_BR: keyword + human_description: + en_US: keyword + zh_Hans: 搜索的关键字 + pt_BR: keyword + form: llm diff --git a/api/core/tools/provider/builtin/tianditu/tools/poisearch.py b/api/core/tools/provider/builtin/tianditu/tools/poisearch.py new file mode 100644 index 00000000000000..08a5b8ef42a8c8 --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tools/poisearch.py @@ -0,0 +1,45 @@ +import json +from typing import Any, Union + +import requests + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class PoiSearchTool(BuiltinTool): + + def _invoke(self, + user_id: str, + tool_parameters: dict[str, Any], + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + """ + invoke tools + """ + geocoder_base_url = 'http://api.tianditu.gov.cn/geocoder' + base_url = 'http://api.tianditu.gov.cn/v2/search' + + keyword = tool_parameters.get('keyword', '') + if not keyword: + return self.create_text_message('Invalid parameter keyword') + + baseAddress = tool_parameters.get('baseAddress', '') + if not baseAddress: + return self.create_text_message('Invalid parameter baseAddress') + + tk = self.runtime.credentials['tianditu_api_key'] + + base_coords = requests.get(geocoder_base_url + '?ds=' + json.dumps({'keyWord': baseAddress,}, ensure_ascii=False) + '&tk=' + tk).json() + + params = { + 'keyWord': keyword, + 'queryRadius': 5000, + 'queryType': 3, + 'pointLonlat': base_coords['location']['lon'] + ',' + base_coords['location']['lat'], + 'start': 0, + 'count': 100, + } + + result = requests.get(base_url + '?postStr=' + json.dumps(params, ensure_ascii=False) + '&type=query&tk=' + tk).json() + + return self.create_json_message(result) diff --git a/api/core/tools/provider/builtin/tianditu/tools/poisearch.yaml b/api/core/tools/provider/builtin/tianditu/tools/poisearch.yaml new file mode 100644 index 00000000000000..01289d24e3d29a --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tools/poisearch.yaml @@ -0,0 +1,38 @@ +identity: + name: point_of_interest_search + author: Listeng + label: + en_US: Point of Interest search + zh_Hans: 兴趣点搜索 + pt_BR: Point of Interest search +description: + human: + en_US: Search for certain types of points of interest around a location + zh_Hans: 搜索某个位置周边的5公里内某种类型的兴趣点 + pt_BR: Search for certain types of points of interest around a location + llm: A tool for searching for certain types of points of interest around a location +parameters: + - name: keyword + type: string + required: true + label: + en_US: poi keyword + zh_Hans: 兴趣点的关键字 + pt_BR: poi keyword + human_description: + en_US: poi keyword + zh_Hans: 兴趣点的关键字 + pt_BR: poi keyword + form: llm + - name: baseAddress + type: string + required: true + label: + en_US: base current point + zh_Hans: 当前位置的关键字 + pt_BR: base current point + human_description: + en_US: base current point + zh_Hans: 当前位置的关键字 + pt_BR: base current point + form: llm diff --git a/api/core/tools/provider/builtin/tianditu/tools/staticmap.py b/api/core/tools/provider/builtin/tianditu/tools/staticmap.py new file mode 100644 index 00000000000000..ecac4404ca28b0 --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tools/staticmap.py @@ -0,0 +1,36 @@ +import json +from typing import Any, Union + +import requests + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class PoiSearchTool(BuiltinTool): + + def _invoke(self, + user_id: str, + tool_parameters: dict[str, Any], + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + """ + invoke tools + """ + + geocoder_base_url = 'http://api.tianditu.gov.cn/geocoder' + base_url = 'http://api.tianditu.gov.cn/staticimage' + + keyword = tool_parameters.get('keyword', '') + if not keyword: + return self.create_text_message('Invalid parameter keyword') + + tk = self.runtime.credentials['tianditu_api_key'] + + keyword_coords = requests.get(geocoder_base_url + '?ds=' + json.dumps({'keyWord': keyword,}, ensure_ascii=False) + '&tk=' + tk).json() + coords = keyword_coords['location']['lon'] + ',' + keyword_coords['location']['lat'] + + result = requests.get(base_url + '?center=' + coords + '&markers=' + coords + '&width=400&height=300&zoom=14&tk=' + tk).content + + return self.create_blob_message(blob=result, + meta={'mime_type': 'image/png'}, + save_as=self.VARIABLE_KEY.IMAGE.value) diff --git a/api/core/tools/provider/builtin/tianditu/tools/staticmap.yaml b/api/core/tools/provider/builtin/tianditu/tools/staticmap.yaml new file mode 100644 index 00000000000000..fc54c428066af5 --- /dev/null +++ b/api/core/tools/provider/builtin/tianditu/tools/staticmap.yaml @@ -0,0 +1,26 @@ +identity: + name: generate_static_map + author: Listeng + label: + en_US: Generate a static map + zh_Hans: 生成静态地图 + pt_BR: Generate a static map +description: + human: + en_US: Generate a static map + zh_Hans: 生成静态地图 + pt_BR: Generate a static map + llm: A tool for generate a static map +parameters: + - name: keyword + type: string + required: true + label: + en_US: keyword + zh_Hans: 搜索的关键字 + pt_BR: keyword + human_description: + en_US: keyword + zh_Hans: 搜索的关键字 + pt_BR: keyword + form: llm From 588615b20e01d4c0e1fc1ec132eb0fcef5181e9d Mon Sep 17 00:00:00 2001 From: William Espegren <131612909+WilliamEspegren@users.noreply.github.com> Date: Thu, 18 Jul 2024 08:29:33 +0200 Subject: [PATCH 27/62] feat: Spider web scraper & crawler tool (#5725) --- .../provider/builtin/spider/_assets/icon.svg | 1 + .../tools/provider/builtin/spider/spider.py | 14 ++ .../tools/provider/builtin/spider/spider.yaml | 27 ++ .../provider/builtin/spider/spiderApp.py | 237 ++++++++++++++++++ .../builtin/spider/tools/scraper_crawler.py | 47 ++++ .../builtin/spider/tools/scraper_crawler.yaml | 100 ++++++++ 6 files changed, 426 insertions(+) create mode 100644 api/core/tools/provider/builtin/spider/_assets/icon.svg create mode 100644 api/core/tools/provider/builtin/spider/spider.py create mode 100644 api/core/tools/provider/builtin/spider/spider.yaml create mode 100644 api/core/tools/provider/builtin/spider/spiderApp.py create mode 100644 api/core/tools/provider/builtin/spider/tools/scraper_crawler.py create mode 100644 api/core/tools/provider/builtin/spider/tools/scraper_crawler.yaml diff --git a/api/core/tools/provider/builtin/spider/_assets/icon.svg b/api/core/tools/provider/builtin/spider/_assets/icon.svg new file mode 100644 index 00000000000000..604a09d01d7444 --- /dev/null +++ b/api/core/tools/provider/builtin/spider/_assets/icon.svg @@ -0,0 +1 @@ +Spider v1 Logo diff --git a/api/core/tools/provider/builtin/spider/spider.py b/api/core/tools/provider/builtin/spider/spider.py new file mode 100644 index 00000000000000..6fa431b6bb6ad6 --- /dev/null +++ b/api/core/tools/provider/builtin/spider/spider.py @@ -0,0 +1,14 @@ +from typing import Any + +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.spider.spiderApp import Spider +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + +class SpiderProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict[str, Any]) -> None: + try: + app = Spider(api_key=credentials["spider_api_key"]) + app.scrape_url(url="https://spider.cloud") + except Exception as e: + raise ToolProviderCredentialValidationError(str(e)) diff --git a/api/core/tools/provider/builtin/spider/spider.yaml b/api/core/tools/provider/builtin/spider/spider.yaml new file mode 100644 index 00000000000000..45702c85ddea24 --- /dev/null +++ b/api/core/tools/provider/builtin/spider/spider.yaml @@ -0,0 +1,27 @@ +identity: + author: William Espegren + name: spider + label: + en_US: Spider + zh_CN: Spider + description: + en_US: Spider API integration, returning LLM-ready data by scraping & crawling websites. + zh_CN: Spider API 集成，通过爬取和抓取网站返回 LLM-ready 数据。 + icon: icon.svg + tags: + - search + - utilities +credentials_for_provider: + spider_api_key: + type: secret-input + required: true + label: + en_US: Spider API Key + zh_CN: Spider API 密钥 + placeholder: + en_US: Please input your Spider API key + zh_CN: 请输入您的 Spider API 密钥 + help: + en_US: Get your Spider API key from your Spider dashboard + zh_CN: 从您的 Spider 仪表板中获取 Spider API 密钥。 + url: https://spider.cloud/ diff --git a/api/core/tools/provider/builtin/spider/spiderApp.py b/api/core/tools/provider/builtin/spider/spiderApp.py new file mode 100644 index 00000000000000..82c0df19ca9567 --- /dev/null +++ b/api/core/tools/provider/builtin/spider/spiderApp.py @@ -0,0 +1,237 @@ +import os +from typing import Literal, Optional, TypedDict + +import requests + + +class RequestParamsDict(TypedDict, total=False): + url: Optional[str] + request: Optional[Literal["http", "chrome", "smart"]] + limit: Optional[int] + return_format: Optional[Literal["raw", "markdown", "html2text", "text", "bytes"]] + tld: Optional[bool] + depth: Optional[int] + cache: Optional[bool] + budget: Optional[dict[str, int]] + locale: Optional[str] + cookies: Optional[str] + stealth: Optional[bool] + headers: Optional[dict[str, str]] + anti_bot: Optional[bool] + metadata: Optional[bool] + viewport: Optional[dict[str, int]] + encoding: Optional[str] + subdomains: Optional[bool] + user_agent: Optional[str] + store_data: Optional[bool] + gpt_config: Optional[list[str]] + fingerprint: Optional[bool] + storageless: Optional[bool] + readability: Optional[bool] + proxy_enabled: Optional[bool] + respect_robots: Optional[bool] + query_selector: Optional[str] + full_resources: Optional[bool] + request_timeout: Optional[int] + run_in_background: Optional[bool] + skip_config_checks: Optional[bool] + + +class Spider: + def __init__(self, api_key: Optional[str] = None): + """ + Initialize the Spider with an API key. + + :param api_key: A string of the API key for Spider. Defaults to the SPIDER_API_KEY environment variable. + :raises ValueError: If no API key is provided. + """ + self.api_key = api_key or os.getenv("SPIDER_API_KEY") + if self.api_key is None: + raise ValueError("No API key provided") + + def api_post( + self, + endpoint: str, + data: dict, + stream: bool, + content_type: str = "application/json", + ): + """ + Send a POST request to the specified API endpoint. + + :param endpoint: The API endpoint to which the POST request is sent. + :param data: The data (dictionary) to be sent in the POST request. + :param stream: Boolean indicating if the response should be streamed. + :return: The JSON response or the raw response stream if stream is True. + """ + headers = self._prepare_headers(content_type) + response = self._post_request( + f"https://api.spider.cloud/v1/{endpoint}", data, headers, stream + ) + + if stream: + return response + elif response.status_code == 200: + return response.json() + else: + self._handle_error(response, f"post to {endpoint}") + + def api_get( + self, endpoint: str, stream: bool, content_type: str = "application/json" + ): + """ + Send a GET request to the specified endpoint. + + :param endpoint: The API endpoint from which to retrieve data. + :return: The JSON decoded response. + """ + headers = self._prepare_headers(content_type) + response = self._get_request( + f"https://api.spider.cloud/v1/{endpoint}", headers, stream + ) + if response.status_code == 200: + return response.json() + else: + self._handle_error(response, f"get from {endpoint}") + + def get_credits(self): + """ + Retrieve the account's remaining credits. + + :return: JSON response containing the number of credits left. + """ + return self.api_get("credits", stream=False) + + def scrape_url( + self, + url: str, + params: Optional[RequestParamsDict] = None, + stream: bool = False, + content_type: str = "application/json", + ): + """ + Scrape data from the specified URL. + + :param url: The URL from which to scrape data. + :param params: Optional dictionary of additional parameters for the scrape request. + :return: JSON response containing the scraping results. + """ + + # Add { "return_format": "markdown" } to the params if not already present + if "return_format" not in params: + params["return_format"] = "markdown" + + # Set limit to 1 + params["limit"] = 1 + + return self.api_post( + "crawl", {"url": url, **(params or {})}, stream, content_type + ) + + def crawl_url( + self, + url: str, + params: Optional[RequestParamsDict] = None, + stream: bool = False, + content_type: str = "application/json", + ): + """ + Start crawling at the specified URL. + + :param url: The URL to begin crawling. + :param params: Optional dictionary with additional parameters to customize the crawl. + :param stream: Boolean indicating if the response should be streamed. Defaults to False. + :return: JSON response or the raw response stream if streaming enabled. + """ + + # Add { "return_format": "markdown" } to the params if not already present + if "return_format" not in params: + params["return_format"] = "markdown" + + return self.api_post( + "crawl", {"url": url, **(params or {})}, stream, content_type + ) + + def links( + self, + url: str, + params: Optional[RequestParamsDict] = None, + stream: bool = False, + content_type: str = "application/json", + ): + """ + Retrieve links from the specified URL. + + :param url: The URL from which to extract links. + :param params: Optional parameters for the link retrieval request. + :return: JSON response containing the links. + """ + return self.api_post( + "links", {"url": url, **(params or {})}, stream, content_type + ) + + def extract_contacts( + self, + url: str, + params: Optional[RequestParamsDict] = None, + stream: bool = False, + content_type: str = "application/json", + ): + """ + Extract contact information from the specified URL. + + :param url: The URL from which to extract contact information. + :param params: Optional parameters for the contact extraction. + :return: JSON response containing extracted contact details. + """ + return self.api_post( + "pipeline/extract-contacts", + {"url": url, **(params or {})}, + stream, + content_type, + ) + + def label( + self, + url: str, + params: Optional[RequestParamsDict] = None, + stream: bool = False, + content_type: str = "application/json", + ): + """ + Apply labeling to data extracted from the specified URL. + + :param url: The URL to label data from. + :param params: Optional parameters to guide the labeling process. + :return: JSON response with labeled data. + """ + return self.api_post( + "pipeline/label", {"url": url, **(params or {})}, stream, content_type + ) + + def _prepare_headers(self, content_type: str = "application/json"): + return { + "Content-Type": content_type, + "Authorization": f"Bearer {self.api_key}", + "User-Agent": "Spider-Client/0.0.27", + } + + def _post_request(self, url: str, data, headers, stream=False): + return requests.post(url, headers=headers, json=data, stream=stream) + + def _get_request(self, url: str, headers, stream=False): + return requests.get(url, headers=headers, stream=stream) + + def _delete_request(self, url: str, headers, stream=False): + return requests.delete(url, headers=headers, stream=stream) + + def _handle_error(self, response, action): + if response.status_code in [402, 409, 500]: + error_message = response.json().get("error", "Unknown error occurred") + raise Exception( + f"Failed to {action}. Status code: {response.status_code}. Error: {error_message}" + ) + else: + raise Exception( + f"Unexpected error occurred while trying to {action}. Status code: {response.status_code}" + ) diff --git a/api/core/tools/provider/builtin/spider/tools/scraper_crawler.py b/api/core/tools/provider/builtin/spider/tools/scraper_crawler.py new file mode 100644 index 00000000000000..64bbcc10ccc13e --- /dev/null +++ b/api/core/tools/provider/builtin/spider/tools/scraper_crawler.py @@ -0,0 +1,47 @@ +from typing import Any, Union + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.provider.builtin.spider.spiderApp import Spider +from core.tools.tool.builtin_tool import BuiltinTool + + +class ScrapeTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + # initialize the app object with the api key + app = Spider(api_key=self.runtime.credentials['spider_api_key']) + + url = tool_parameters['url'] + mode = tool_parameters['mode'] + + options = { + 'limit': tool_parameters.get('limit', 0), + 'depth': tool_parameters.get('depth', 0), + 'blacklist': tool_parameters.get('blacklist', '').split(',') if tool_parameters.get('blacklist') else [], + 'whitelist': tool_parameters.get('whitelist', '').split(',') if tool_parameters.get('whitelist') else [], + 'readability': tool_parameters.get('readability', False), + } + + result = "" + + try: + if mode == 'scrape': + scrape_result = app.scrape_url( + url=url, + params=options, + ) + + for i in scrape_result: + result += "URL: " + i.get('url', '') + "\n" + result += "CONTENT: " + i.get('content', '') + "\n\n" + elif mode == 'crawl': + crawl_result = app.crawl_url( + url=tool_parameters['url'], + params=options, + ) + for i in crawl_result: + result += "URL: " + i.get('url', '') + "\n" + result += "CONTENT: " + i.get('content', '') + "\n\n" + except Exception as e: + return self.create_text_message("An error occured", str(e)) + + return self.create_text_message(result) diff --git a/api/core/tools/provider/builtin/spider/tools/scraper_crawler.yaml b/api/core/tools/provider/builtin/spider/tools/scraper_crawler.yaml new file mode 100644 index 00000000000000..4ebdce61ff300e --- /dev/null +++ b/api/core/tools/provider/builtin/spider/tools/scraper_crawler.yaml @@ -0,0 +1,100 @@ +identity: + name: scraper_crawler + author: William Espegren + label: + en_US: Web Scraper & Crawler + zh_Hans: 网页抓取与爬虫 +description: + human: + en_US: A tool for scraping & crawling webpages. Input should be a url. + zh_Hans: 用于抓取和爬取网页的工具。输入应该是一个网址。 + llm: A tool for scraping & crawling webpages. Input should be a url. +parameters: + - name: url + type: string + required: true + label: + en_US: URL + zh_Hans: 网址 + human_description: + en_US: url to be scraped or crawled + zh_Hans: 要抓取或爬取的网址 + llm_description: url to either be scraped or crawled + form: llm + - name: mode + type: select + required: true + options: + - value: scrape + label: + en_US: scrape + zh_Hans: 抓取 + - value: crawl + label: + en_US: crawl + zh_Hans: 爬取 + default: crawl + label: + en_US: Mode + zh_Hans: 模式 + human_description: + en_US: used for selecting to either scrape the website or crawl the entire website following subpages + zh_Hans: 用于选择抓取网站或爬取整个网站及其子页面 + form: form + - name: limit + type: number + required: false + label: + en_US: maximum number of pages to crawl + zh_Hans: 最大爬取页面数 + human_description: + en_US: specify the maximum number of pages to crawl per website. the crawler will stop after reaching this limit. + zh_Hans: 指定每个网站要爬取的最大页面数。爬虫将在达到此限制后停止。 + form: form + min: 0 + default: 0 + - name: depth + type: number + required: false + label: + en_US: maximum depth of pages to crawl + zh_Hans: 最大爬取深度 + human_description: + en_US: the crawl limit for maximum depth. + zh_Hans: 最大爬取深度的限制。 + form: form + min: 0 + default: 0 + - name: blacklist + type: string + required: false + label: + en_US: url patterns to exclude + zh_Hans: 要排除的URL模式 + human_description: + en_US: blacklist a set of paths that you do not want to crawl. you can use regex patterns to help with the list. + zh_Hans: 指定一组不想爬取的路径。您可以使用正则表达式模式来帮助定义列表。 + placeholder: /blog/*, /about + form: form + - name: whitelist + type: string + required: false + label: + en_US: URL patterns to include + zh_Hans: 要包含的URL模式 + human_description: + en_US: Whitelist a set of paths that you want to crawl, ignoring all other routes that do not match the patterns. You can use regex patterns to help with the list. + zh_Hans: 指定一组要爬取的路径，忽略所有不匹配模式的其他路由。您可以使用正则表达式模式来帮助定义列表。 + placeholder: /blog/*, /about + form: form + - name: readability + type: boolean + required: false + label: + en_US: Pre-process the content for LLM usage + zh_Hans: 仅返回页面的主要内容 + human_description: + en_US: Use Mozilla's readability to pre-process the content for reading. This may drastically improve the content for LLM usage. + zh_Hans: 如果启用，爬虫将仅返回页面的主要内容，不包括标题、导航、页脚等。 + form: form + default: false From 166a40c66e1192922c78baee48b37badc56ce329 Mon Sep 17 00:00:00 2001 From: Kuizuo Date: Thu, 18 Jul 2024 14:44:34 +0800 Subject: [PATCH 28/62] fix: improve separation element in prompt log and TTS buttons in the operation (#6413) --- web/app/components/base/chat/chat/answer/operation.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/web/app/components/base/chat/chat/answer/operation.tsx b/web/app/components/base/chat/chat/answer/operation.tsx index e3e912d2890829..d46aa3437596b2 100644 --- a/web/app/components/base/chat/chat/answer/operation.tsx +++ b/web/app/components/base/chat/chat/answer/operation.tsx @@ -113,11 +113,14 @@ const Operation: FC = ({ {!isOpeningStatement && (showPromptLog || config?.text_to_speech?.enabled) && (
{showPromptLog && ( - + <> + +
+ )} + {(config?.text_to_speech?.enabled) && ( <> -
Date: Thu, 18 Jul 2024 15:06:14 +0800 Subject: [PATCH 29/62] feat: support get workflow task execution status (#6411) --- api/controllers/service_api/app/workflow.py | 33 ++++++++++- .../develop/template/template_workflow.en.mdx | 57 +++++++++++++++++++ .../develop/template/template_workflow.zh.mdx | 57 +++++++++++++++++++ 3 files changed, 145 insertions(+), 2 deletions(-) diff --git a/api/controllers/service_api/app/workflow.py b/api/controllers/service_api/app/workflow.py index dd11949e840b2b..10484c90279846 100644 --- a/api/controllers/service_api/app/workflow.py +++ b/api/controllers/service_api/app/workflow.py @@ -1,6 +1,6 @@ import logging -from flask_restful import Resource, reqparse +from flask_restful import Resource, fields, marshal_with, reqparse from werkzeug.exceptions import InternalServerError from controllers.service_api import api @@ -21,14 +21,43 @@ QuotaExceededError, ) from core.model_runtime.errors.invoke import InvokeError +from extensions.ext_database import db from libs import helper from models.model import App, AppMode, EndUser +from models.workflow import WorkflowRun from services.app_generate_service import AppGenerateService logger = logging.getLogger(__name__) class WorkflowRunApi(Resource): + workflow_run_fields = { + 'id': fields.String, + 'workflow_id': fields.String, + 'status': fields.String, + 'inputs': fields.Raw, + 'outputs': fields.Raw, + 'error': fields.String, + 'total_steps': fields.Integer, + 'total_tokens': fields.Integer, + 'created_at': fields.DateTime, + 'finished_at': fields.DateTime, + 'elapsed_time': fields.Float, + } + + @validate_app_token + @marshal_with(workflow_run_fields) + def get(self, app_model: App, workflow_id: str): + """ + Get a workflow task running detail + """ + app_mode = AppMode.value_of(app_model.mode) + if app_mode != AppMode.WORKFLOW: + raise NotWorkflowAppError() + + workflow_run = db.session.query(WorkflowRun).filter(WorkflowRun.id == workflow_id).first() + return workflow_run + @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON, required=True)) def post(self, app_model: App, end_user: EndUser): """ @@ -88,5 +117,5 @@ def post(self, app_model: App, end_user: EndUser, task_id: str): } -api.add_resource(WorkflowRunApi, '/workflows/run') +api.add_resource(WorkflowRunApi, '/workflows/run/', '/workflows/run') api.add_resource(WorkflowTaskStopApi, '/workflows/tasks//stop') diff --git a/web/app/components/develop/template/template_workflow.en.mdx b/web/app/components/develop/template/template_workflow.en.mdx index 64a9092f591d59..1c86aad508de66 100644 --- a/web/app/components/develop/template/template_workflow.en.mdx +++ b/web/app/components/develop/template/template_workflow.en.mdx @@ -224,6 +224,63 @@ Workflow applications offers non-session support and is ideal for translation, a --- + + + + Retrieve the current execution results of a workflow task based on the workflow execution ID. + ### Path + - `workflow_id` (string) Workflow ID, can be obtained from the streaming chunk return + ### Response + - `id` (string) ID of workflow execution + - `workflow_id` (string) ID of relatied workflow + - `status` (string) status of execution, `running` / `succeeded` / `failed` / `stopped` + - `inputs` (json) content of input + - `outputs` (json) content of output + - `error` (string) reason of error + - `total_steps` (int) total steps of task + - `total_tokens` (int) total tokens to be used + - `created_at` (timestamp) start time + - `finished_at` (timestamp) end time + - `elapsed_time` (float) total seconds to be used + + + ### Request Example + + ```bash {{ title: 'cURL' }} + curl -X GET '${props.appDetail.api_base_url}/workflows/run/:workflow_id' \ + -H 'Authorization: Bearer {api_key}' \ + -H 'Content-Type: application/json' + ``` + + + ### Response Example + + ```json {{ title: 'Response' }} + { + "id": "b1ad3277-089e-42c6-9dff-6820d94fbc76", + "workflow_id": "19eff89f-ec03-4f75-b0fc-897e7effea02", + "status": "succeeded", + "inputs": "{\"sys.files\": [], \"sys.user_id\": \"abc-123\"}", + "outputs": null, + "error": null, + "total_steps": 3, + "total_tokens": 0, + "created_at": "Thu, 18 Jul 2024 03:17:40 -0000", + "finished_at": "Thu, 18 Jul 2024 03:18:10 -0000", + "elapsed_time": 30.098514399956912 + } + ``` + + + + +--- + + + + 根据 workflow 执行 ID 获取 workflow 任务当前执行结果 + ### Path + - `workflow_id` (string) workflow 执行 ID，可在流式返回 Chunk 中获取 + ### Response + - `id` (string) workflow 执行 ID + - `workflow_id` (string) 关联的 Workflow ID + - `status` (string) 执行状态 `running` / `succeeded` / `failed` / `stopped` + - `inputs` (json) 任务输入内容 + - `outputs` (json) 任务输出内容 + - `error` (string) 错误原因 + - `total_steps` (int) 任务执行总步数 + - `total_tokens` (int) 任务执行总 tokens + - `created_at` (timestamp) 任务开始时间 + - `finished_at` (timestamp) 任务结束时间 + - `elapsed_time` (float) 耗时(s) + + + ### Request Example + + ```bash {{ title: 'cURL' }} + curl -X GET '${props.appDetail.api_base_url}/workflows/run/:workflow_id' \ + -H 'Authorization: Bearer {api_key}' \ + -H 'Content-Type: application/json' + ``` + + + ### Response Example + + ```json {{ title: 'Response' }} + { + "id": "b1ad3277-089e-42c6-9dff-6820d94fbc76", + "workflow_id": "19eff89f-ec03-4f75-b0fc-897e7effea02", + "status": "succeeded", + "inputs": "{\"sys.files\": [], \"sys.user_id\": \"abc-123\"}", + "outputs": null, + "error": null, + "total_steps": 3, + "total_tokens": 0, + "created_at": "Thu, 18 Jul 2024 03:17:40 -0000", + "finished_at": "Thu, 18 Jul 2024 03:18:10 -0000", + "elapsed_time": 30.098514399956912 + } + ``` + + + + +--- + Date: Thu, 18 Jul 2024 17:35:10 +0800 Subject: [PATCH 30/62] feat: add frontend unit test framework (#6426) --- web/.vscode/extensions.json | 6 + web/README.md | 19 + web/app/components/base/button/index.spec.tsx | 49 + web/app/components/base/button/index.tsx | 2 +- web/jest.config.ts | 208 ++ web/package.json | 11 +- web/utils/classnames.spec.ts | 55 + web/yarn.lock | 2114 ++++++++++++++++- 8 files changed, 2441 insertions(+), 23 deletions(-) create mode 100644 web/.vscode/extensions.json create mode 100644 web/app/components/base/button/index.spec.tsx create mode 100644 web/jest.config.ts create mode 100644 web/utils/classnames.spec.ts diff --git a/web/.vscode/extensions.json b/web/.vscode/extensions.json new file mode 100644 index 00000000000000..d7680d74a5fd89 --- /dev/null +++ b/web/.vscode/extensions.json @@ -0,0 +1,6 @@ +{ + "recommendations": [ + "bradlc.vscode-tailwindcss", + "firsttris.vscode-jest-runner" + ] +} diff --git a/web/README.md b/web/README.md index 2ecba1c8ffcb21..867d822e271088 100644 --- a/web/README.md +++ b/web/README.md @@ -74,6 +74,25 @@ npm run start --port=3001 --host=0.0.0.0 If your IDE is VSCode, rename `web/.vscode/settings.example.json` to `web/.vscode/settings.json` for lint code setting. +## Test + +We start to use [Jest](https://jestjs.io/) and [React Testing Library](https://testing-library.com/docs/react-testing-library/intro/) for Unit Testing. + +You can create a test file with a suffix of `.spec` beside the file that to be tested. For example, if you want to test a file named `util.ts`. The test file name should be `util.spec.ts`. + +Run test: + +```bash +npm run test +``` + +If you are not familiar with writing tests, here is some code to refer to: +* [classnames.spec.ts](./utils/classnames.spec.ts) +* [index.spec.tsx](./app/components/base/button/index.spec.tsx) + + + + ## Documentation Visit to view the full documentation. diff --git a/web/app/components/base/button/index.spec.tsx b/web/app/components/base/button/index.spec.tsx new file mode 100644 index 00000000000000..308656c238e382 --- /dev/null +++ b/web/app/components/base/button/index.spec.tsx @@ -0,0 +1,49 @@ +import React from 'react' +import { cleanup, fireEvent, render } from '@testing-library/react' +import Button from './index' + +afterEach(cleanup) +// https://testing-library.com/docs/queries/about +describe('Button text', () => { + test('Button text should be same as children', async () => { + const { getByRole, container } = render() + expect(getByRole('button').textContent).toBe('Click me') + expect(container.querySelector('button')?.textContent).toBe('Click me') + }) + + test('Loading button text should include same as children', async () => { + const { getByRole } = render() + expect(getByRole('button').textContent?.includes('Loading')).toBe(true) + }) +}) + +describe('Button style', () => { + test('Button should have default variant', async () => { + const { getByRole } = render() + expect(getByRole('button').className).toContain('btn-secondary') + }) + + test('Button should have primary variant', async () => { + const { getByRole } = render() + expect(getByRole('button').className).toContain('btn-primary') + }) + + test('Button should have warning variant', async () => { + const { getByRole } = render() + expect(getByRole('button').className).toContain('btn-warning') + }) + + test('Button disabled should have disabled variant', async () => { + const { getByRole } = render() + expect(getByRole('button').className).toContain('btn-disabled') + }) +}) + +describe('Button events', () => { + test('onClick should been call after clicked', async () => { + const onClick = jest.fn() + const { getByRole } = render() + fireEvent.click(getByRole('button')) + expect(onClick).toHaveBeenCalled() + }) +}) diff --git a/web/app/components/base/button/index.tsx b/web/app/components/base/button/index.tsx index 959b2dbe7b6d40..901c813fac7b6b 100644 --- a/web/app/components/base/button/index.tsx +++ b/web/app/components/base/button/index.tsx @@ -45,7 +45,7 @@ const Button = React.forwardRef( {...props} > {children} - + {loading && } ) }, diff --git a/web/jest.config.ts b/web/jest.config.ts new file mode 100644 index 00000000000000..d7c68308cb8c50 --- /dev/null +++ b/web/jest.config.ts @@ -0,0 +1,208 @@ +/** + * For a detailed explanation regarding each configuration property, visit: + * https://jestjs.io/docs/configuration + */ + +import type { Config } from 'jest' +import nextJest from 'next/jest.js' + +// https://nextjs.org/docs/app/building-your-application/testing/jest +const createJestConfig = nextJest({ + // Provide the path to your Next.js app to load next.config.js and .env files in your test environment + dir: './', +}) + +const config: Config = { + // All imported modules in your tests should be mocked automatically + // automock: false, + + // Stop running tests after `n` failures + // bail: 0, + + // The directory where Jest should store its cached dependency information + // cacheDirectory: "/private/var/folders/9c/7gly5yl90qxdjljqsvkk758h0000gn/T/jest_dx", + + // Automatically clear mock calls, instances, contexts and results before every test + clearMocks: true, + + // Indicates whether the coverage information should be collected while executing the test + collectCoverage: false, + + // An array of glob patterns indicating a set of files for which coverage information should be collected + // collectCoverageFrom: undefined, + + // The directory where Jest should output its coverage files + // coverageDirectory: "coverage", + + // An array of regexp pattern strings used to skip coverage collection + // coveragePathIgnorePatterns: [ + // "/node_modules/" + // ], + + // Indicates which provider should be used to instrument code for coverage + coverageProvider: 'v8', + + // A list of reporter names that Jest uses when writing coverage reports + // coverageReporters: [ + // "json", + // "text", + // "lcov", + // "clover" + // ], + + // An object that configures minimum threshold enforcement for coverage results + // coverageThreshold: undefined, + + // A path to a custom dependency extractor + // dependencyExtractor: undefined, + + // Make calling deprecated APIs throw helpful error messages + // errorOnDeprecated: false, + + // The default configuration for fake timers + // fakeTimers: { + // "enableGlobally": false + // }, + + // Force coverage collection from ignored files using an array of glob patterns + // forceCoverageMatch: [], + + // A path to a module which exports an async function that is triggered once before all test suites + // globalSetup: undefined, + + // A path to a module which exports an async function that is triggered once after all test suites + // globalTeardown: undefined, + + // A set of global variables that need to be available in all test environments + // globals: {}, + + // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers. + // maxWorkers: "50%", + + // An array of directory names to be searched recursively up from the requiring module's location + // moduleDirectories: [ + // "node_modules" + // ], + + // An array of file extensions your modules use + // moduleFileExtensions: [ + // "js", + // "mjs", + // "cjs", + // "jsx", + // "ts", + // "tsx", + // "json", + // "node" + // ], + + // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module + moduleNameMapper: { + '^@/components/(.*)$': '/components/$1', + }, + + // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader + // modulePathIgnorePatterns: [], + + // Activates notifications for test results + // notify: false, + + // An enum that specifies notification mode. Requires { notify: true } + // notifyMode: "failure-change", + + // A preset that is used as a base for Jest's configuration + // preset: undefined, + + // Run tests from one or more projects + // projects: undefined, + + // Use this configuration option to add custom reporters to Jest + // reporters: undefined, + + // Automatically reset mock state before every test + // resetMocks: false, + + // Reset the module registry before running each individual test + // resetModules: false, + + // A path to a custom resolver + // resolver: undefined, + + // Automatically restore mock state and implementation before every test + // restoreMocks: false, + + // The root directory that Jest should scan for tests and modules within + // rootDir: undefined, + + // A list of paths to directories that Jest should use to search for files in + // roots: [ + // "" + // ], + + // Allows you to use a custom runner instead of Jest's default test runner + // runner: "jest-runner", + + // The paths to modules that run some code to configure or set up the testing environment before each test + // setupFiles: [], + + // A list of paths to modules that run some code to configure or set up the testing framework before each test + // setupFilesAfterEnv: [], + + // The number of seconds after which a test is considered as slow and reported as such in the results. + // slowTestThreshold: 5, + + // A list of paths to snapshot serializer modules Jest should use for snapshot testing + // snapshotSerializers: [], + + // The test environment that will be used for testing + testEnvironment: 'jsdom', + + // Options that will be passed to the testEnvironment + // testEnvironmentOptions: {}, + + // Adds a location field to test results + // testLocationInResults: false, + + // The glob patterns Jest uses to detect test files + // testMatch: [ + // "**/__tests__/**/*.[jt]s?(x)", + // "**/?(*.)+(spec|test).[tj]s?(x)" + // ], + + // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped + // testPathIgnorePatterns: [ + // "/node_modules/" + // ], + + // The regexp pattern or array of patterns that Jest uses to detect test files + // testRegex: [], + + // This option allows the use of a custom results processor + // testResultsProcessor: undefined, + + // This option allows use of a custom test runner + // testRunner: "jest-circus/runner", + + // A map from regular expressions to paths to transformers + // transform: undefined, + + // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation + // transformIgnorePatterns: [ + // "/node_modules/", + // "\\.pnp\\.[^\\/]+$" + // ], + + // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them + // unmockedModulePathPatterns: undefined, + + // Indicates whether each individual test should be reported during the run + // verbose: undefined, + + // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode + // watchPathIgnorePatterns: [], + + // Whether to use watchman for file crawling + // watchman: true, +} + +export default createJestConfig(config) diff --git a/web/package.json b/web/package.json index 6e4aff557ed371..b4a40596229bf5 100644 --- a/web/package.json +++ b/web/package.json @@ -15,7 +15,9 @@ "prepare": "cd ../ && node -e \"if (process.env.NODE_ENV !== 'production'){process.exit(1)} \" || husky install ./web/.husky", "gen-icons": "node ./app/components/base/icons/script.js", "uglify-embed": "node ./bin/uglify-embed", - "check-i18n": "node ./i18n/script.js" + "check-i18n": "node ./i18n/script.js", + "test": "jest", + "test:watch": "jest --watch" }, "dependencies": { "@babel/runtime": "^7.22.3", @@ -102,8 +104,12 @@ "@antfu/eslint-config": "^0.36.0", "@faker-js/faker": "^7.6.0", "@rgrove/parse-xml": "^4.1.0", + "@testing-library/dom": "^10.3.2", + "@testing-library/jest-dom": "^6.4.6", + "@testing-library/react": "^16.0.0", "@types/crypto-js": "^4.1.1", "@types/dagre": "^0.7.52", + "@types/jest": "^29.5.12", "@types/js-cookie": "^3.0.3", "@types/lodash-es": "^4.17.7", "@types/negotiator": "^0.6.1", @@ -124,10 +130,13 @@ "eslint": "^8.36.0", "eslint-config-next": "^14.0.4", "husky": "^8.0.3", + "jest": "^29.7.0", + "jest-environment-jsdom": "^29.7.0", "lint-staged": "^13.2.2", "postcss": "^8.4.31", "sass": "^1.61.0", "tailwindcss": "^3.4.4", + "ts-node": "^10.9.2", "typescript": "4.9.5", "uglify-js": "^3.17.4" }, diff --git a/web/utils/classnames.spec.ts b/web/utils/classnames.spec.ts new file mode 100644 index 00000000000000..268981d78c094d --- /dev/null +++ b/web/utils/classnames.spec.ts @@ -0,0 +1,55 @@ +import cn from './classnames' + +describe('classnames', () => { + test('classnames libs feature', () => { + expect(cn('foo')).toBe('foo') + expect(cn('foo', 'bar')).toBe('foo bar') + expect(cn(['foo', 'bar'])).toBe('foo bar') + + expect(cn(undefined)).toBe('') + expect(cn(null)).toBe('') + expect(cn(false)).toBe('') + + expect(cn({ + foo: true, + bar: false, + baz: true, + })).toBe('foo baz') + }) + + test('tailwind-merge', () => { + expect(cn('p-0')).toBe('p-0') + expect(cn('text-right text-center text-left')).toBe('text-left') + expect(cn('pl-4 p-8')).toBe('p-8') + expect(cn('m-[2px] m-[4px]')).toBe('m-[4px]') + expect(cn('m-1 m-[4px]')).toBe('m-[4px]') + expect(cn('overflow-x-auto hover:overflow-x-hidden overflow-x-scroll')).toBe( + 'hover:overflow-x-hidden overflow-x-scroll', + ) + expect(cn('h-10 h-min')).toBe('h-min') + expect(cn('bg-grey-5 bg-hotpink')).toBe('bg-hotpink') + + expect(cn('hover:block hover:inline')).toBe('hover:inline') + + expect(cn('font-medium !font-bold')).toBe('font-medium !font-bold') + expect(cn('!font-medium !font-bold')).toBe('!font-bold') + + expect(cn('text-gray-100 text-primary-200')).toBe('text-primary-200') + expect(cn('text-some-unknown-color text-components-input-bg-disabled text-primary-200')).toBe('text-primary-200') + expect(cn('bg-some-unknown-color bg-components-input-bg-disabled bg-primary-200')).toBe('bg-primary-200') + + expect(cn('border-t border-white/10')).toBe('border-t border-white/10') + expect(cn('border-t border-white')).toBe('border-t border-white') + expect(cn('text-3.5xl text-black')).toBe('text-3.5xl text-black') + }) + + test('classnames combined with tailwind-merge', () => { + expect(cn('text-right', { + 'text-center': true, + })).toBe('text-center') + + expect(cn('text-right', { + 'text-center': false, + })).toBe('text-right') + }) +}) diff --git a/web/yarn.lock b/web/yarn.lock index 57e59d05b448da..16207b6ee07e18 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -2,11 +2,24 @@ # yarn lockfile v1 +"@adobe/css-tools@^4.4.0": + version "4.4.0" + resolved "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.4.0.tgz#728c484f4e10df03d5a3acd0d8adcbbebff8ad63" + integrity sha512-Ff9+ksdQQB3rMncgqDK78uLznstjyfIf2Arnh22pW8kBpLs6rpKDwgnZT46hin5Hl1WzazzK64DOrhSwYpS7bQ== + "@alloc/quick-lru@^5.2.0": version "5.2.0" resolved "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz" integrity sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw== +"@ampproject/remapping@^2.2.0": + version "2.3.0" + resolved "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz#ed441b6fa600072520ce18b43d2c8cc8caecc7f4" + integrity sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw== + dependencies: + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.24" + "@antfu/eslint-config-basic@0.36.0": version "0.36.0" resolved "https://registry.npmjs.org/@antfu/eslint-config-basic/-/eslint-config-basic-0.36.0.tgz" @@ -74,11 +87,150 @@ dependencies: "@babel/highlight" "^7.18.6" +"@babel/code-frame@^7.10.4", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.24.7": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.7.tgz#882fd9e09e8ee324e496bd040401c6f046ef4465" + integrity sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA== + dependencies: + "@babel/highlight" "^7.24.7" + picocolors "^1.0.0" + +"@babel/compat-data@^7.24.8": + version "7.24.9" + resolved "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.24.9.tgz#53eee4e68f1c1d0282aa0eb05ddb02d033fc43a0" + integrity sha512-e701mcfApCJqMMueQI0Fb68Amflj83+dvAvHawoBpAz+GDjCIyGHzNwnefjsWJ3xiYAqqiQFoWbspGYBdb2/ng== + +"@babel/core@^7.11.6", "@babel/core@^7.12.3", "@babel/core@^7.23.9": + version "7.24.9" + resolved "https://registry.npmjs.org/@babel/core/-/core-7.24.9.tgz#dc07c9d307162c97fa9484ea997ade65841c7c82" + integrity sha512-5e3FI4Q3M3Pbr21+5xJwCv6ZT6KmGkI0vw3Tozy5ODAQFTIWe37iT8Cr7Ice2Ntb+M3iSKCEWMB1MBgKrW3whg== + dependencies: + "@ampproject/remapping" "^2.2.0" + "@babel/code-frame" "^7.24.7" + "@babel/generator" "^7.24.9" + "@babel/helper-compilation-targets" "^7.24.8" + "@babel/helper-module-transforms" "^7.24.9" + "@babel/helpers" "^7.24.8" + "@babel/parser" "^7.24.8" + "@babel/template" "^7.24.7" + "@babel/traverse" "^7.24.8" + "@babel/types" "^7.24.9" + convert-source-map "^2.0.0" + debug "^4.1.0" + gensync "^1.0.0-beta.2" + json5 "^2.2.3" + semver "^6.3.1" + +"@babel/generator@^7.24.8", "@babel/generator@^7.24.9", "@babel/generator@^7.7.2": + version "7.24.10" + resolved "https://registry.npmjs.org/@babel/generator/-/generator-7.24.10.tgz#a4ab681ec2a78bbb9ba22a3941195e28a81d8e76" + integrity sha512-o9HBZL1G2129luEUlG1hB4N/nlYNWHnpwlND9eOMclRqqu1YDy2sSYVCFUZwl8I1Gxh+QSRrP2vD7EpUmFVXxg== + dependencies: + "@babel/types" "^7.24.9" + "@jridgewell/gen-mapping" "^0.3.5" + "@jridgewell/trace-mapping" "^0.3.25" + jsesc "^2.5.1" + +"@babel/helper-compilation-targets@^7.24.8": + version "7.24.8" + resolved "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.24.8.tgz#b607c3161cd9d1744977d4f97139572fe778c271" + integrity sha512-oU+UoqCHdp+nWVDkpldqIQL/i/bvAv53tRqLG/s+cOXxe66zOYLU7ar/Xs3LdmBihrUMEUhwu6dMZwbNOYDwvw== + dependencies: + "@babel/compat-data" "^7.24.8" + "@babel/helper-validator-option" "^7.24.8" + browserslist "^4.23.1" + lru-cache "^5.1.1" + semver "^6.3.1" + +"@babel/helper-environment-visitor@^7.24.7": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.24.7.tgz#4b31ba9551d1f90781ba83491dd59cf9b269f7d9" + integrity sha512-DoiN84+4Gnd0ncbBOM9AZENV4a5ZiL39HYMyZJGZ/AZEykHYdJw0wW3kdcsh9/Kn+BRXHLkkklZ51ecPKmI1CQ== + dependencies: + "@babel/types" "^7.24.7" + +"@babel/helper-function-name@^7.24.7": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.24.7.tgz#75f1e1725742f39ac6584ee0b16d94513da38dd2" + integrity sha512-FyoJTsj/PEUWu1/TYRiXTIHc8lbw+TDYkZuoE43opPS5TrI7MyONBE1oNvfguEXAD9yhQRrVBnXdXzSLQl9XnA== + dependencies: + "@babel/template" "^7.24.7" + "@babel/types" "^7.24.7" + +"@babel/helper-hoist-variables@^7.24.7": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.24.7.tgz#b4ede1cde2fd89436397f30dc9376ee06b0f25ee" + integrity sha512-MJJwhkoGy5c4ehfoRyrJ/owKeMl19U54h27YYftT0o2teQ3FJ3nQUf/I3LlJsX4l3qlw7WRXUmiyajvHXoTubQ== + dependencies: + "@babel/types" "^7.24.7" + +"@babel/helper-module-imports@^7.24.7": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.24.7.tgz#f2f980392de5b84c3328fc71d38bd81bbb83042b" + integrity sha512-8AyH3C+74cgCVVXow/myrynrAGv+nTVg5vKu2nZph9x7RcRwzmh0VFallJuFTZ9mx6u4eSdXZfcOzSqTUm0HCA== + dependencies: + "@babel/traverse" "^7.24.7" + "@babel/types" "^7.24.7" + +"@babel/helper-module-transforms@^7.24.9": + version "7.24.9" + resolved "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.24.9.tgz#e13d26306b89eea569180868e652e7f514de9d29" + integrity sha512-oYbh+rtFKj/HwBQkFlUzvcybzklmVdVV3UU+mN7n2t/q3yGHbuVdNxyFvSBO1tfvjyArpHNcWMAzsSPdyI46hw== + dependencies: + "@babel/helper-environment-visitor" "^7.24.7" + "@babel/helper-module-imports" "^7.24.7" + "@babel/helper-simple-access" "^7.24.7" + "@babel/helper-split-export-declaration" "^7.24.7" + "@babel/helper-validator-identifier" "^7.24.7" + +"@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.4", "@babel/helper-plugin-utils@^7.12.13", "@babel/helper-plugin-utils@^7.14.5", "@babel/helper-plugin-utils@^7.24.7", "@babel/helper-plugin-utils@^7.8.0": + version "7.24.8" + resolved "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.24.8.tgz#94ee67e8ec0e5d44ea7baeb51e571bd26af07878" + integrity sha512-FFWx5142D8h2Mgr/iPVGH5G7w6jDn4jUSpZTyDnQO0Yn7Ks2Kuz6Pci8H6MPCoUJegd/UZQ3tAvfLCxQSnWWwg== + +"@babel/helper-simple-access@^7.24.7": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.24.7.tgz#bcade8da3aec8ed16b9c4953b74e506b51b5edb3" + integrity sha512-zBAIvbCMh5Ts+b86r/CjU+4XGYIs+R1j951gxI3KmmxBMhCg4oQMsv6ZXQ64XOm/cvzfU1FmoCyt6+owc5QMYg== + dependencies: + "@babel/traverse" "^7.24.7" + "@babel/types" "^7.24.7" + +"@babel/helper-split-export-declaration@^7.24.7": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.24.7.tgz#83949436890e07fa3d6873c61a96e3bbf692d856" + integrity sha512-oy5V7pD+UvfkEATUKvIjvIAH/xCzfsFVw7ygW2SI6NClZzquT+mwdTfgfdbUiceh6iQO0CHtCPsyze/MZ2YbAA== + dependencies: + "@babel/types" "^7.24.7" + +"@babel/helper-string-parser@^7.24.8": + version "7.24.8" + resolved "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.8.tgz#5b3329c9a58803d5df425e5785865881a81ca48d" + integrity sha512-pO9KhhRcuUyGnJWwyEgnRJTSIZHiT+vMD0kPeD+so0l7mxkMT19g3pjY9GTnHySck/hDzq+dtW/4VgnMkippsQ== + "@babel/helper-validator-identifier@^7.18.6", "@babel/helper-validator-identifier@^7.19.1": version "7.19.1" resolved "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz" integrity sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w== +"@babel/helper-validator-identifier@^7.24.7": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz#75b889cfaf9e35c2aaf42cf0d72c8e91719251db" + integrity sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w== + +"@babel/helper-validator-option@^7.24.8": + version "7.24.8" + resolved "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.24.8.tgz#3725cdeea8b480e86d34df15304806a06975e33d" + integrity sha512-xb8t9tD1MHLungh/AIoWYN+gVHaB9kwlu8gffXGSt3FFEIT7RjS+xWbc2vUD1UTZdIpKj/ab3rdqJ7ufngyi2Q== + +"@babel/helpers@^7.24.8": + version "7.24.8" + resolved "https://registry.npmjs.org/@babel/helpers/-/helpers-7.24.8.tgz#2820d64d5d6686cca8789dd15b074cd862795873" + integrity sha512-gV2265Nkcz7weJJfvDoAEVzC1e2OTDpkGbEsebse8koXUJUXPsCMi7sRo/+SPMuMZ9MtUPnGwITTnQnU5YjyaQ== + dependencies: + "@babel/template" "^7.24.7" + "@babel/types" "^7.24.8" + "@babel/highlight@^7.18.6": version "7.18.6" resolved "https://registry.npmjs.org/@babel/highlight/-/highlight-7.18.6.tgz" @@ -88,11 +240,124 @@ chalk "^2.0.0" js-tokens "^4.0.0" +"@babel/highlight@^7.24.7": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/highlight/-/highlight-7.24.7.tgz#a05ab1df134b286558aae0ed41e6c5f731bf409d" + integrity sha512-EStJpq4OuY8xYfhGVXngigBJRWxftKX9ksiGDnmlY3o7B/V7KIAc9X4oiK87uPJSc/vs5L869bem5fhZa8caZw== + dependencies: + "@babel/helper-validator-identifier" "^7.24.7" + chalk "^2.4.2" + js-tokens "^4.0.0" + picocolors "^1.0.0" + +"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.23.9", "@babel/parser@^7.24.7", "@babel/parser@^7.24.8": + version "7.24.8" + resolved "https://registry.npmjs.org/@babel/parser/-/parser-7.24.8.tgz#58a4dbbcad7eb1d48930524a3fd93d93e9084c6f" + integrity sha512-WzfbgXOkGzZiXXCqk43kKwZjzwx4oulxZi3nq2TYL9mOjQv6kYwul9mz6ID36njuL7Xkp6nJEfok848Zj10j/w== + "@babel/parser@^7.24.4": version "7.24.4" resolved "https://registry.npmjs.org/@babel/parser/-/parser-7.24.4.tgz" integrity sha512-zTvEBcghmeBma9QIGunWevvBAp4/Qu9Bdq+2k0Ot4fVMD6v3dsC9WOcRSKk7tRRyBM/53yKMJko9xOatGQAwSg== +"@babel/plugin-syntax-async-generators@^7.8.4": + version "7.8.4" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz#a983fb1aeb2ec3f6ed042a210f640e90e786fe0d" + integrity sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-bigint@^7.8.3": + version "7.8.3" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz#4c9a6f669f5d0cdf1b90a1671e9a146be5300cea" + integrity sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-class-properties@^7.8.3": + version "7.12.13" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz#b5c987274c4a3a82b89714796931a6b53544ae10" + integrity sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-syntax-import-meta@^7.8.3": + version "7.10.4" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz#ee601348c370fa334d2207be158777496521fd51" + integrity sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g== + dependencies: + "@babel/helper-plugin-utils" "^7.10.4" + +"@babel/plugin-syntax-json-strings@^7.8.3": + version "7.8.3" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz#01ca21b668cd8218c9e640cb6dd88c5412b2c96a" + integrity sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-jsx@^7.7.2": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.24.7.tgz#39a1fa4a7e3d3d7f34e2acc6be585b718d30e02d" + integrity sha512-6ddciUPe/mpMnOKv/U+RSd2vvVy+Yw/JfBB0ZHYjEZt9NLHmCUylNYlsbqCCS1Bffjlb0fCwC9Vqz+sBz6PsiQ== + dependencies: + "@babel/helper-plugin-utils" "^7.24.7" + +"@babel/plugin-syntax-logical-assignment-operators@^7.8.3": + version "7.10.4" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz#ca91ef46303530448b906652bac2e9fe9941f699" + integrity sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig== + dependencies: + "@babel/helper-plugin-utils" "^7.10.4" + +"@babel/plugin-syntax-nullish-coalescing-operator@^7.8.3": + version "7.8.3" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz#167ed70368886081f74b5c36c65a88c03b66d1a9" + integrity sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-numeric-separator@^7.8.3": + version "7.10.4" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz#b9b070b3e33570cd9fd07ba7fa91c0dd37b9af97" + integrity sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug== + dependencies: + "@babel/helper-plugin-utils" "^7.10.4" + +"@babel/plugin-syntax-object-rest-spread@^7.8.3": + version "7.8.3" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz#60e225edcbd98a640332a2e72dd3e66f1af55871" + integrity sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-optional-catch-binding@^7.8.3": + version "7.8.3" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz#6111a265bcfb020eb9efd0fdfd7d26402b9ed6c1" + integrity sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-optional-chaining@^7.8.3": + version "7.8.3" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz#4f69c2ab95167e0180cd5336613f8c5788f7d48a" + integrity sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-top-level-await@^7.8.3": + version "7.14.5" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz#c1cfdadc35a646240001f06138247b741c34d94c" + integrity sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw== + dependencies: + "@babel/helper-plugin-utils" "^7.14.5" + +"@babel/plugin-syntax-typescript@^7.7.2": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.24.7.tgz#58d458271b4d3b6bb27ee6ac9525acbb259bad1c" + integrity sha512-c/+fVeJBB0FeKsFvwytYiUD+LBvhHjGSI0g446PRGdSVGZLRNArBUno2PETbAly3tpiNAQR5XaZ+JslxkotsbA== + dependencies: + "@babel/helper-plugin-utils" "^7.24.7" + "@babel/runtime@^7.0.0", "@babel/runtime@^7.10.1", "@babel/runtime@^7.11.1", "@babel/runtime@^7.12.5", "@babel/runtime@^7.18.3", "@babel/runtime@^7.20.6", "@babel/runtime@^7.20.7", "@babel/runtime@^7.21.0", "@babel/runtime@^7.21.5", "@babel/runtime@^7.22.3", "@babel/runtime@^7.3.1": version "7.22.3" resolved "https://registry.npmjs.org/@babel/runtime/-/runtime-7.22.3.tgz" @@ -100,11 +365,64 @@ dependencies: regenerator-runtime "^0.13.11" +"@babel/runtime@^7.9.2": + version "7.24.8" + resolved "https://registry.npmjs.org/@babel/runtime/-/runtime-7.24.8.tgz#5d958c3827b13cc6d05e038c07fb2e5e3420d82e" + integrity sha512-5F7SDGs1T72ZczbRwbGO9lQi0NLjQxzl6i4lJxLxfW9U5UluCSyEJeniWvnhl3/euNiqQVbo8zruhsDfid0esA== + dependencies: + regenerator-runtime "^0.14.0" + +"@babel/template@^7.24.7", "@babel/template@^7.3.3": + version "7.24.7" + resolved "https://registry.npmjs.org/@babel/template/-/template-7.24.7.tgz#02efcee317d0609d2c07117cb70ef8fb17ab7315" + integrity sha512-jYqfPrU9JTF0PmPy1tLYHW4Mp4KlgxJD9l2nP9fD6yT/ICi554DmrWBAEYpIelzjHf1msDP3PxJIRt/nFNfBig== + dependencies: + "@babel/code-frame" "^7.24.7" + "@babel/parser" "^7.24.7" + "@babel/types" "^7.24.7" + +"@babel/traverse@^7.24.7", "@babel/traverse@^7.24.8": + version "7.24.8" + resolved "https://registry.npmjs.org/@babel/traverse/-/traverse-7.24.8.tgz#6c14ed5232b7549df3371d820fbd9abfcd7dfab7" + integrity sha512-t0P1xxAPzEDcEPmjprAQq19NWum4K0EQPjMwZQZbHt+GiZqvjCHjj755Weq1YRPVzBI+3zSfvScfpnuIecVFJQ== + dependencies: + "@babel/code-frame" "^7.24.7" + "@babel/generator" "^7.24.8" + "@babel/helper-environment-visitor" "^7.24.7" + "@babel/helper-function-name" "^7.24.7" + "@babel/helper-hoist-variables" "^7.24.7" + "@babel/helper-split-export-declaration" "^7.24.7" + "@babel/parser" "^7.24.8" + "@babel/types" "^7.24.8" + debug "^4.3.1" + globals "^11.1.0" + +"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.24.7", "@babel/types@^7.24.8", "@babel/types@^7.24.9", "@babel/types@^7.3.3": + version "7.24.9" + resolved "https://registry.npmjs.org/@babel/types/-/types-7.24.9.tgz#228ce953d7b0d16646e755acf204f4cf3d08cc73" + integrity sha512-xm8XrMKz0IlUdocVbYJe0Z9xEgidU7msskG8BbhnTPK/HZ2z/7FP7ykqPgrUH+C+r414mNfNWam1f2vqOjqjYQ== + dependencies: + "@babel/helper-string-parser" "^7.24.8" + "@babel/helper-validator-identifier" "^7.24.7" + to-fast-properties "^2.0.0" + +"@bcoe/v8-coverage@^0.2.3": + version "0.2.3" + resolved "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" + integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== + "@braintree/sanitize-url@^6.0.1": version "6.0.4" resolved "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-6.0.4.tgz" integrity sha512-s3jaWicZd0pkP0jf5ysyHUI/RE7MHos6qlToFcGWXVp+ykHOy77OUMrfbgJ9it2C5bow7OIQwYYaHjk9XlBQ2A== +"@cspotcode/source-map-support@^0.8.0": + version "0.8.1" + resolved "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1" + integrity sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw== + dependencies: + "@jridgewell/trace-mapping" "0.3.9" + "@dagrejs/dagre@^1.1.2": version "1.1.2" resolved "https://registry.npmjs.org/@dagrejs/dagre/-/dagre-1.1.2.tgz" @@ -377,7 +695,215 @@ wrap-ansi "^8.1.0" wrap-ansi-cjs "npm:wrap-ansi@^7.0.0" -"@jridgewell/gen-mapping@^0.3.2": +"@istanbuljs/load-nyc-config@^1.0.0": + version "1.1.0" + resolved "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz#fd3db1d59ecf7cf121e80650bb86712f9b55eced" + integrity sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ== + dependencies: + camelcase "^5.3.1" + find-up "^4.1.0" + get-package-type "^0.1.0" + js-yaml "^3.13.1" + resolve-from "^5.0.0" + +"@istanbuljs/schema@^0.1.2", "@istanbuljs/schema@^0.1.3": + version "0.1.3" + resolved "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz#e45e384e4b8ec16bce2fd903af78450f6bf7ec98" + integrity sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA== + +"@jest/console@^29.7.0": + version "29.7.0" + resolved "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz#cd4822dbdb84529265c5a2bdb529a3c9cc950ffc" + integrity sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg== + dependencies: + "@jest/types" "^29.6.3" + "@types/node" "*" + chalk "^4.0.0" + jest-message-util "^29.7.0" + jest-util "^29.7.0" + slash "^3.0.0" + +"@jest/core@^29.7.0": + version "29.7.0" + resolved "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz#b6cccc239f30ff36609658c5a5e2291757ce448f" + integrity sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg== + dependencies: + "@jest/console" "^29.7.0" + "@jest/reporters" "^29.7.0" + "@jest/test-result" "^29.7.0" + "@jest/transform" "^29.7.0" + "@jest/types" "^29.6.3" + "@types/node" "*" + ansi-escapes "^4.2.1" + chalk "^4.0.0" + ci-info "^3.2.0" + exit "^0.1.2" + graceful-fs "^4.2.9" + jest-changed-files "^29.7.0" + jest-config "^29.7.0" + jest-haste-map "^29.7.0" + jest-message-util "^29.7.0" + jest-regex-util "^29.6.3" + jest-resolve "^29.7.0" + jest-resolve-dependencies "^29.7.0" + jest-runner "^29.7.0" + jest-runtime "^29.7.0" + jest-snapshot "^29.7.0" + jest-util "^29.7.0" + jest-validate "^29.7.0" + jest-watcher "^29.7.0" + micromatch "^4.0.4" + pretty-format "^29.7.0" + slash "^3.0.0" + strip-ansi "^6.0.0" + +"@jest/environment@^29.7.0": + version "29.7.0" + resolved "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz#24d61f54ff1f786f3cd4073b4b94416383baf2a7" + integrity sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw== + dependencies: + "@jest/fake-timers" "^29.7.0" + "@jest/types" "^29.6.3" + "@types/node" "*" + jest-mock "^29.7.0" + +"@jest/expect-utils@^29.7.0": + version "29.7.0" + resolved "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz#023efe5d26a8a70f21677d0a1afc0f0a44e3a1c6" + integrity sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA== + dependencies: + jest-get-type "^29.6.3" + +"@jest/expect@^29.7.0": + version "29.7.0" + resolved "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz#76a3edb0cb753b70dfbfe23283510d3d45432bf2" + integrity sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ== + dependencies: + expect "^29.7.0" + jest-snapshot "^29.7.0" + +"@jest/fake-timers@^29.7.0": + version "29.7.0" + resolved "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz#fd91bf1fffb16d7d0d24a426ab1a47a49881a565" + integrity sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ== + dependencies: + "@jest/types" "^29.6.3" + "@sinonjs/fake-timers" "^10.0.2" + "@types/node" "*" + jest-message-util "^29.7.0" + jest-mock "^29.7.0" + jest-util "^29.7.0" + +"@jest/globals@^29.7.0": + version "29.7.0" + resolved "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz#8d9290f9ec47ff772607fa864ca1d5a2efae1d4d" + integrity sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ== + dependencies: + "@jest/environment" "^29.7.0" + "@jest/expect" "^29.7.0" + "@jest/types" "^29.6.3" + jest-mock "^29.7.0" + +"@jest/reporters@^29.7.0": + version "29.7.0" + resolved "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz#04b262ecb3b8faa83b0b3d321623972393e8f4c7" + integrity sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg== + dependencies: + "@bcoe/v8-coverage" "^0.2.3" + "@jest/console" "^29.7.0" + "@jest/test-result" "^29.7.0" + "@jest/transform" "^29.7.0" + "@jest/types" "^29.6.3" + "@jridgewell/trace-mapping" "^0.3.18" + "@types/node" "*" + chalk "^4.0.0" + collect-v8-coverage "^1.0.0" + exit "^0.1.2" + glob "^7.1.3" + graceful-fs "^4.2.9" + istanbul-lib-coverage "^3.0.0" + istanbul-lib-instrument "^6.0.0" + istanbul-lib-report "^3.0.0" + istanbul-lib-source-maps "^4.0.0" + istanbul-reports "^3.1.3" + jest-message-util "^29.7.0" + jest-util "^29.7.0" + jest-worker "^29.7.0" + slash "^3.0.0" + string-length "^4.0.1" + strip-ansi "^6.0.0" + v8-to-istanbul "^9.0.1" + +"@jest/schemas@^29.6.3": + version "29.6.3" + resolved "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz#430b5ce8a4e0044a7e3819663305a7b3091c8e03" + integrity sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA== + dependencies: + "@sinclair/typebox" "^0.27.8" + +"@jest/source-map@^29.6.3": + version "29.6.3" + resolved "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz#d90ba772095cf37a34a5eb9413f1b562a08554c4" + integrity sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw== + dependencies: + "@jridgewell/trace-mapping" "^0.3.18" + callsites "^3.0.0" + graceful-fs "^4.2.9" + +"@jest/test-result@^29.7.0": + version "29.7.0" + resolved "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz#8db9a80aa1a097bb2262572686734baed9b1657c" + integrity sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA== + dependencies: + "@jest/console" "^29.7.0" + "@jest/types" "^29.6.3" + "@types/istanbul-lib-coverage" "^2.0.0" + collect-v8-coverage "^1.0.0" + +"@jest/test-sequencer@^29.7.0": + version "29.7.0" + resolved "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz#6cef977ce1d39834a3aea887a1726628a6f072ce" + integrity sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw== + dependencies: + "@jest/test-result" "^29.7.0" + graceful-fs "^4.2.9" + jest-haste-map "^29.7.0" + slash "^3.0.0" + +"@jest/transform@^29.7.0": + version "29.7.0" + resolved "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz#df2dd9c346c7d7768b8a06639994640c642e284c" + integrity sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw== + dependencies: + "@babel/core" "^7.11.6" + "@jest/types" "^29.6.3" + "@jridgewell/trace-mapping" "^0.3.18" + babel-plugin-istanbul "^6.1.1" + chalk "^4.0.0" + convert-source-map "^2.0.0" + fast-json-stable-stringify "^2.1.0" + graceful-fs "^4.2.9" + jest-haste-map "^29.7.0" + jest-regex-util "^29.6.3" + jest-util "^29.7.0" + micromatch "^4.0.4" + pirates "^4.0.4" + slash "^3.0.0" + write-file-atomic "^4.0.2" + +"@jest/types@^29.6.3": + version "29.6.3" + resolved "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz#1131f8cf634e7e84c5e77bab12f052af585fba59" + integrity sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw== + dependencies: + "@jest/schemas" "^29.6.3" + "@types/istanbul-lib-coverage" "^2.0.0" + "@types/istanbul-reports" "^3.0.0" + "@types/node" "*" + "@types/yargs" "^17.0.8" + chalk "^4.0.0" + +"@jridgewell/gen-mapping@^0.3.2", "@jridgewell/gen-mapping@^0.3.5": version "0.3.5" resolved "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz" integrity sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg== @@ -386,6 +912,11 @@ "@jridgewell/sourcemap-codec" "^1.4.10" "@jridgewell/trace-mapping" "^0.3.24" +"@jridgewell/resolve-uri@^3.0.3": + version "3.1.2" + resolved "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz#7a0ee601f60f99a20c7c7c5ff0c80388c1189bd6" + integrity sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw== + "@jridgewell/resolve-uri@^3.1.0": version "3.1.0" resolved "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz" @@ -401,7 +932,15 @@ resolved "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz" integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== -"@jridgewell/trace-mapping@^0.3.24": +"@jridgewell/trace-mapping@0.3.9": + version "0.3.9" + resolved "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz#6534fd5933a53ba7cbf3a17615e273a0d1273ff9" + integrity sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ== + dependencies: + "@jridgewell/resolve-uri" "^3.0.3" + "@jridgewell/sourcemap-codec" "^1.4.10" + +"@jridgewell/trace-mapping@^0.3.12", "@jridgewell/trace-mapping@^0.3.18", "@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25": version "0.3.25" resolved "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz" integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ== @@ -910,6 +1449,25 @@ "@sentry/types" "7.54.0" tslib "^1.9.3" +"@sinclair/typebox@^0.27.8": + version "0.27.8" + resolved "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz#6667fac16c436b5434a387a34dedb013198f6e6e" + integrity sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA== + +"@sinonjs/commons@^3.0.0": + version "3.0.1" + resolved "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz#1029357e44ca901a615585f6d27738dbc89084cd" + integrity sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ== + dependencies: + type-detect "4.0.8" + +"@sinonjs/fake-timers@^10.0.2": + version "10.3.0" + resolved "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz#55fdff1ecab9f354019129daf4df0dd4d923ea66" + integrity sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA== + dependencies: + "@sinonjs/commons" "^3.0.0" + "@swc/counter@^0.1.3": version "0.1.3" resolved "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz" @@ -938,6 +1496,66 @@ lodash.merge "^4.6.2" postcss-selector-parser "6.0.10" +"@testing-library/dom@^10.3.2": + version "10.3.2" + resolved "https://registry.npmjs.org/@testing-library/dom/-/dom-10.3.2.tgz#0285f643510d5ff4a0b12e4efa7f734a78d80aa3" + integrity sha512-0bxIdP9mmPiOJ6wHLj8bdJRq+51oddObeCGdEf6PNEhYd93ZYAN+lPRnEOVFtheVwDM7+p+tza3LAQgp0PTudg== + dependencies: + "@babel/code-frame" "^7.10.4" + "@babel/runtime" "^7.12.5" + "@types/aria-query" "^5.0.1" + aria-query "5.3.0" + chalk "^4.1.0" + dom-accessibility-api "^0.5.9" + lz-string "^1.5.0" + pretty-format "^27.0.2" + +"@testing-library/jest-dom@^6.4.6": + version "6.4.6" + resolved "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-6.4.6.tgz#ec1df8108651bed5475534955565bed88c6732ce" + integrity sha512-8qpnGVincVDLEcQXWaHOf6zmlbwTKc6Us6PPu4CRnPXCzo2OGBS5cwgMMOWdxDpEz1mkbvXHpEy99M5Yvt682w== + dependencies: + "@adobe/css-tools" "^4.4.0" + "@babel/runtime" "^7.9.2" + aria-query "^5.0.0" + chalk "^3.0.0" + css.escape "^1.5.1" + dom-accessibility-api "^0.6.3" + lodash "^4.17.21" + redent "^3.0.0" + +"@testing-library/react@^16.0.0": + version "16.0.0" + resolved "https://registry.npmjs.org/@testing-library/react/-/react-16.0.0.tgz#0a1e0c7a3de25841c3591b8cb7fb0cf0c0a27321" + integrity sha512-guuxUKRWQ+FgNX0h0NS0FIq3Q3uLtWVpBzcLOggmfMoUpgBnzBzvLLd4fbm6yS8ydJd94cIfY4yP9qUQjM2KwQ== + dependencies: + "@babel/runtime" "^7.12.5" + +"@tootallnate/once@2": + version "2.0.0" + resolved "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz#f544a148d3ab35801c1f633a7441fd87c2e484bf" + integrity sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A== + +"@tsconfig/node10@^1.0.7": + version "1.0.11" + resolved "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz#6ee46400685f130e278128c7b38b7e031ff5b2f2" + integrity sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw== + +"@tsconfig/node12@^1.0.7": + version "1.0.11" + resolved "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz#ee3def1f27d9ed66dac6e46a295cffb0152e058d" + integrity sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag== + +"@tsconfig/node14@^1.0.0": + version "1.0.3" + resolved "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz#e4386316284f00b98435bf40f72f75a09dabf6c1" + integrity sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow== + +"@tsconfig/node16@^1.0.2": + version "1.0.4" + resolved "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz#0b92dcc0cc1c81f6f306a381f28e31b1a56536e9" + integrity sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA== + "@types/acorn@^4.0.0": version "4.0.6" resolved "https://registry.npmjs.org/@types/acorn/-/acorn-4.0.6.tgz" @@ -945,6 +1563,44 @@ dependencies: "@types/estree" "*" +"@types/aria-query@^5.0.1": + version "5.0.4" + resolved "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz#1a31c3d378850d2778dabb6374d036dcba4ba708" + integrity sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw== + +"@types/babel__core@^7.1.14": + version "7.20.5" + resolved "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz#3df15f27ba85319caa07ba08d0721889bb39c017" + integrity sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA== + dependencies: + "@babel/parser" "^7.20.7" + "@babel/types" "^7.20.7" + "@types/babel__generator" "*" + "@types/babel__template" "*" + "@types/babel__traverse" "*" + +"@types/babel__generator@*": + version "7.6.8" + resolved "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.8.tgz#f836c61f48b1346e7d2b0d93c6dacc5b9535d3ab" + integrity sha512-ASsj+tpEDsEiFr1arWrlN6V3mdfjRMZt6LtK/Vp/kreFLnr5QH5+DhvD5nINYZXzwJvXeGq+05iUXcAzVrqWtw== + dependencies: + "@babel/types" "^7.0.0" + +"@types/babel__template@*": + version "7.4.4" + resolved "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz#5672513701c1b2199bc6dad636a9d7491586766f" + integrity sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A== + dependencies: + "@babel/parser" "^7.1.0" + "@babel/types" "^7.0.0" + +"@types/babel__traverse@*", "@types/babel__traverse@^7.0.6": + version "7.20.6" + resolved "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.6.tgz#8dc9f0ae0f202c08d8d4dab648912c8d6038e3f7" + integrity sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg== + dependencies: + "@babel/types" "^7.20.7" + "@types/crypto-js@^4.1.1": version "4.1.1" resolved "https://registry.npmjs.org/@types/crypto-js/-/crypto-js-4.1.1.tgz" @@ -1189,6 +1845,13 @@ resolved "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.14.tgz" integrity sha512-WCfD5Ht3ZesJUsONdhvm84dmzWOiOzOAqOncN0++w0lBw1o8OuDNJF2McvvCef/yBqb/HYRahp1BYtODFQ8bRg== +"@types/graceful-fs@^4.1.3": + version "4.1.9" + resolved "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz#2a06bc0f68a20ab37b3e36aa238be6abdf49e8b4" + integrity sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ== + dependencies: + "@types/node" "*" + "@types/hast@^2.0.0": version "2.3.4" resolved "https://registry.npmjs.org/@types/hast/-/hast-2.3.4.tgz" @@ -1196,6 +1859,33 @@ dependencies: "@types/unist" "*" +"@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1": + version "2.0.6" + resolved "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz#7739c232a1fee9b4d3ce8985f314c0c6d33549d7" + integrity sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w== + +"@types/istanbul-lib-report@*": + version "3.0.3" + resolved "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz#53047614ae72e19fc0401d872de3ae2b4ce350bf" + integrity sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA== + dependencies: + "@types/istanbul-lib-coverage" "*" + +"@types/istanbul-reports@^3.0.0": + version "3.0.4" + resolved "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz#0f03e3d2f670fbdac586e34b433783070cc16f54" + integrity sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ== + dependencies: + "@types/istanbul-lib-report" "*" + +"@types/jest@^29.5.12": + version "29.5.12" + resolved "https://registry.npmjs.org/@types/jest/-/jest-29.5.12.tgz#7f7dc6eb4cf246d2474ed78744b05d06ce025544" + integrity sha512-eDC8bTvT/QhYdxJAulQikueigY5AsdBRH2yDKW3yveW7svY3+DzN84/2NUgkw10RTiJbWqZrTtoGVdYlvFJdLw== + dependencies: + expect "^29.0.0" + pretty-format "^29.0.0" + "@types/js-cookie@^2.x.x": version "2.2.7" resolved "https://registry.npmjs.org/@types/js-cookie/-/js-cookie-2.2.7.tgz" @@ -1206,6 +1896,15 @@ resolved "https://registry.npmjs.org/@types/js-cookie/-/js-cookie-3.0.3.tgz" integrity sha512-Xe7IImK09HP1sv2M/aI+48a20VX+TdRJucfq4vfRVy6nWN8PYPOEnlMRSgxJAgYQIXJVL8dZ4/ilAM7dWNaOww== +"@types/jsdom@^20.0.0": + version "20.0.1" + resolved "https://registry.npmjs.org/@types/jsdom/-/jsdom-20.0.1.tgz#07c14bc19bd2f918c1929541cdaacae894744808" + integrity sha512-d0r18sZPmMQr1eG35u12FZfhIXNrnsPU/g5wvRKCUf/tOGilKKwYMYGqh33BNR6ba+2gkHw1EUiHoN3mn7E5IQ== + dependencies: + "@types/node" "*" + "@types/tough-cookie" "*" + parse5 "^7.0.0" + "@types/json-schema@^7.0.9": version "7.0.12" resolved "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.12.tgz" @@ -1346,6 +2045,16 @@ resolved "https://registry.npmjs.org/@types/sortablejs/-/sortablejs-1.15.1.tgz" integrity sha512-g/JwBNToh6oCTAwNS8UGVmjO7NLDKsejVhvE4x1eWiPTC3uCuNsa/TD4ssvX3du+MLiM+SHPNDuijp8y76JzLQ== +"@types/stack-utils@^2.0.0": + version "2.0.3" + resolved "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz#6209321eb2c1712a7e7466422b8cb1fc0d9dd5d8" + integrity sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw== + +"@types/tough-cookie@*": + version "4.0.5" + resolved "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz#cb6e2a691b70cb177c6e3ae9c1d2e8b2ea8cd304" + integrity sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA== + "@types/unist@*", "@types/unist@^2.0.0", "@types/unist@^2.0.2": version "2.0.6" resolved "https://registry.npmjs.org/@types/unist/-/unist-2.0.6.tgz" @@ -1356,6 +2065,18 @@ resolved "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz" integrity sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA== +"@types/yargs-parser@*": + version "21.0.3" + resolved "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz#815e30b786d2e8f0dcd85fd5bcf5e1a04d008f15" + integrity sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ== + +"@types/yargs@^17.0.8": + version "17.0.32" + resolved "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.32.tgz#030774723a2f7faafebf645f4e5a48371dca6229" + integrity sha512-xQ67Yc/laOG5uMfX/093MRlGGCIBzZMarVa+gfNKJxWAIgykYpVGkBdbqEzGDDfCrVUj6Hiff4mTZ5BA6TmAog== + dependencies: + "@types/yargs-parser" "*" + "@typescript-eslint/eslint-plugin@^5.53.0": version "5.59.9" resolved "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.59.9.tgz" @@ -1464,16 +2185,48 @@ resolved "https://registry.npmjs.org/@vue/shared/-/shared-3.4.25.tgz" integrity sha512-k0yappJ77g2+KNrIaF0FFnzwLvUBLUYr8VOwz+/6vLsmItFp51AcxLL7Ey3iPd7BIRyWPOcqUjMnm7OkahXllA== +abab@^2.0.6: + version "2.0.6" + resolved "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz#41b80f2c871d19686216b82309231cfd3cb3d291" + integrity sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA== + +acorn-globals@^7.0.0: + version "7.0.1" + resolved "https://registry.npmjs.org/acorn-globals/-/acorn-globals-7.0.1.tgz#0dbf05c44fa7c94332914c02066d5beff62c40c3" + integrity sha512-umOSDSDrfHbTNPuNpC2NSnnA3LUrqpevPb4T9jRx4MagXNS0rs+gwiTcAvqCRmsD6utzsrzNt+ebm00SNWiC3Q== + dependencies: + acorn "^8.1.0" + acorn-walk "^8.0.2" + acorn-jsx@^5.0.0, acorn-jsx@^5.3.2: version "5.3.2" resolved "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz" integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== +acorn-walk@^8.0.2, acorn-walk@^8.1.1: + version "8.3.3" + resolved "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.3.tgz#9caeac29eefaa0c41e3d4c65137de4d6f34df43e" + integrity sha512-MxXdReSRhGO7VlFe1bRG/oI7/mdLV9B9JJT0N8vZOhF7gFRR5l3M8W9G8JxmKV+JC5mGqJ0QvqfSOLsCPa4nUw== + dependencies: + acorn "^8.11.0" + acorn@^8.0.0, acorn@^8.5.0, acorn@^8.8.0: version "8.8.2" resolved "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz" integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== +acorn@^8.1.0, acorn@^8.11.0, acorn@^8.4.1, acorn@^8.8.1: + version "8.12.1" + resolved "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz#71616bdccbe25e27a54439e0046e89ca76df2248" + integrity sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg== + +agent-base@6: + version "6.0.2" + resolved "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" + integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== + dependencies: + debug "4" + aggregate-error@^3.0.0: version "3.1.0" resolved "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz" @@ -1513,7 +2266,7 @@ ajv@^6.10.0, ajv@^6.12.4: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ansi-escapes@^4.3.0: +ansi-escapes@^4.2.1, ansi-escapes@^4.3.0: version "4.3.2" resolved "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz" integrity sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ== @@ -1544,6 +2297,11 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0: dependencies: color-convert "^2.0.1" +ansi-styles@^5.0.0: + version "5.2.0" + resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz#07449690ad45777d1924ac2abb2fc8895dba836b" + integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA== + ansi-styles@^6.0.0, ansi-styles@^6.1.0: version "6.2.1" resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz" @@ -1554,7 +2312,7 @@ any-promise@^1.0.0: resolved "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz" integrity sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A== -anymatch@~3.1.2: +anymatch@^3.0.3, anymatch@~3.1.2: version "3.1.3" resolved "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz" integrity sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw== @@ -1562,16 +2320,35 @@ anymatch@~3.1.2: normalize-path "^3.0.0" picomatch "^2.0.4" +arg@^4.1.0: + version "4.1.3" + resolved "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089" + integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA== + arg@^5.0.2: version "5.0.2" resolved "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz" integrity sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg== +argparse@^1.0.7: + version "1.0.10" + resolved "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" + integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== + dependencies: + sprintf-js "~1.0.2" + argparse@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== +aria-query@5.3.0, aria-query@^5.0.0: + version "5.3.0" + resolved "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz#650c569e41ad90b51b3d7df5e5eed1c7549c103e" + integrity sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A== + dependencies: + dequal "^2.0.3" + aria-query@^5.1.3: version "5.1.3" resolved "https://registry.npmjs.org/aria-query/-/aria-query-5.1.3.tgz" @@ -1687,6 +2464,11 @@ asynciterator.prototype@^1.0.0: dependencies: has-symbols "^1.0.3" +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== + autoprefixer@^10.4.14: version "10.4.14" resolved "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.14.tgz" @@ -1716,6 +2498,66 @@ axobject-query@^3.1.1: dependencies: deep-equal "^2.0.5" +babel-jest@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz#f4369919225b684c56085998ac63dbd05be020d5" + integrity sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg== + dependencies: + "@jest/transform" "^29.7.0" + "@types/babel__core" "^7.1.14" + babel-plugin-istanbul "^6.1.1" + babel-preset-jest "^29.6.3" + chalk "^4.0.0" + graceful-fs "^4.2.9" + slash "^3.0.0" + +babel-plugin-istanbul@^6.1.1: + version "6.1.1" + resolved "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz#fa88ec59232fd9b4e36dbbc540a8ec9a9b47da73" + integrity sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA== + dependencies: + "@babel/helper-plugin-utils" "^7.0.0" + "@istanbuljs/load-nyc-config" "^1.0.0" + "@istanbuljs/schema" "^0.1.2" + istanbul-lib-instrument "^5.0.4" + test-exclude "^6.0.0" + +babel-plugin-jest-hoist@^29.6.3: + version "29.6.3" + resolved "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz#aadbe943464182a8922c3c927c3067ff40d24626" + integrity sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg== + dependencies: + "@babel/template" "^7.3.3" + "@babel/types" "^7.3.3" + "@types/babel__core" "^7.1.14" + "@types/babel__traverse" "^7.0.6" + +babel-preset-current-node-syntax@^1.0.0: + version "1.0.1" + resolved "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.0.1.tgz#b4399239b89b2a011f9ddbe3e4f401fc40cff73b" + integrity sha512-M7LQ0bxarkxQoN+vz5aJPsLBn77n8QgTFmo8WK0/44auK2xlCXrYcUxHFxgU7qW5Yzw/CjmLRK2uJzaCd7LvqQ== + dependencies: + "@babel/plugin-syntax-async-generators" "^7.8.4" + "@babel/plugin-syntax-bigint" "^7.8.3" + "@babel/plugin-syntax-class-properties" "^7.8.3" + "@babel/plugin-syntax-import-meta" "^7.8.3" + "@babel/plugin-syntax-json-strings" "^7.8.3" + "@babel/plugin-syntax-logical-assignment-operators" "^7.8.3" + "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.3" + "@babel/plugin-syntax-numeric-separator" "^7.8.3" + "@babel/plugin-syntax-object-rest-spread" "^7.8.3" + "@babel/plugin-syntax-optional-catch-binding" "^7.8.3" + "@babel/plugin-syntax-optional-chaining" "^7.8.3" + "@babel/plugin-syntax-top-level-await" "^7.8.3" + +babel-preset-jest@^29.6.3: + version "29.6.3" + resolved "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz#fa05fa510e7d493896d7b0dd2033601c840f171c" + integrity sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA== + dependencies: + babel-plugin-jest-hoist "^29.6.3" + babel-preset-current-node-syntax "^1.0.0" + bail@^2.0.0: version "2.0.2" resolved "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz" @@ -1780,6 +2622,28 @@ browserslist@^4.21.5: node-releases "^2.0.14" update-browserslist-db "^1.0.13" +browserslist@^4.23.1: + version "4.23.2" + resolved "https://registry.npmjs.org/browserslist/-/browserslist-4.23.2.tgz#244fe803641f1c19c28c48c4b6ec9736eb3d32ed" + integrity sha512-qkqSyistMYdxAcw+CzbZwlBy8AGmS/eEWs+sEV5TnLRGDOL+C5M2EnH6tlZyg0YoAxGJAFKh61En9BR941GnHA== + dependencies: + caniuse-lite "^1.0.30001640" + electron-to-chromium "^1.4.820" + node-releases "^2.0.14" + update-browserslist-db "^1.1.0" + +bser@2.1.1: + version "2.1.1" + resolved "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz#e6787da20ece9d07998533cfd9de6f5c38f4bc05" + integrity sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ== + dependencies: + node-int64 "^0.4.0" + +buffer-from@^1.0.0: + version "1.1.2" + resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" + integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== + builtin-modules@^3.3.0: version "3.3.0" resolved "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.3.0.tgz" @@ -1825,11 +2689,26 @@ camelcase-css@^2.0.1: resolved "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz" integrity sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA== +camelcase@^5.3.1: + version "5.3.1" + resolved "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" + integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== + +camelcase@^6.2.0: + version "6.3.0" + resolved "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" + integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== + caniuse-lite@^1.0.30001464, caniuse-lite@^1.0.30001579, caniuse-lite@^1.0.30001587: version "1.0.30001620" resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001620.tgz" integrity sha512-WJvYsOjd1/BYUY6SNGUosK9DUidBPDTnOARHp3fSmFO1ekdxaY6nKRttEVrfMmYi80ctS0kz1wiWmm14fVc3ew== +caniuse-lite@^1.0.30001640: + version "1.0.30001642" + resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001642.tgz#6aa6610eb24067c246d30c57f055a9d0a7f8d05f" + integrity sha512-3XQ0DoRgLijXJErLSl+bLnJ+Et4KqV1PY6JJBGAFlsNsz31zeAIncyeZfLCabHK/jtSh+671RM9YMldxjUPZtA== + ccount@^2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz" @@ -1848,7 +2727,7 @@ chalk@5.2.0: resolved "https://registry.npmjs.org/chalk/-/chalk-5.2.0.tgz" integrity sha512-ree3Gqw/nazQAPuJJEy+avdl7QfZMcUvmHIKgEZkGL+xOBzRvup5Hxo6LHuMceSxOabuJLJm5Yp/92R9eMmMvA== -chalk@^2.0.0: +chalk@^2.0.0, chalk@^2.4.2: version "2.4.2" resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== @@ -1857,6 +2736,27 @@ chalk@^2.0.0: escape-string-regexp "^1.0.5" supports-color "^5.3.0" +chalk@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz#3f73c2bf526591f574cc492c51e2456349f844e4" + integrity sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + +chalk@^4.1.0: + version "4.1.2" + resolved "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" + integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + +char-regex@^1.0.2: + version "1.0.2" + resolved "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf" + integrity sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw== + character-entities-html4@^2.0.0: version "2.1.0" resolved "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz" @@ -1907,11 +2807,21 @@ character-reference-invalid@^2.0.0: optionalDependencies: fsevents "~2.3.2" +ci-info@^3.2.0: + version "3.9.0" + resolved "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz#4279a62028a7b1f262f3473fc9605f5e218c59b4" + integrity sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ== + ci-info@^3.6.1: version "3.8.0" resolved "https://registry.npmjs.org/ci-info/-/ci-info-3.8.0.tgz" integrity sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw== +cjs-module-lexer@^1.0.0: + version "1.3.1" + resolved "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.3.1.tgz#c485341ae8fd999ca4ee5af2d7a1c9ae01e0099c" + integrity sha512-a3KdPAANPbNE4ZUv9h6LckSl9zLsYOP4MBmhIPkRaeyybt+r4UghLvq+xw/YwUcC1gqylCkL4rdVs3Lwupjm4Q== + class-variance-authority@^0.7.0: version "0.7.0" resolved "https://registry.npmjs.org/class-variance-authority/-/class-variance-authority-0.7.0.tgz" @@ -1974,11 +2884,25 @@ client-only@0.0.1, client-only@^0.0.1: resolved "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz" integrity sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA== +cliui@^8.0.1: + version "8.0.1" + resolved "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz#0c04b075db02cbfe60dc8e6cf2f5486b1a3608aa" + integrity sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ== + dependencies: + string-width "^4.2.0" + strip-ansi "^6.0.1" + wrap-ansi "^7.0.0" + clsx@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/clsx/-/clsx-2.0.0.tgz" integrity sha512-rQ1+kcj+ttHG0MKVGBUXwayCCF1oh39BF5COIpRzuCEv8Mwjv0XucrI2ExNTOn9IlLifGClWQcU9BrZORvtw6Q== +co@^4.6.0: + version "4.6.0" + resolved "https://registry.npmjs.org/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" + integrity sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ== + code-inspector-core@0.13.0: version "0.13.0" resolved "https://registry.npmjs.org/code-inspector-core/-/code-inspector-core-0.13.0.tgz" @@ -1998,6 +2922,11 @@ code-inspector-plugin@^0.13.0: vite-code-inspector-plugin "0.13.0" webpack-code-inspector-plugin "0.13.0" +collect-v8-coverage@^1.0.0: + version "1.0.2" + resolved "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.2.tgz#c0b29bcd33bcd0779a1344c2136051e6afd3d9e9" + integrity sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q== + color-convert@^1.9.0: version "1.9.3" resolved "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz" @@ -2043,6 +2972,13 @@ colorette@^2.0.19: resolved "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz" integrity sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w== +combined-stream@^1.0.8: + version "1.0.8" + resolved "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + comma-separated-tokens@^1.0.0: version "1.0.8" resolved "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz" @@ -2078,6 +3014,11 @@ concat-map@0.0.1: resolved "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== +convert-source-map@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz#4b560f649fc4e918dd0ab75cf4961e8bc882d82a" + integrity sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg== + copy-to-clipboard@^3.3.3: version "3.3.3" resolved "https://registry.npmjs.org/copy-to-clipboard/-/copy-to-clipboard-3.3.3.tgz" @@ -2099,6 +3040,24 @@ cose-base@^2.2.0: dependencies: layout-base "^2.0.0" +create-jest@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz#a355c5b3cb1e1af02ba177fe7afd7feee49a5320" + integrity sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q== + dependencies: + "@jest/types" "^29.6.3" + chalk "^4.0.0" + exit "^0.1.2" + graceful-fs "^4.2.9" + jest-config "^29.7.0" + jest-util "^29.7.0" + prompts "^2.0.1" + +create-require@^1.1.0: + version "1.1.1" + resolved "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" + integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== + cross-env@^7.0.3: version "7.0.3" resolved "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz" @@ -2120,11 +3079,33 @@ crypto-js@^4.2.0: resolved "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz" integrity sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q== +css.escape@^1.5.1: + version "1.5.1" + resolved "https://registry.npmjs.org/css.escape/-/css.escape-1.5.1.tgz#42e27d4fa04ae32f931a4b4d4191fa9cddee97cb" + integrity sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg== + cssesc@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz" integrity sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg== +cssom@^0.5.0: + version "0.5.0" + resolved "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz#d254fa92cd8b6fbd83811b9fbaed34663cc17c36" + integrity sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw== + +cssom@~0.3.6: + version "0.3.8" + resolved "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz#9f1276f5b2b463f2114d3f2c75250af8c1a36f4a" + integrity sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg== + +cssstyle@^2.3.0: + version "2.3.0" + resolved "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz#ff665a0ddbdc31864b09647f34163443d90b0852" + integrity sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A== + dependencies: + cssom "~0.3.6" + csstype@^3.0.2: version "3.1.2" resolved "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz" @@ -2436,11 +3417,27 @@ damerau-levenshtein@^1.0.8: resolved "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz" integrity sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA== +data-urls@^3.0.2: + version "3.0.2" + resolved "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz#9cf24a477ae22bcef5cd5f6f0bfbc1d2d3be9143" + integrity sha512-Jy/tj3ldjZJo63sVAvg6LHt2mHvl4V6AgRAmNDtLdm7faqtsx+aJG42rsyCo9JCoRVKwPFzKlIPx3DIibwSIaQ== + dependencies: + abab "^2.0.6" + whatwg-mimetype "^3.0.0" + whatwg-url "^11.0.0" + dayjs@^1.11.7, dayjs@^1.9.1: version "1.11.8" resolved "https://registry.npmjs.org/dayjs/-/dayjs-1.11.8.tgz" integrity sha512-LcgxzFoWMEPO7ggRv1Y2N31hUf2R0Vj7fuy/m+Bg1K8rr+KAs1AEy4y9jd5DXe8pbHgX+srkHNS7TH6Q6ZhYeQ== +debug@4, debug@^4.1.0, debug@^4.3.1: + version "4.3.5" + resolved "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz#e83444eceb9fedd4a1da56d671ae2446a01a6e1e" + integrity sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg== + dependencies: + ms "2.1.2" + debug@^3.2.7: version "3.2.7" resolved "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz" @@ -2455,6 +3452,11 @@ debug@^4.0.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.4: dependencies: ms "2.1.2" +decimal.js@^10.4.2: + version "10.4.3" + resolved "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz#1044092884d245d1b7f65725fa4ad4c6f781cc23" + integrity sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA== + decode-named-character-reference@^1.0.0: version "1.0.2" resolved "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz" @@ -2462,6 +3464,11 @@ decode-named-character-reference@^1.0.0: dependencies: character-entities "^2.0.0" +dedent@^1.0.0: + version "1.5.3" + resolved "https://registry.npmjs.org/dedent/-/dedent-1.5.3.tgz#99aee19eb9bae55a67327717b6e848d0bf777e5a" + integrity sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ== + deep-equal@^2.0.5: version "2.2.1" resolved "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.1.tgz" @@ -2491,6 +3498,11 @@ deep-is@^0.1.3: resolved "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz" integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== +deepmerge@^4.2.2: + version "4.3.1" + resolved "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a" + integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A== + default-browser-id@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/default-browser-id/-/default-browser-id-3.0.0.tgz" @@ -2539,7 +3551,12 @@ delaunator@5: dependencies: robust-predicates "^3.0.0" -dequal@^2.0.0: +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== + +dequal@^2.0.0, dequal@^2.0.3: version "2.0.3" resolved "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz" integrity sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA== @@ -2549,11 +3566,26 @@ detect-libc@^2.0.2: resolved "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz" integrity sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw== +detect-newline@^3.0.0: + version "3.1.0" + resolved "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" + integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== + didyoumean@^1.2.2: version "1.2.2" resolved "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz" integrity sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw== +diff-sequences@^29.6.3: + version "29.6.3" + resolved "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz#4deaf894d11407c51efc8418012f9e70b84ea921" + integrity sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q== + +diff@^4.0.1: + version "4.0.2" + resolved "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" + integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== + diff@^5.0.0: version "5.1.0" resolved "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz" @@ -2585,6 +3617,16 @@ doctrine@^3.0.0: dependencies: esutils "^2.0.2" +dom-accessibility-api@^0.5.9: + version "0.5.16" + resolved "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz#5a7429e6066eb3664d911e33fb0e45de8eb08453" + integrity sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg== + +dom-accessibility-api@^0.6.3: + version "0.6.3" + resolved "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.6.3.tgz#993e925cc1d73f2c662e7d75dd5a5445259a8fd8" + integrity sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w== + dom-serializer@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz" @@ -2599,6 +3641,13 @@ domelementtype@^2.3.0: resolved "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz" integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw== +domexception@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz#4ad1be56ccadc86fc76d033353999a8037d03673" + integrity sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw== + dependencies: + webidl-conversions "^7.0.0" + domhandler@^5.0.2, domhandler@^5.0.3: version "5.0.3" resolved "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz" @@ -2646,11 +3695,21 @@ electron-to-chromium@^1.4.668: resolved "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.775.tgz" integrity sha512-JpOfl1aNAiZ88wFzjPczTLwYIoPIsij8S9/XQH9lqMpiJOf23kxea68B8wje4f68t4rOIq4Bh+vP4I65njiJBw== +electron-to-chromium@^1.4.820: + version "1.4.829" + resolved "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.829.tgz#3034a865b5eac9064c9db8b38ba99b60a446bb73" + integrity sha512-5qp1N2POAfW0u1qGAxXEtz6P7bO1m6gpZr5hdf5ve6lxpLM7MpiM4jIPz7xcrNlClQMafbyUDDWjlIQZ1Mw0Rw== + elkjs@^0.8.2: version "0.8.2" resolved "https://registry.npmjs.org/elkjs/-/elkjs-0.8.2.tgz" integrity sha512-L6uRgvZTH+4OF5NE/MBbzQx/WYpru1xCBE9respNj6qznEewGUIfhzmm7horWWxbNO2M0WckQypGctR8lH79xQ== +emittery@^0.13.1: + version "0.13.1" + resolved "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz#c04b8c3457490e0847ae51fced3af52d338e3dad" + integrity sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ== + emoji-mart@^5.5.2: version "5.5.2" resolved "https://registry.npmjs.org/emoji-mart/-/emoji-mart-5.5.2.tgz" @@ -2791,7 +3850,7 @@ es-to-primitive@^1.2.1: is-date-object "^1.0.1" is-symbol "^1.0.2" -escalade@^3.1.2: +escalade@^3.1.1, escalade@^3.1.2: version "3.1.2" resolved "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz" integrity sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA== @@ -2801,6 +3860,11 @@ escape-string-regexp@^1.0.5: resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" integrity sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg== +escape-string-regexp@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz#a30304e99daa32e23b2fd20f51babd07cffca344" + integrity sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w== + escape-string-regexp@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz" @@ -2811,6 +3875,17 @@ escape-string-regexp@^5.0.0: resolved "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz" integrity sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw== +escodegen@^2.0.0: + version "2.1.0" + resolved "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz#ba93bbb7a43986d29d6041f99f5262da773e2e17" + integrity sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w== + dependencies: + esprima "^4.0.1" + estraverse "^5.2.0" + esutils "^2.0.2" + optionalDependencies: + source-map "~0.6.1" + eslint-config-next@^14.0.4: version "14.1.0" resolved "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-14.1.0.tgz" @@ -3162,6 +4237,11 @@ espree@^9.0.0, espree@^9.3.1, espree@^9.5.0, espree@^9.5.2: acorn-jsx "^5.3.2" eslint-visitor-keys "^3.4.1" +esprima@^4.0.0, esprima@^4.0.1: + version "4.0.1" + resolved "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" + integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== + esquery@^1.4.0, esquery@^1.4.2: version "1.5.0" resolved "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz" @@ -3271,6 +4351,22 @@ execa@^7.0.0, execa@^7.1.1: signal-exit "^3.0.7" strip-final-newline "^3.0.0" +exit@^0.1.2: + version "0.1.2" + resolved "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" + integrity sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ== + +expect@^29.0.0, expect@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz#578874590dcb3214514084c08115d8aee61e11bc" + integrity sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw== + dependencies: + "@jest/expect-utils" "^29.7.0" + jest-get-type "^29.6.3" + jest-matcher-utils "^29.7.0" + jest-message-util "^29.7.0" + jest-util "^29.7.0" + extend@^3.0.0: version "3.0.2" resolved "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz" @@ -3303,7 +4399,7 @@ fast-glob@^3.2.9, fast-glob@^3.3.0: merge2 "^1.3.0" micromatch "^4.0.4" -fast-json-stable-stringify@^2.0.0: +fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== @@ -3327,6 +4423,13 @@ fault@^1.0.0: dependencies: format "^0.2.0" +fb-watchman@^2.0.0: + version "2.0.2" + resolved "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz#e9524ee6b5c77e9e5001af0f85f3adbb8623255c" + integrity sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA== + dependencies: + bser "2.1.1" + file-entry-cache@^6.0.1: version "6.0.1" resolved "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz" @@ -3341,7 +4444,7 @@ fill-range@^7.1.1: dependencies: to-regex-range "^5.0.1" -find-up@^4.1.0: +find-up@^4.0.0, find-up@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz" integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw== @@ -3385,6 +4488,15 @@ foreground-child@^3.1.0: cross-spawn "^7.0.0" signal-exit "^4.0.1" +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + format@^0.2.0: version "0.2.2" resolved "https://registry.npmjs.org/format/-/format-0.2.2.tgz" @@ -3400,7 +4512,7 @@ fs.realpath@^1.0.0: resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== -fsevents@~2.3.2: +fsevents@^2.3.2, fsevents@~2.3.2: version "2.3.3" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== @@ -3425,6 +4537,16 @@ functions-have-names@^1.2.3: resolved "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz" integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ== +gensync@^1.0.0-beta.2: + version "1.0.0-beta.2" + resolved "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" + integrity sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg== + +get-caller-file@^2.0.5: + version "2.0.5" + resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" + integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== + get-intrinsic@^1.0.2, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3, get-intrinsic@^1.2.0, get-intrinsic@^1.2.1, get-intrinsic@^1.2.2: version "1.2.2" resolved "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz" @@ -3435,6 +4557,11 @@ get-intrinsic@^1.0.2, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3, get-intrinsic@ has-symbols "^1.0.3" hasown "^2.0.0" +get-package-type@^0.1.0: + version "0.1.0" + resolved "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" + integrity sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q== + get-stream@^6.0.0, get-stream@^6.0.1: version "6.0.1" resolved "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz" @@ -3492,7 +4619,7 @@ glob@7.1.6: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.1.3: +glob@^7.1.3, glob@^7.1.4: version "7.2.3" resolved "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz" integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== @@ -3504,6 +4631,11 @@ glob@^7.1.3: once "^1.3.0" path-is-absolute "^1.0.0" +globals@^11.1.0: + version "11.12.0" + resolved "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" + integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== + globals@^13.19.0: version "13.20.0" resolved "https://registry.npmjs.org/globals/-/globals-13.20.0.tgz" @@ -3548,7 +4680,7 @@ gopd@^1.0.1: dependencies: get-intrinsic "^1.1.3" -graceful-fs@^4.2.11, graceful-fs@^4.2.4: +graceful-fs@^4.2.11, graceful-fs@^4.2.4, graceful-fs@^4.2.9: version "4.2.11" resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== @@ -3753,6 +4885,18 @@ hosted-git-info@^2.1.4: resolved "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz" integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw== +html-encoding-sniffer@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz#2cb1a8cf0db52414776e5b2a7a04d5dd98158de9" + integrity sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA== + dependencies: + whatwg-encoding "^2.0.0" + +html-escaper@^2.0.0: + version "2.0.2" + resolved "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" + integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg== + html-parse-stringify@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/html-parse-stringify/-/html-parse-stringify-3.0.1.tgz" @@ -3770,6 +4914,23 @@ htmlparser2@^8.0.1: domutils "^3.0.1" entities "^4.4.0" +http-proxy-agent@^5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz#5129800203520d434f142bc78ff3c170800f2b43" + integrity sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w== + dependencies: + "@tootallnate/once" "2" + agent-base "6" + debug "4" + +https-proxy-agent@^5.0.1: + version "5.0.1" + resolved "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" + integrity sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA== + dependencies: + agent-base "6" + debug "4" + human-signals@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz" @@ -3799,7 +4960,7 @@ i18next@^22.4.13: dependencies: "@babel/runtime" "^7.20.6" -iconv-lite@0.6: +iconv-lite@0.6, iconv-lite@0.6.3: version "0.6.3" resolved "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz" integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== @@ -3826,8 +4987,16 @@ import-fresh@^3.0.0, import-fresh@^3.2.1: resolved "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz" integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== dependencies: - parent-module "^1.0.0" - resolve-from "^4.0.0" + parent-module "^1.0.0" + resolve-from "^4.0.0" + +import-local@^3.0.2: + version "3.1.0" + resolved "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz#b4479df8a5fd44f6cdce24070675676063c95cb4" + integrity sha512-ASB07uLtnDs1o6EHjKpX34BKYDSqnFerfTOJL2HvMqF70LnxpjkzDB8J44oT9pu4AMPkQwf8jl6szgvNd2tRIg== + dependencies: + pkg-dir "^4.2.0" + resolve-cwd "^3.0.0" imurmurhash@^0.1.4: version "0.1.4" @@ -4036,6 +5205,11 @@ is-fullwidth-code-point@^4.0.0: resolved "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-4.0.0.tgz" integrity sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ== +is-generator-fn@^2.0.0: + version "2.1.0" + resolved "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz#7d140adc389aaf3011a8f2a2a4cfa6faadffb118" + integrity sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ== + is-generator-function@^1.0.10: version "1.0.10" resolved "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz" @@ -4099,6 +5273,11 @@ is-plain-obj@^4.0.0: resolved "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz" integrity sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg== +is-potential-custom-element-name@^1.0.1: + version "1.0.1" + resolved "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz#171ed6f19e3ac554394edf78caa05784a45bebb5" + integrity sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ== + is-reference@^3.0.0: version "3.0.1" resolved "https://registry.npmjs.org/is-reference/-/is-reference-3.0.1.tgz" @@ -4194,6 +5373,59 @@ isexe@^2.0.0: resolved "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz" integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== +istanbul-lib-coverage@^3.0.0, istanbul-lib-coverage@^3.2.0: + version "3.2.2" + resolved "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz#2d166c4b0644d43a39f04bf6c2edd1e585f31756" + integrity sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg== + +istanbul-lib-instrument@^5.0.4: + version "5.2.1" + resolved "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz#d10c8885c2125574e1c231cacadf955675e1ce3d" + integrity sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg== + dependencies: + "@babel/core" "^7.12.3" + "@babel/parser" "^7.14.7" + "@istanbuljs/schema" "^0.1.2" + istanbul-lib-coverage "^3.2.0" + semver "^6.3.0" + +istanbul-lib-instrument@^6.0.0: + version "6.0.3" + resolved "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz#fa15401df6c15874bcb2105f773325d78c666765" + integrity sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q== + dependencies: + "@babel/core" "^7.23.9" + "@babel/parser" "^7.23.9" + "@istanbuljs/schema" "^0.1.3" + istanbul-lib-coverage "^3.2.0" + semver "^7.5.4" + +istanbul-lib-report@^3.0.0: + version "3.0.1" + resolved "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz#908305bac9a5bd175ac6a74489eafd0fc2445a7d" + integrity sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw== + dependencies: + istanbul-lib-coverage "^3.0.0" + make-dir "^4.0.0" + supports-color "^7.1.0" + +istanbul-lib-source-maps@^4.0.0: + version "4.0.1" + resolved "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz#895f3a709fcfba34c6de5a42939022f3e4358551" + integrity sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw== + dependencies: + debug "^4.1.1" + istanbul-lib-coverage "^3.0.0" + source-map "^0.6.1" + +istanbul-reports@^3.1.3: + version "3.1.7" + resolved "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz#daed12b9e1dca518e15c056e1e537e741280fa0b" + integrity sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g== + dependencies: + html-escaper "^2.0.0" + istanbul-lib-report "^3.0.0" + iterator.prototype@^1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.2.tgz" @@ -4214,6 +5446,378 @@ jackspeak@^2.3.5: optionalDependencies: "@pkgjs/parseargs" "^0.11.0" +jest-changed-files@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz#1c06d07e77c78e1585d020424dedc10d6e17ac3a" + integrity sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w== + dependencies: + execa "^5.0.0" + jest-util "^29.7.0" + p-limit "^3.1.0" + +jest-circus@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz#b6817a45fcc835d8b16d5962d0c026473ee3668a" + integrity sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw== + dependencies: + "@jest/environment" "^29.7.0" + "@jest/expect" "^29.7.0" + "@jest/test-result" "^29.7.0" + "@jest/types" "^29.6.3" + "@types/node" "*" + chalk "^4.0.0" + co "^4.6.0" + dedent "^1.0.0" + is-generator-fn "^2.0.0" + jest-each "^29.7.0" + jest-matcher-utils "^29.7.0" + jest-message-util "^29.7.0" + jest-runtime "^29.7.0" + jest-snapshot "^29.7.0" + jest-util "^29.7.0" + p-limit "^3.1.0" + pretty-format "^29.7.0" + pure-rand "^6.0.0" + slash "^3.0.0" + stack-utils "^2.0.3" + +jest-cli@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz#5592c940798e0cae677eec169264f2d839a37995" + integrity sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg== + dependencies: + "@jest/core" "^29.7.0" + "@jest/test-result" "^29.7.0" + "@jest/types" "^29.6.3" + chalk "^4.0.0" + create-jest "^29.7.0" + exit "^0.1.2" + import-local "^3.0.2" + jest-config "^29.7.0" + jest-util "^29.7.0" + jest-validate "^29.7.0" + yargs "^17.3.1" + +jest-config@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz#bcbda8806dbcc01b1e316a46bb74085a84b0245f" + integrity sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ== + dependencies: + "@babel/core" "^7.11.6" + "@jest/test-sequencer" "^29.7.0" + "@jest/types" "^29.6.3" + babel-jest "^29.7.0" + chalk "^4.0.0" + ci-info "^3.2.0" + deepmerge "^4.2.2" + glob "^7.1.3" + graceful-fs "^4.2.9" + jest-circus "^29.7.0" + jest-environment-node "^29.7.0" + jest-get-type "^29.6.3" + jest-regex-util "^29.6.3" + jest-resolve "^29.7.0" + jest-runner "^29.7.0" + jest-util "^29.7.0" + jest-validate "^29.7.0" + micromatch "^4.0.4" + parse-json "^5.2.0" + pretty-format "^29.7.0" + slash "^3.0.0" + strip-json-comments "^3.1.1" + +jest-diff@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz#017934a66ebb7ecf6f205e84699be10afd70458a" + integrity sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw== + dependencies: + chalk "^4.0.0" + diff-sequences "^29.6.3" + jest-get-type "^29.6.3" + pretty-format "^29.7.0" + +jest-docblock@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz#8fddb6adc3cdc955c93e2a87f61cfd350d5d119a" + integrity sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g== + dependencies: + detect-newline "^3.0.0" + +jest-each@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz#162a9b3f2328bdd991beaabffbb74745e56577d1" + integrity sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ== + dependencies: + "@jest/types" "^29.6.3" + chalk "^4.0.0" + jest-get-type "^29.6.3" + jest-util "^29.7.0" + pretty-format "^29.7.0" + +jest-environment-jsdom@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-environment-jsdom/-/jest-environment-jsdom-29.7.0.tgz#d206fa3551933c3fd519e5dfdb58a0f5139a837f" + integrity sha512-k9iQbsf9OyOfdzWH8HDmrRT0gSIcX+FLNW7IQq94tFX0gynPwqDTW0Ho6iMVNjGz/nb+l/vW3dWM2bbLLpkbXA== + dependencies: + "@jest/environment" "^29.7.0" + "@jest/fake-timers" "^29.7.0" + "@jest/types" "^29.6.3" + "@types/jsdom" "^20.0.0" + "@types/node" "*" + jest-mock "^29.7.0" + jest-util "^29.7.0" + jsdom "^20.0.0" + +jest-environment-node@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz#0b93e111dda8ec120bc8300e6d1fb9576e164376" + integrity sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw== + dependencies: + "@jest/environment" "^29.7.0" + "@jest/fake-timers" "^29.7.0" + "@jest/types" "^29.6.3" + "@types/node" "*" + jest-mock "^29.7.0" + jest-util "^29.7.0" + +jest-get-type@^29.6.3: + version "29.6.3" + resolved "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz#36f499fdcea197c1045a127319c0481723908fd1" + integrity sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw== + +jest-haste-map@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz#3c2396524482f5a0506376e6c858c3bbcc17b104" + integrity sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA== + dependencies: + "@jest/types" "^29.6.3" + "@types/graceful-fs" "^4.1.3" + "@types/node" "*" + anymatch "^3.0.3" + fb-watchman "^2.0.0" + graceful-fs "^4.2.9" + jest-regex-util "^29.6.3" + jest-util "^29.7.0" + jest-worker "^29.7.0" + micromatch "^4.0.4" + walker "^1.0.8" + optionalDependencies: + fsevents "^2.3.2" + +jest-leak-detector@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz#5b7ec0dadfdfec0ca383dc9aa016d36b5ea4c728" + integrity sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw== + dependencies: + jest-get-type "^29.6.3" + pretty-format "^29.7.0" + +jest-matcher-utils@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz#ae8fec79ff249fd592ce80e3ee474e83a6c44f12" + integrity sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g== + dependencies: + chalk "^4.0.0" + jest-diff "^29.7.0" + jest-get-type "^29.6.3" + pretty-format "^29.7.0" + +jest-message-util@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz#8bc392e204e95dfe7564abbe72a404e28e51f7f3" + integrity sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w== + dependencies: + "@babel/code-frame" "^7.12.13" + "@jest/types" "^29.6.3" + "@types/stack-utils" "^2.0.0" + chalk "^4.0.0" + graceful-fs "^4.2.9" + micromatch "^4.0.4" + pretty-format "^29.7.0" + slash "^3.0.0" + stack-utils "^2.0.3" + +jest-mock@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz#4e836cf60e99c6fcfabe9f99d017f3fdd50a6347" + integrity sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw== + dependencies: + "@jest/types" "^29.6.3" + "@types/node" "*" + jest-util "^29.7.0" + +jest-pnp-resolver@^1.2.2: + version "1.2.3" + resolved "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz#930b1546164d4ad5937d5540e711d4d38d4cad2e" + integrity sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w== + +jest-regex-util@^29.6.3: + version "29.6.3" + resolved "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz#4a556d9c776af68e1c5f48194f4d0327d24e8a52" + integrity sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg== + +jest-resolve-dependencies@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz#1b04f2c095f37fc776ff40803dc92921b1e88428" + integrity sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA== + dependencies: + jest-regex-util "^29.6.3" + jest-snapshot "^29.7.0" + +jest-resolve@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz#64d6a8992dd26f635ab0c01e5eef4399c6bcbc30" + integrity sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA== + dependencies: + chalk "^4.0.0" + graceful-fs "^4.2.9" + jest-haste-map "^29.7.0" + jest-pnp-resolver "^1.2.2" + jest-util "^29.7.0" + jest-validate "^29.7.0" + resolve "^1.20.0" + resolve.exports "^2.0.0" + slash "^3.0.0" + +jest-runner@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz#809af072d408a53dcfd2e849a4c976d3132f718e" + integrity sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ== + dependencies: + "@jest/console" "^29.7.0" + "@jest/environment" "^29.7.0" + "@jest/test-result" "^29.7.0" + "@jest/transform" "^29.7.0" + "@jest/types" "^29.6.3" + "@types/node" "*" + chalk "^4.0.0" + emittery "^0.13.1" + graceful-fs "^4.2.9" + jest-docblock "^29.7.0" + jest-environment-node "^29.7.0" + jest-haste-map "^29.7.0" + jest-leak-detector "^29.7.0" + jest-message-util "^29.7.0" + jest-resolve "^29.7.0" + jest-runtime "^29.7.0" + jest-util "^29.7.0" + jest-watcher "^29.7.0" + jest-worker "^29.7.0" + p-limit "^3.1.0" + source-map-support "0.5.13" + +jest-runtime@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz#efecb3141cf7d3767a3a0cc8f7c9990587d3d817" + integrity sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ== + dependencies: + "@jest/environment" "^29.7.0" + "@jest/fake-timers" "^29.7.0" + "@jest/globals" "^29.7.0" + "@jest/source-map" "^29.6.3" + "@jest/test-result" "^29.7.0" + "@jest/transform" "^29.7.0" + "@jest/types" "^29.6.3" + "@types/node" "*" + chalk "^4.0.0" + cjs-module-lexer "^1.0.0" + collect-v8-coverage "^1.0.0" + glob "^7.1.3" + graceful-fs "^4.2.9" + jest-haste-map "^29.7.0" + jest-message-util "^29.7.0" + jest-mock "^29.7.0" + jest-regex-util "^29.6.3" + jest-resolve "^29.7.0" + jest-snapshot "^29.7.0" + jest-util "^29.7.0" + slash "^3.0.0" + strip-bom "^4.0.0" + +jest-snapshot@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz#c2c574c3f51865da1bb329036778a69bf88a6be5" + integrity sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw== + dependencies: + "@babel/core" "^7.11.6" + "@babel/generator" "^7.7.2" + "@babel/plugin-syntax-jsx" "^7.7.2" + "@babel/plugin-syntax-typescript" "^7.7.2" + "@babel/types" "^7.3.3" + "@jest/expect-utils" "^29.7.0" + "@jest/transform" "^29.7.0" + "@jest/types" "^29.6.3" + babel-preset-current-node-syntax "^1.0.0" + chalk "^4.0.0" + expect "^29.7.0" + graceful-fs "^4.2.9" + jest-diff "^29.7.0" + jest-get-type "^29.6.3" + jest-matcher-utils "^29.7.0" + jest-message-util "^29.7.0" + jest-util "^29.7.0" + natural-compare "^1.4.0" + pretty-format "^29.7.0" + semver "^7.5.3" + +jest-util@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz#23c2b62bfb22be82b44de98055802ff3710fc0bc" + integrity sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA== + dependencies: + "@jest/types" "^29.6.3" + "@types/node" "*" + chalk "^4.0.0" + ci-info "^3.2.0" + graceful-fs "^4.2.9" + picomatch "^2.2.3" + +jest-validate@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz#7bf705511c64da591d46b15fce41400d52147d9c" + integrity sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw== + dependencies: + "@jest/types" "^29.6.3" + camelcase "^6.2.0" + chalk "^4.0.0" + jest-get-type "^29.6.3" + leven "^3.1.0" + pretty-format "^29.7.0" + +jest-watcher@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz#7810d30d619c3a62093223ce6bb359ca1b28a2f2" + integrity sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g== + dependencies: + "@jest/test-result" "^29.7.0" + "@jest/types" "^29.6.3" + "@types/node" "*" + ansi-escapes "^4.2.1" + chalk "^4.0.0" + emittery "^0.13.1" + jest-util "^29.7.0" + string-length "^4.0.1" + +jest-worker@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz#acad073acbbaeb7262bd5389e1bcf43e10058d4a" + integrity sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw== + dependencies: + "@types/node" "*" + jest-util "^29.7.0" + merge-stream "^2.0.0" + supports-color "^8.0.0" + +jest@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz#994676fc24177f088f1c5e3737f5697204ff2613" + integrity sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw== + dependencies: + "@jest/core" "^29.7.0" + "@jest/types" "^29.6.3" + import-local "^3.0.2" + jest-cli "^29.7.0" + jiti@^1.21.0: version "1.21.6" resolved "https://registry.npmjs.org/jiti/-/jiti-1.21.6.tgz#6c7f7398dd4b3142767f9a168af2f317a428d268" @@ -4244,6 +5848,14 @@ js-sdsl@^4.1.4: resolved "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz" integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== +js-yaml@^3.13.1: + version "3.14.1" + resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz#dae812fdb3825fa306609a8717383c50c36a0537" + integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== + dependencies: + argparse "^1.0.7" + esprima "^4.0.0" + js-yaml@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz" @@ -4251,6 +5863,43 @@ js-yaml@^4.1.0: dependencies: argparse "^2.0.1" +jsdom@^20.0.0: + version "20.0.3" + resolved "https://registry.npmjs.org/jsdom/-/jsdom-20.0.3.tgz#886a41ba1d4726f67a8858028c99489fed6ad4db" + integrity sha512-SYhBvTh89tTfCD/CRdSOm13mOBa42iTaTyfyEWBdKcGdPxPtLFBXuHR8XHb33YNYaP+lLbmSvBTsnoesCNJEsQ== + dependencies: + abab "^2.0.6" + acorn "^8.8.1" + acorn-globals "^7.0.0" + cssom "^0.5.0" + cssstyle "^2.3.0" + data-urls "^3.0.2" + decimal.js "^10.4.2" + domexception "^4.0.0" + escodegen "^2.0.0" + form-data "^4.0.0" + html-encoding-sniffer "^3.0.0" + http-proxy-agent "^5.0.0" + https-proxy-agent "^5.0.1" + is-potential-custom-element-name "^1.0.1" + nwsapi "^2.2.2" + parse5 "^7.1.1" + saxes "^6.0.0" + symbol-tree "^3.2.4" + tough-cookie "^4.1.2" + w3c-xmlserializer "^4.0.0" + webidl-conversions "^7.0.0" + whatwg-encoding "^2.0.0" + whatwg-mimetype "^3.0.0" + whatwg-url "^11.0.0" + ws "^8.11.0" + xml-name-validator "^4.0.0" + +jsesc@^2.5.1: + version "2.5.2" + resolved "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" + integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== + jsesc@^3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz" @@ -4283,6 +5932,11 @@ json5@^1.0.2: dependencies: minimist "^1.2.0" +json5@^2.2.3: + version "2.2.3" + resolved "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" + integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== + jsonc-eslint-parser@^2.0.4, jsonc-eslint-parser@^2.1.0: version "2.3.0" resolved "https://registry.npmjs.org/jsonc-eslint-parser/-/jsonc-eslint-parser-2.3.0.tgz" @@ -4313,6 +5967,11 @@ khroma@^2.0.0: resolved "https://registry.npmjs.org/khroma/-/khroma-2.0.0.tgz" integrity sha512-2J8rDNlQWbtiNYThZRvmMv5yt44ZakX+Tz5ZIp/mN1pt4snn+m030Va5Z4v8xA0cQFDXBwO/8i42xL4QPsVk3g== +kleur@^3.0.3: + version "3.0.3" + resolved "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" + integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w== + kleur@^4.0.3: version "4.1.5" resolved "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz" @@ -4347,6 +6006,11 @@ layout-base@^2.0.0: resolved "https://registry.npmjs.org/layout-base/-/layout-base-2.0.1.tgz" integrity sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg== +leven@^3.1.0: + version "3.1.0" + resolved "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz#77891de834064cccba82ae7842bb6b14a13ed7f2" + integrity sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A== + levn@^0.4.1: version "0.4.1" resolved "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz" @@ -4477,6 +6141,13 @@ lowlight@^1.17.0: fault "^1.0.0" highlight.js "~10.7.0" +lru-cache@^5.1.1: + version "5.1.1" + resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920" + integrity sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w== + dependencies: + yallist "^3.0.2" + lru-cache@^6.0.0: version "6.0.0" resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz" @@ -4489,6 +6160,30 @@ lru-cache@^6.0.0: resolved "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz" integrity sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q== +lz-string@^1.5.0: + version "1.5.0" + resolved "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz#c1ab50f77887b712621201ba9fd4e3a6ed099941" + integrity sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ== + +make-dir@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz#c3c2307a771277cd9638305f915c29ae741b614e" + integrity sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw== + dependencies: + semver "^7.5.3" + +make-error@^1.1.1: + version "1.3.6" + resolved "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" + integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== + +makeerror@1.0.12: + version "1.0.12" + resolved "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz#3e5dd2079a82e812e983cc6610c4a2cb0eaa801a" + integrity sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg== + dependencies: + tmpl "1.0.5" + markdown-extensions@^1.0.0: version "1.1.1" resolved "https://registry.npmjs.org/markdown-extensions/-/markdown-extensions-1.1.1.tgz" @@ -5158,6 +6853,18 @@ micromatch@^4.0.4, micromatch@^4.0.5: braces "^3.0.2" picomatch "^2.3.1" +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@^2.1.12: + version "2.1.35" + resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz" @@ -5273,6 +6980,11 @@ next@^14.1.1: "@next/swc-win32-ia32-msvc" "14.2.4" "@next/swc-win32-x64-msvc" "14.2.4" +node-int64@^0.4.0: + version "0.4.0" + resolved "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" + integrity sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw== + node-releases@^2.0.14: version "2.0.14" resolved "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz" @@ -5329,6 +7041,11 @@ nth-check@^2.0.1: dependencies: boolbase "^1.0.0" +nwsapi@^2.2.2: + version "2.2.12" + resolved "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.12.tgz#fb6af5c0ec35b27b4581eb3bbad34ec9e5c696f8" + integrity sha512-qXDmcVlZV4XRtKFzddidpfVP4oMSGhga+xdMc25mv8kaLUHtgzCDhUxkrN8exkGdTlLNaXj7CV3GtON7zuGZ+w== + object-assign@^4.0.1, object-assign@^4.1.1: version "4.1.1" resolved "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz" @@ -5462,7 +7179,7 @@ p-limit@^2.2.0: dependencies: p-try "^2.0.0" -p-limit@^3.0.2: +p-limit@^3.0.2, p-limit@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz" integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== @@ -5533,7 +7250,7 @@ parse-entities@^4.0.0: is-decimal "^2.0.0" is-hexadecimal "^2.0.0" -parse-json@^5.0.0: +parse-json@^5.0.0, parse-json@^5.2.0: version "5.2.0" resolved "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz" integrity sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg== @@ -5543,7 +7260,7 @@ parse-json@^5.0.0: json-parse-even-better-errors "^2.3.0" lines-and-columns "^1.1.6" -parse5@^7.0.0: +parse5@^7.0.0, parse5@^7.1.1: version "7.1.2" resolved "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz" integrity sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw== @@ -5602,7 +7319,7 @@ picocolors@^1.0.0, picocolors@^1.0.1: resolved "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz" integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew== -picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.3.1: +picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.3, picomatch@^2.3.1: version "2.3.1" resolved "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== @@ -5627,6 +7344,18 @@ pirates@^4.0.1: resolved "https://registry.npmjs.org/pirates/-/pirates-4.0.5.tgz" integrity sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ== +pirates@^4.0.4: + version "4.0.6" + resolved "https://registry.npmjs.org/pirates/-/pirates-4.0.6.tgz#3018ae32ecfcff6c29ba2267cbf21166ac1f36b9" + integrity sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg== + +pkg-dir@^4.2.0: + version "4.2.0" + resolved "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" + integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== + dependencies: + find-up "^4.0.0" + pluralize@^8.0.0: version "8.0.0" resolved "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz" @@ -5707,6 +7436,24 @@ prelude-ls@^1.2.1: resolved "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz" integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== +pretty-format@^27.0.2: + version "27.5.1" + resolved "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz#2181879fdea51a7a5851fb39d920faa63f01d88e" + integrity sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ== + dependencies: + ansi-regex "^5.0.1" + ansi-styles "^5.0.0" + react-is "^17.0.1" + +pretty-format@^29.0.0, pretty-format@^29.7.0: + version "29.7.0" + resolved "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz#ca42c758310f365bfa71a0bda0a807160b776812" + integrity sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ== + dependencies: + "@jest/schemas" "^29.6.3" + ansi-styles "^5.0.0" + react-is "^18.0.0" + prismjs@^1.27.0: version "1.29.0" resolved "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz" @@ -5717,6 +7464,14 @@ prismjs@~1.27.0: resolved "https://registry.npmjs.org/prismjs/-/prismjs-1.27.0.tgz" integrity sha512-t13BGPUlFDR7wRB5kQDG4jjl7XeuH6jbJGt11JHPL96qwsEHNX2+68tFXqc1/k+/jALsbSWJKUOT/hcYAZ5LkA== +prompts@^2.0.1: + version "2.4.2" + resolved "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069" + integrity sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q== + dependencies: + kleur "^3.0.3" + sisteransi "^1.0.5" + prop-types@^15.0.0, prop-types@^15.5.8, prop-types@^15.8.1: version "15.8.1" resolved "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz" @@ -5738,11 +7493,26 @@ property-information@^6.0.0: resolved "https://registry.npmjs.org/property-information/-/property-information-6.2.0.tgz" integrity sha512-kma4U7AFCTwpqq5twzC1YVIDXSqg6qQK6JN0smOw8fgRy1OkMi0CYSzFmsy6dnqSenamAtj0CyXMUJ1Mf6oROg== +psl@^1.1.33: + version "1.9.0" + resolved "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" + integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== + punycode@^2.1.0: version "2.3.0" resolved "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz" integrity sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA== +punycode@^2.1.1: + version "2.3.1" + resolved "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" + integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg== + +pure-rand@^6.0.0: + version "6.1.0" + resolved "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz#d173cf23258231976ccbdb05247c9787957604f2" + integrity sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA== + qrcode.react@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/qrcode.react/-/qrcode.react-3.1.0.tgz" @@ -5755,6 +7525,11 @@ qs@^6.11.1: dependencies: side-channel "^1.0.4" +querystringify@^2.1.1: + version "2.2.0" + resolved "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz#3345941b4153cb9d082d8eee4cda2016a9aef7f6" + integrity sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ== + queue-microtask@^1.2.2: version "1.2.3" resolved "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz" @@ -5859,6 +7634,11 @@ react-is@^16.13.1, react-is@^16.7.0: resolved "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz" integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ== +react-is@^17.0.1: + version "17.0.2" + resolved "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0" + integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w== + react-is@^18.0.0, react-is@^18.2.0: version "18.2.0" resolved "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz" @@ -6002,6 +7782,14 @@ recordrtc@^5.6.2: resolved "https://registry.npmjs.org/recordrtc/-/recordrtc-5.6.2.tgz" integrity sha512-1QNKKNtl7+KcwD1lyOgP3ZlbiJ1d0HtXnypUy7yq49xEERxk31PHvE9RCciDrulPCY7WJ+oz0R9hpNxgsIurGQ== +redent@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz#e557b7998316bb53c9f1f56fa626352c6963059f" + integrity sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg== + dependencies: + indent-string "^4.0.0" + strip-indent "^3.0.0" + reflect.getprototypeof@^1.0.4: version "1.0.4" resolved "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.4.tgz" @@ -6028,6 +7816,11 @@ regenerator-runtime@^0.13.11: resolved "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz" integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg== +regenerator-runtime@^0.14.0: + version "0.14.1" + resolved "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz#356ade10263f685dda125100cd862c1db895327f" + integrity sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw== + regexp-tree@^0.1.24, regexp-tree@~0.1.1: version "0.1.27" resolved "https://registry.npmjs.org/regexp-tree/-/regexp-tree-0.1.27.tgz" @@ -6122,22 +7915,49 @@ remark-rehype@^10.0.0: mdast-util-to-hast "^12.1.0" unified "^10.0.0" +require-directory@^2.1.1: + version "2.1.1" + resolved "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" + integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q== + +requires-port@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" + integrity sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ== + resize-observer-polyfill@^1.5.1: version "1.5.1" resolved "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz" integrity sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg== +resolve-cwd@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d" + integrity sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg== + dependencies: + resolve-from "^5.0.0" + resolve-from@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz" integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== +resolve-from@^5.0.0: + version "5.0.0" + resolved "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz#c35225843df8f776df21c57557bc087e9dfdfc69" + integrity sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw== + resolve-pkg-maps@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz" integrity sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw== -resolve@^1.1.7, resolve@^1.10.0, resolve@^1.22.1, resolve@^1.22.2, resolve@^1.22.4: +resolve.exports@^2.0.0: + version "2.0.2" + resolved "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.2.tgz#f8c934b8e6a13f539e38b7098e2e36134f01e800" + integrity sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg== + +resolve@^1.1.7, resolve@^1.10.0, resolve@^1.20.0, resolve@^1.22.1, resolve@^1.22.2, resolve@^1.22.4: version "1.22.8" resolved "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz" integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== @@ -6258,6 +8078,13 @@ sass@^1.61.0: immutable "^4.0.0" source-map-js ">=0.6.2 <2.0.0" +saxes@^6.0.0: + version "6.0.0" + resolved "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz#fe5b4a4768df4f14a201b1ba6a65c1f3d9988cc5" + integrity sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA== + dependencies: + xmlchars "^2.2.0" + scheduler@^0.23.0: version "0.23.0" resolved "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz" @@ -6287,6 +8114,11 @@ semver@^7.0.0, semver@^7.3.5, semver@^7.3.6, semver@^7.3.7, semver@^7.3.8, semve dependencies: lru-cache "^6.0.0" +semver@^7.5.3: + version "7.6.3" + resolved "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" + integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== + server-only@^0.0.1: version "0.0.1" resolved "https://registry.npmjs.org/server-only/-/server-only-0.0.1.tgz" @@ -6379,6 +8211,11 @@ simple-swizzle@^0.2.2: dependencies: is-arrayish "^0.3.1" +sisteransi@^1.0.5: + version "1.0.5" + resolved "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed" + integrity sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg== + size-sensor@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/size-sensor/-/size-sensor-1.0.1.tgz" @@ -6430,6 +8267,19 @@ sortablejs@^1.15.0: resolved "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz" integrity sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg== +source-map-support@0.5.13: + version "0.5.13" + resolved "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz#31b24a9c2e73c2de85066c0feb7d44767ed52932" + integrity sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w== + dependencies: + buffer-from "^1.0.0" + source-map "^0.6.0" + +source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.1: + version "0.6.1" + resolved "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" + integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== + source-map@^0.7.0: version "0.7.4" resolved "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz" @@ -6471,6 +8321,18 @@ spdx-license-ids@^3.0.0: resolved "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.13.tgz" integrity sha512-XkD+zwiqXHikFZm4AX/7JSCXA98U5Db4AFd5XUg/+9UNtnH75+Z9KxtpYiJZx36mUDVOwH83pl7yvCer6ewM3w== +sprintf-js@~1.0.2: + version "1.0.3" + resolved "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" + integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== + +stack-utils@^2.0.3: + version "2.0.6" + resolved "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz#aaf0748169c02fc33c8232abccf933f54a1cc34f" + integrity sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ== + dependencies: + escape-string-regexp "^2.0.0" + state-local@^1.0.6: version "1.0.7" resolved "https://registry.npmjs.org/state-local/-/state-local-1.0.7.tgz" @@ -6493,6 +8355,14 @@ string-argv@^0.3.1: resolved "https://registry.npmjs.org/string-argv/-/string-argv-0.3.2.tgz" integrity sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q== +string-length@^4.0.1: + version "4.0.2" + resolved "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz#a8a8dc7bd5c1a82b9b3c8b87e125f66871b6e57a" + integrity sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ== + dependencies: + char-regex "^1.0.2" + strip-ansi "^6.0.0" + "string-width-cjs@npm:string-width@^4.2.0": version "4.2.3" resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" @@ -6502,7 +8372,7 @@ string-argv@^0.3.1: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" -string-width@^4.1.0, string-width@^4.2.0: +string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -6596,6 +8466,11 @@ strip-bom@^3.0.0: resolved "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz" integrity sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA== +strip-bom@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz#9c3505c1db45bcedca3d9cf7a16f5c5aa3901878" + integrity sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w== + strip-final-newline@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz" @@ -6664,6 +8539,13 @@ supports-color@^7.1.0: dependencies: has-flag "^4.0.0" +supports-color@^8.0.0: + version "8.1.1" + resolved "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" + integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== + dependencies: + has-flag "^4.0.0" + supports-preserve-symlinks-flag@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz" @@ -6676,6 +8558,11 @@ swr@^2.1.0: dependencies: use-sync-external-store "^1.2.0" +symbol-tree@^3.2.4: + version "3.2.4" + resolved "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" + integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== + synckit@^0.8.5: version "0.8.5" resolved "https://registry.npmjs.org/synckit/-/synckit-0.8.5.tgz" @@ -6727,6 +8614,15 @@ tapable@^2.2.0: resolved "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz" integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== +test-exclude@^6.0.0: + version "6.0.0" + resolved "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz#04a8698661d805ea6fa293b6cb9e63ac044ef15e" + integrity sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w== + dependencies: + "@istanbuljs/schema" "^0.1.2" + glob "^7.1.4" + minimatch "^3.0.4" + text-table@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz" @@ -6766,6 +8662,16 @@ titleize@^3.0.0: resolved "https://registry.npmjs.org/titleize/-/titleize-3.0.0.tgz" integrity sha512-KxVu8EYHDPBdUYdKZdKtU2aj2XfEx9AfjXxE/Aj0vT06w2icA09Vus1rh6eSu1y01akYg6BjIK/hxyLJINoMLQ== +tmpl@1.0.5: + version "1.0.5" + resolved "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc" + integrity sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw== + +to-fast-properties@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" + integrity sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog== + to-regex-range@^5.0.1: version "5.0.1" resolved "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz" @@ -6778,6 +8684,23 @@ toggle-selection@^1.0.6: resolved "https://registry.npmjs.org/toggle-selection/-/toggle-selection-1.0.6.tgz" integrity sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ== +tough-cookie@^4.1.2: + version "4.1.4" + resolved "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.4.tgz#945f1461b45b5a8c76821c33ea49c3ac192c1b36" + integrity sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag== + dependencies: + psl "^1.1.33" + punycode "^2.1.1" + universalify "^0.2.0" + url-parse "^1.5.3" + +tr46@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz#555c4e297a950617e8eeddef633c87d4d9d6cbf9" + integrity sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA== + dependencies: + punycode "^2.1.1" + trim-lines@^3.0.0: version "3.0.1" resolved "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz" @@ -6798,6 +8721,25 @@ ts-interface-checker@^0.1.9: resolved "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz" integrity sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA== +ts-node@^10.9.2: + version "10.9.2" + resolved "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz#70f021c9e185bccdca820e26dc413805c101c71f" + integrity sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ== + dependencies: + "@cspotcode/source-map-support" "^0.8.0" + "@tsconfig/node10" "^1.0.7" + "@tsconfig/node12" "^1.0.7" + "@tsconfig/node14" "^1.0.0" + "@tsconfig/node16" "^1.0.2" + acorn "^8.4.1" + acorn-walk "^8.1.1" + arg "^4.1.0" + create-require "^1.1.0" + diff "^4.0.1" + make-error "^1.1.1" + v8-compile-cache-lib "^3.0.1" + yn "3.1.1" + tsconfig-paths@^3.15.0: version "3.15.0" resolved "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz" @@ -6837,6 +8779,11 @@ type-check@^0.4.0, type-check@~0.4.0: dependencies: prelude-ls "^1.2.1" +type-detect@4.0.8: + version "4.0.8" + resolved "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" + integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g== + type-fest@^0.20.2: version "0.20.2" resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz" @@ -7002,6 +8949,11 @@ unist-util-visit@^4.0.0: unist-util-is "^5.0.0" unist-util-visit-parents "^5.1.1" +universalify@^0.2.0: + version "0.2.0" + resolved "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz#6451760566fa857534745ab1dde952d1b1761be0" + integrity sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg== + untildify@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz" @@ -7015,6 +8967,14 @@ update-browserslist-db@^1.0.13: escalade "^3.1.2" picocolors "^1.0.1" +update-browserslist-db@^1.1.0: + version "1.1.0" + resolved "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz#7ca61c0d8650766090728046e416a8cde682859e" + integrity sha512-EdRAaAyk2cUE1wOf2DkEhzxqOQvFOoRJFNS6NeyJ01Gp2beMRpBAINjM2iDXE3KCuKhwnvHIQCJm6ThL2Z+HzQ== + dependencies: + escalade "^3.1.2" + picocolors "^1.0.1" + uri-js@^4.2.2: version "4.4.1" resolved "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz" @@ -7022,6 +8982,14 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" +url-parse@^1.5.3: + version "1.5.10" + resolved "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1" + integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ== + dependencies: + querystringify "^2.1.1" + requires-port "^1.0.0" + use-context-selector@^1.4.1: version "1.4.1" resolved "https://registry.npmjs.org/use-context-selector/-/use-context-selector-1.4.1.tgz" @@ -7057,6 +9025,20 @@ uvu@^0.5.0: kleur "^4.0.3" sade "^1.7.3" +v8-compile-cache-lib@^3.0.1: + version "3.0.1" + resolved "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" + integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg== + +v8-to-istanbul@^9.0.1: + version "9.3.0" + resolved "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz#b9572abfa62bd556c16d75fdebc1a411d5ff3175" + integrity sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA== + dependencies: + "@jridgewell/trace-mapping" "^0.3.12" + "@types/istanbul-lib-coverage" "^2.0.1" + convert-source-map "^2.0.0" + validate-npm-package-license@^3.0.1: version "3.0.4" resolved "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz" @@ -7116,6 +9098,20 @@ vue-eslint-parser@^9.3.0: lodash "^4.17.21" semver "^7.3.6" +w3c-xmlserializer@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz#aebdc84920d806222936e3cdce408e32488a3073" + integrity sha512-d+BFHzbiCx6zGfz0HyQ6Rg69w9k19nviJspaj4yNscGjrHu94sVP+aRm75yEbCh+r2/yR+7q6hux9LVtbuTGBw== + dependencies: + xml-name-validator "^4.0.0" + +walker@^1.0.8: + version "1.0.8" + resolved "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz#bd498db477afe573dc04185f011d3ab8a8d7653f" + integrity sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ== + dependencies: + makeerror "1.0.12" + web-namespaces@^2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.1.tgz" @@ -7126,6 +9122,11 @@ web-worker@^1.2.0: resolved "https://registry.npmjs.org/web-worker/-/web-worker-1.2.0.tgz" integrity sha512-PgF341avzqyx60neE9DD+XS26MMNMoUQRz9NOZwW32nPQrF6p77f1htcnjBSEV8BGMKZ16choqUG4hyI0Hx7mA== +webidl-conversions@^7.0.0: + version "7.0.0" + resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz#256b4e1882be7debbf01d05f0aa2039778ea080a" + integrity sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g== + webpack-code-inspector-plugin@0.13.0: version "0.13.0" resolved "https://registry.npmjs.org/webpack-code-inspector-plugin/-/webpack-code-inspector-plugin-0.13.0.tgz" @@ -7133,6 +9134,26 @@ webpack-code-inspector-plugin@0.13.0: dependencies: code-inspector-core "0.13.0" +whatwg-encoding@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz#e7635f597fd87020858626805a2729fa7698ac53" + integrity sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg== + dependencies: + iconv-lite "0.6.3" + +whatwg-mimetype@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz#5fa1a7623867ff1af6ca3dc72ad6b8a4208beba7" + integrity sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q== + +whatwg-url@^11.0.0: + version "11.0.0" + resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz#0a849eebb5faf2119b901bb76fd795c2848d4018" + integrity sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ== + dependencies: + tr46 "^3.0.0" + webidl-conversions "^7.0.0" + which-boxed-primitive@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz" @@ -7236,16 +9257,44 @@ wrappy@1: resolved "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz" integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== +write-file-atomic@^4.0.2: + version "4.0.2" + resolved "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz#a9df01ae5b77858a027fd2e80768ee433555fcfd" + integrity sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg== + dependencies: + imurmurhash "^0.1.4" + signal-exit "^3.0.7" + +ws@^8.11.0: + version "8.18.0" + resolved "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc" + integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw== + xml-name-validator@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz" integrity sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw== +xmlchars@^2.2.0: + version "2.2.0" + resolved "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" + integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw== + xtend@^4.0.0: version "4.0.2" resolved "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== +y18n@^5.0.5: + version "5.0.8" + resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" + integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== + +yallist@^3.0.2: + version "3.1.1" + resolved "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" + integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== + yallist@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz" @@ -7265,6 +9314,29 @@ yaml@^2.0.0, yaml@^2.1.1, yaml@^2.2.2: resolved "https://registry.npmjs.org/yaml/-/yaml-2.3.1.tgz" integrity sha512-2eHWfjaoXgTBC2jNM1LRef62VQa0umtvRiDSk6HSzW7RvS5YtkabJrwYLLEKWBc8a5U2PTSCs+dJjUTJdlHsWQ== +yargs-parser@^21.1.1: + version "21.1.1" + resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35" + integrity sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw== + +yargs@^17.3.1: + version "17.7.2" + resolved "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269" + integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== + dependencies: + cliui "^8.0.1" + escalade "^3.1.1" + get-caller-file "^2.0.5" + require-directory "^2.1.1" + string-width "^4.2.3" + y18n "^5.0.5" + yargs-parser "^21.1.1" + +yn@3.1.1: + version "3.1.1" + resolved "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" + integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q== + yocto-queue@^0.1.0: version "0.1.0" resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz" From 27c8deb4ec4dfa53873069206af92f320d1dca9b Mon Sep 17 00:00:00 2001 From: forrestsocool Date: Thu, 18 Jul 2024 18:40:17 +0800 Subject: [PATCH 31/62] feat: add custom tool timeout config to docker-compose.yaml and .env (#6419) Signed-off-by: forrestsocool --- docker/.env.example | 4 ++++ docker/docker-compose.yaml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/docker/.env.example b/docker/.env.example index 4f7e13e8234b12..2f8ec358f493ef 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -124,6 +124,10 @@ GUNICORN_TIMEOUT=360 # The number of Celery workers. The default is 1, and can be set as needed. CELERY_WORKER_AMOUNT= +# API Tool configuration +API_TOOL_DEFAULT_CONNECT_TIMEOUT=10 +API_TOOL_DEFAULT_READ_TIMEOUT=60 + # ------------------------------ # Database Configuration # The database uses PostgreSQL. Please use the public schema. diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index cffaa5a6a3eec1..30fdf16b171930 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -22,6 +22,8 @@ x-shared-env: &shared-api-worker-env CELERY_WORKER_CLASS: ${CELERY_WORKER_CLASS:-} GUNICORN_TIMEOUT: ${GUNICORN_TIMEOUT:-360} CELERY_WORKER_AMOUNT: ${CELERY_WORKER_AMOUNT:-} + API_TOOL_DEFAULT_CONNECT_TIMEOUT: ${API_TOOL_DEFAULT_CONNECT_TIMEOUT:-10} + API_TOOL_DEFAULT_READ_TIMEOUT: ${API_TOOL_DEFAULT_READ_TIMEOUT:-60} DB_USERNAME: ${DB_USERNAME:-postgres} DB_PASSWORD: ${DB_PASSWORD:-difyai123456} DB_HOST: ${DB_HOST:-db} From c8f5dfcf17b0522789d0bff814c315bd2018a0a2 Mon Sep 17 00:00:00 2001 From: Poorandy Date: Thu, 18 Jul 2024 18:40:36 +0800 Subject: [PATCH 32/62] refactor(rag): switch to dify_config. (#6410) Co-authored-by: -LAN- --- .../vdb/analyticdb/analyticdb_vector.py | 37 ++++++++++--------- .../datasource/vdb/chroma/chroma_vector.py | 15 ++++---- .../datasource/vdb/milvus/milvus_vector.py | 15 ++++---- .../datasource/vdb/myscale/myscale_vector.py | 16 ++++---- .../vdb/opensearch/opensearch_vector.py | 13 +++---- .../rag/datasource/vdb/oracle/oraclevector.py | 17 ++++----- .../datasource/vdb/pgvecto_rs/pgvecto_rs.py | 18 ++++----- .../rag/datasource/vdb/pgvector/pgvector.py | 17 ++++----- .../datasource/vdb/qdrant/qdrant_vector.py | 11 +++--- .../rag/datasource/vdb/relyt/relyt_vector.py | 15 ++++---- .../datasource/vdb/tencent/tencent_vector.py | 19 +++++----- .../datasource/vdb/tidb_vector/tidb_vector.py | 21 +++++------ api/core/rag/datasource/vdb/vector_factory.py | 6 +-- .../vdb/weaviate/weaviate_vector.py | 8 ++-- api/core/rag/extractor/extract_processor.py | 8 ++-- api/core/rag/extractor/notion_extractor.py | 4 +- api/core/rag/extractor/word_extractor.py | 7 ++-- .../index_processor/index_processor_base.py | 5 +-- 18 files changed, 121 insertions(+), 131 deletions(-) diff --git a/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py b/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py index d7a5dd5dcc4f6e..123b93fcd5aa2d 100644 --- a/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py +++ b/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py @@ -7,8 +7,8 @@ "`alibabacloud_gpdb20160503` and `alibabacloud_tea_openapi` packages not found, " "please run `pip install alibabacloud_gpdb20160503 alibabacloud_tea_openapi`" ) -from flask import current_app +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory @@ -36,7 +36,7 @@ def to_analyticdb_client_params(self): "region_id": self.region_id, "read_timeout": self.read_timeout, } - + class AnalyticdbVector(BaseVector): _instance = None _init = False @@ -45,7 +45,7 @@ def __new__(cls, *args, **kwargs): if cls._instance is None: cls._instance = super().__new__(cls) return cls._instance - + def __init__(self, collection_name: str, config: AnalyticdbConfig): # collection_name must be updated every time self._collection_name = collection_name.lower() @@ -105,7 +105,7 @@ def _create_namespace_if_not_exists(self) -> None: raise ValueError( f"failed to create namespace {self.config.namespace}: {e}" ) - + def _create_collection_if_not_exists(self, embedding_dimension: int): from alibabacloud_gpdb20160503 import models as gpdb_20160503_models from Tea.exceptions import TeaException @@ -149,7 +149,7 @@ def _create_collection_if_not_exists(self, embedding_dimension: int): def get_type(self) -> str: return VectorType.ANALYTICDB - + def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs): dimension = len(embeddings[0]) self._create_collection_if_not_exists(dimension) @@ -199,7 +199,7 @@ def text_exists(self, id: str) -> bool: ) response = self._client.query_collection_data(request) return len(response.body.matches.match) > 0 - + def delete_by_ids(self, ids: list[str]) -> None: from alibabacloud_gpdb20160503 import models as gpdb_20160503_models ids_str = ",".join(f"'{id}'" for id in ids) @@ -260,7 +260,7 @@ def search_by_vector( ) documents.append(doc) return documents - + def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: from alibabacloud_gpdb20160503 import models as gpdb_20160503_models score_threshold = ( @@ -291,7 +291,7 @@ def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: ) documents.append(doc) return documents - + def delete(self) -> None: from alibabacloud_gpdb20160503 import models as gpdb_20160503_models request = gpdb_20160503_models.DeleteCollectionRequest( @@ -316,17 +316,18 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.ANALYTICDB, collection_name) ) - config = current_app.config + + # TODO handle optional params return AnalyticdbVector( collection_name, AnalyticdbConfig( - access_key_id=config.get("ANALYTICDB_KEY_ID"), - access_key_secret=config.get("ANALYTICDB_KEY_SECRET"), - region_id=config.get("ANALYTICDB_REGION_ID"), - instance_id=config.get("ANALYTICDB_INSTANCE_ID"), - account=config.get("ANALYTICDB_ACCOUNT"), - account_password=config.get("ANALYTICDB_PASSWORD"), - namespace=config.get("ANALYTICDB_NAMESPACE"), - namespace_password=config.get("ANALYTICDB_NAMESPACE_PASSWORD"), + access_key_id=dify_config.ANALYTICDB_KEY_ID, + access_key_secret=dify_config.ANALYTICDB_KEY_SECRET, + region_id=dify_config.ANALYTICDB_REGION_ID, + instance_id=dify_config.ANALYTICDB_INSTANCE_ID, + account=dify_config.ANALYTICDB_ACCOUNT, + account_password=dify_config.ANALYTICDB_PASSWORD, + namespace=dify_config.ANALYTICDB_NAMESPACE, + namespace_password=dify_config.ANALYTICDB_NAMESPACE_PASSWORD, ), - ) \ No newline at end of file + ) diff --git a/api/core/rag/datasource/vdb/chroma/chroma_vector.py b/api/core/rag/datasource/vdb/chroma/chroma_vector.py index 2d4e1975ea3665..1d85fa78c0437c 100644 --- a/api/core/rag/datasource/vdb/chroma/chroma_vector.py +++ b/api/core/rag/datasource/vdb/chroma/chroma_vector.py @@ -3,9 +3,9 @@ import chromadb from chromadb import QueryResult, Settings -from flask import current_app from pydantic import BaseModel +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory @@ -133,15 +133,14 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings } dataset.index_struct = json.dumps(index_struct_dict) - config = current_app.config return ChromaVector( collection_name=collection_name, config=ChromaConfig( - host=config.get('CHROMA_HOST'), - port=int(config.get('CHROMA_PORT')), - tenant=config.get('CHROMA_TENANT', chromadb.DEFAULT_TENANT), - database=config.get('CHROMA_DATABASE', chromadb.DEFAULT_DATABASE), - auth_provider=config.get('CHROMA_AUTH_PROVIDER'), - auth_credentials=config.get('CHROMA_AUTH_CREDENTIALS'), + host=dify_config.CHROMA_HOST, + port=dify_config.CHROMA_PORT, + tenant=dify_config.CHROMA_TENANT or chromadb.DEFAULT_TENANT, + database=dify_config.CHROMA_DATABASE or chromadb.DEFAULT_DATABASE, + auth_provider=dify_config.CHROMA_AUTH_PROVIDER, + auth_credentials=dify_config.CHROMA_AUTH_CREDENTIALS, ), ) diff --git a/api/core/rag/datasource/vdb/milvus/milvus_vector.py b/api/core/rag/datasource/vdb/milvus/milvus_vector.py index 02b715d76881c7..5f2ab7c5fc4f2c 100644 --- a/api/core/rag/datasource/vdb/milvus/milvus_vector.py +++ b/api/core/rag/datasource/vdb/milvus/milvus_vector.py @@ -3,10 +3,10 @@ from typing import Any, Optional from uuid import uuid4 -from flask import current_app from pydantic import BaseModel, model_validator from pymilvus import MilvusClient, MilvusException, connections +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.field import Field from core.rag.datasource.vdb.vector_base import BaseVector @@ -275,15 +275,14 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.MILVUS, collection_name)) - config = current_app.config return MilvusVector( collection_name=collection_name, config=MilvusConfig( - host=config.get('MILVUS_HOST'), - port=config.get('MILVUS_PORT'), - user=config.get('MILVUS_USER'), - password=config.get('MILVUS_PASSWORD'), - secure=config.get('MILVUS_SECURE'), - database=config.get('MILVUS_DATABASE'), + host=dify_config.MILVUS_HOST, + port=dify_config.MILVUS_PORT, + user=dify_config.MILVUS_USER, + password=dify_config.MILVUS_PASSWORD, + secure=dify_config.MILVUS_SECURE, + database=dify_config.MILVUS_DATABASE, ) ) diff --git a/api/core/rag/datasource/vdb/myscale/myscale_vector.py b/api/core/rag/datasource/vdb/myscale/myscale_vector.py index 811b08818c881c..33ee8259c524f2 100644 --- a/api/core/rag/datasource/vdb/myscale/myscale_vector.py +++ b/api/core/rag/datasource/vdb/myscale/myscale_vector.py @@ -5,9 +5,9 @@ from typing import Any from clickhouse_connect import get_client -from flask import current_app from pydantic import BaseModel +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory @@ -156,15 +156,15 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.MYSCALE, collection_name)) - config = current_app.config return MyScaleVector( collection_name=collection_name, config=MyScaleConfig( - host=config.get("MYSCALE_HOST", "localhost"), - port=int(config.get("MYSCALE_PORT", 8123)), - user=config.get("MYSCALE_USER", "default"), - password=config.get("MYSCALE_PASSWORD", ""), - database=config.get("MYSCALE_DATABASE", "default"), - fts_params=config.get("MYSCALE_FTS_PARAMS", ""), + # TODO: I think setting those values as the default config would be a better option. + host=dify_config.MYSCALE_HOST or "localhost", + port=dify_config.MYSCALE_PORT or 8123, + user=dify_config.MYSCALE_USER or "default", + password=dify_config.MYSCALE_PASSWORD or "", + database=dify_config.MYSCALE_DATABASE or "default", + fts_params=dify_config.MYSCALE_FTS_PARAMS or "", ), ) diff --git a/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py b/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py index 744ff2d517718e..d834e8ce14c51d 100644 --- a/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py +++ b/api/core/rag/datasource/vdb/opensearch/opensearch_vector.py @@ -4,11 +4,11 @@ from typing import Any, Optional from uuid import uuid4 -from flask import current_app from opensearchpy import OpenSearch, helpers from opensearchpy.helpers import BulkIndexError from pydantic import BaseModel, model_validator +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.field import Field from core.rag.datasource.vdb.vector_base import BaseVector @@ -257,14 +257,13 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.OPENSEARCH, collection_name)) - config = current_app.config open_search_config = OpenSearchConfig( - host=config.get('OPENSEARCH_HOST'), - port=config.get('OPENSEARCH_PORT'), - user=config.get('OPENSEARCH_USER'), - password=config.get('OPENSEARCH_PASSWORD'), - secure=config.get('OPENSEARCH_SECURE'), + host=dify_config.OPENSEARCH_HOST, + port=dify_config.OPENSEARCH_PORT, + user=dify_config.OPENSEARCH_USER, + password=dify_config.OPENSEARCH_PASSWORD, + secure=dify_config.OPENSEARCH_SECURE, ) return OpenSearchVector( diff --git a/api/core/rag/datasource/vdb/oracle/oraclevector.py b/api/core/rag/datasource/vdb/oracle/oraclevector.py index 5f7723508ca710..f75310205c7738 100644 --- a/api/core/rag/datasource/vdb/oracle/oraclevector.py +++ b/api/core/rag/datasource/vdb/oracle/oraclevector.py @@ -6,9 +6,9 @@ import numpy import oracledb -from flask import current_app from pydantic import BaseModel, model_validator +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory @@ -44,11 +44,11 @@ def validate_config(cls, values: dict) -> dict: SQL_CREATE_TABLE = """ CREATE TABLE IF NOT EXISTS {table_name} ( - id varchar2(100) + id varchar2(100) ,text CLOB NOT NULL ,meta JSON ,embedding vector NOT NULL -) +) """ @@ -219,14 +219,13 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.ORACLE, collection_name)) - config = current_app.config return OracleVector( collection_name=collection_name, config=OracleVectorConfig( - host=config.get("ORACLE_HOST"), - port=config.get("ORACLE_PORT"), - user=config.get("ORACLE_USER"), - password=config.get("ORACLE_PASSWORD"), - database=config.get("ORACLE_DATABASE"), + host=dify_config.ORACLE_HOST, + port=dify_config.ORACLE_PORT, + user=dify_config.ORACLE_USER, + password=dify_config.ORACLE_PASSWORD, + database=dify_config.ORACLE_DATABASE, ), ) diff --git a/api/core/rag/datasource/vdb/pgvecto_rs/pgvecto_rs.py b/api/core/rag/datasource/vdb/pgvecto_rs/pgvecto_rs.py index 63c8edfbc3046d..82bdc5d4b90075 100644 --- a/api/core/rag/datasource/vdb/pgvecto_rs/pgvecto_rs.py +++ b/api/core/rag/datasource/vdb/pgvecto_rs/pgvecto_rs.py @@ -3,7 +3,6 @@ from typing import Any from uuid import UUID, uuid4 -from flask import current_app from numpy import ndarray from pgvecto_rs.sqlalchemy import Vector from pydantic import BaseModel, model_validator @@ -12,6 +11,7 @@ from sqlalchemy.dialects import postgresql from sqlalchemy.orm import Mapped, Session, mapped_column +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.pgvecto_rs.collection import CollectionORM from core.rag.datasource.vdb.vector_base import BaseVector @@ -93,7 +93,7 @@ def create_collection(self, dimension: int): text TEXT NOT NULL, meta JSONB NOT NULL, vector vector({dimension}) NOT NULL - ) using heap; + ) using heap; """) session.execute(create_statement) index_statement = sql_text(f""" @@ -233,15 +233,15 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.WEAVIATE, collection_name)) dim = len(embeddings.embed_query("pgvecto_rs")) - config = current_app.config + return PGVectoRS( collection_name=collection_name, config=PgvectoRSConfig( - host=config.get('PGVECTO_RS_HOST'), - port=config.get('PGVECTO_RS_PORT'), - user=config.get('PGVECTO_RS_USER'), - password=config.get('PGVECTO_RS_PASSWORD'), - database=config.get('PGVECTO_RS_DATABASE'), + host=dify_config.PGVECTO_RS_HOST, + port=dify_config.PGVECTO_RS_PORT, + user=dify_config.PGVECTO_RS_USER, + password=dify_config.PGVECTO_RS_PASSWORD, + database=dify_config.PGVECTO_RS_DATABASE, ), dim=dim - ) \ No newline at end of file + ) diff --git a/api/core/rag/datasource/vdb/pgvector/pgvector.py b/api/core/rag/datasource/vdb/pgvector/pgvector.py index 72d0a85f8d4b0d..33ca5bc028b052 100644 --- a/api/core/rag/datasource/vdb/pgvector/pgvector.py +++ b/api/core/rag/datasource/vdb/pgvector/pgvector.py @@ -5,9 +5,9 @@ import psycopg2.extras import psycopg2.pool -from flask import current_app from pydantic import BaseModel, model_validator +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory @@ -45,7 +45,7 @@ def validate_config(cls, values: dict) -> dict: text TEXT NOT NULL, meta JSONB NOT NULL, embedding vector({dimension}) NOT NULL -) using heap; +) using heap; """ @@ -185,14 +185,13 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.PGVECTOR, collection_name)) - config = current_app.config return PGVector( collection_name=collection_name, config=PGVectorConfig( - host=config.get("PGVECTOR_HOST"), - port=config.get("PGVECTOR_PORT"), - user=config.get("PGVECTOR_USER"), - password=config.get("PGVECTOR_PASSWORD"), - database=config.get("PGVECTOR_DATABASE"), + host=dify_config.PGVECTOR_HOST, + port=dify_config.PGVECTOR_PORT, + user=dify_config.PGVECTOR_USER, + password=dify_config.PGVECTOR_PASSWORD, + database=dify_config.PGVECTOR_DATABASE, ), - ) \ No newline at end of file + ) diff --git a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py index bccc3a39f6479d..c7c0b7f6f47152 100644 --- a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py +++ b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py @@ -19,6 +19,7 @@ ) from qdrant_client.local.qdrant_local import QdrantLocal +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.field import Field from core.rag.datasource.vdb.vector_base import BaseVector @@ -444,11 +445,11 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings collection_name=collection_name, group_id=dataset.id, config=QdrantConfig( - endpoint=config.get('QDRANT_URL'), - api_key=config.get('QDRANT_API_KEY'), + endpoint=dify_config.QDRANT_URL, + api_key=dify_config.QDRANT_API_KEY, root_path=config.root_path, - timeout=config.get('QDRANT_CLIENT_TIMEOUT'), - grpc_port=config.get('QDRANT_GRPC_PORT'), - prefer_grpc=config.get('QDRANT_GRPC_ENABLED') + timeout=dify_config.QDRANT_CLIENT_TIMEOUT, + grpc_port=dify_config.QDRANT_GRPC_PORT, + prefer_grpc=dify_config.QDRANT_GRPC_ENABLED ) ) diff --git a/api/core/rag/datasource/vdb/relyt/relyt_vector.py b/api/core/rag/datasource/vdb/relyt/relyt_vector.py index 4fe1df717a3bf2..2e0bd6f303c985 100644 --- a/api/core/rag/datasource/vdb/relyt/relyt_vector.py +++ b/api/core/rag/datasource/vdb/relyt/relyt_vector.py @@ -2,7 +2,6 @@ import uuid from typing import Any, Optional -from flask import current_app from pydantic import BaseModel, model_validator from sqlalchemy import Column, Sequence, String, Table, create_engine, insert from sqlalchemy import text as sql_text @@ -19,6 +18,7 @@ except ImportError: from sqlalchemy.ext.declarative import declarative_base +from configs import dify_config from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.models.document import Document from extensions.ext_redis import redis_client @@ -85,7 +85,7 @@ def create_collection(self, dimension: int): document TEXT NOT NULL, metadata JSON NOT NULL, embedding vector({dimension}) NOT NULL - ) using heap; + ) using heap; """) session.execute(create_statement) index_statement = sql_text(f""" @@ -313,15 +313,14 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.RELYT, collection_name)) - config = current_app.config return RelytVector( collection_name=collection_name, config=RelytConfig( - host=config.get('RELYT_HOST'), - port=config.get('RELYT_PORT'), - user=config.get('RELYT_USER'), - password=config.get('RELYT_PASSWORD'), - database=config.get('RELYT_DATABASE'), + host=dify_config.RELYT_HOST, + port=dify_config.RELYT_PORT, + user=dify_config.RELYT_USER, + password=dify_config.RELYT_PASSWORD, + database=dify_config.RELYT_DATABASE, ), group_id=dataset.id ) diff --git a/api/core/rag/datasource/vdb/tencent/tencent_vector.py b/api/core/rag/datasource/vdb/tencent/tencent_vector.py index 3af85854d27f41..cdcc22aec99367 100644 --- a/api/core/rag/datasource/vdb/tencent/tencent_vector.py +++ b/api/core/rag/datasource/vdb/tencent/tencent_vector.py @@ -1,13 +1,13 @@ import json from typing import Any, Optional -from flask import current_app from pydantic import BaseModel from tcvectordb import VectorDBClient from tcvectordb.model import document, enum from tcvectordb.model import index as vdb_index from tcvectordb.model.document import Filter +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory @@ -212,16 +212,15 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.TENCENT, collection_name)) - config = current_app.config return TencentVector( collection_name=collection_name, config=TencentConfig( - url=config.get('TENCENT_VECTOR_DB_URL'), - api_key=config.get('TENCENT_VECTOR_DB_API_KEY'), - timeout=config.get('TENCENT_VECTOR_DB_TIMEOUT'), - username=config.get('TENCENT_VECTOR_DB_USERNAME'), - database=config.get('TENCENT_VECTOR_DB_DATABASE'), - shard=config.get('TENCENT_VECTOR_DB_SHARD'), - replicas=config.get('TENCENT_VECTOR_DB_REPLICAS'), + url=dify_config.TENCENT_VECTOR_DB_URL, + api_key=dify_config.TENCENT_VECTOR_DB_API_KEY, + timeout=dify_config.TENCENT_VECTOR_DB_TIMEOUT, + username=dify_config.TENCENT_VECTOR_DB_USERNAME, + database=dify_config.TENCENT_VECTOR_DB_DATABASE, + shard=dify_config.TENCENT_VECTOR_DB_SHARD, + replicas=dify_config.TENCENT_VECTOR_DB_REPLICAS, ) - ) \ No newline at end of file + ) diff --git a/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py b/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py index 5922db1176621b..d3685c099162fd 100644 --- a/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py +++ b/api/core/rag/datasource/vdb/tidb_vector/tidb_vector.py @@ -3,12 +3,12 @@ from typing import Any import sqlalchemy -from flask import current_app from pydantic import BaseModel, model_validator from sqlalchemy import JSON, TEXT, Column, DateTime, String, Table, create_engine, insert from sqlalchemy import text as sql_text from sqlalchemy.orm import Session, declarative_base +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory @@ -198,8 +198,8 @@ def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Doc with Session(self._engine) as session: select_statement = sql_text( f"""SELECT meta, text, distance FROM ( - SELECT meta, text, {tidb_func}(vector, "{query_vector_str}") as distance - FROM {self._collection_name} + SELECT meta, text, {tidb_func}(vector, "{query_vector_str}") as distance + FROM {self._collection_name} ORDER BY distance LIMIT {top_k} ) t WHERE distance < {distance};""" @@ -234,15 +234,14 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings dataset.index_struct = json.dumps( self.gen_index_struct_dict(VectorType.TIDB_VECTOR, collection_name)) - config = current_app.config return TiDBVector( collection_name=collection_name, config=TiDBVectorConfig( - host=config.get('TIDB_VECTOR_HOST'), - port=config.get('TIDB_VECTOR_PORT'), - user=config.get('TIDB_VECTOR_USER'), - password=config.get('TIDB_VECTOR_PASSWORD'), - database=config.get('TIDB_VECTOR_DATABASE'), - program_name=config.get('APPLICATION_NAME'), + host=dify_config.TIDB_VECTOR_HOST, + port=dify_config.TIDB_VECTOR_PORT, + user=dify_config.TIDB_VECTOR_USER, + password=dify_config.TIDB_VECTOR_PASSWORD, + database=dify_config.TIDB_VECTOR_DATABASE, + program_name=dify_config.APPLICATION_NAME, ), - ) \ No newline at end of file + ) diff --git a/api/core/rag/datasource/vdb/vector_factory.py b/api/core/rag/datasource/vdb/vector_factory.py index f8b58e1b9a3a86..949a4b58471a07 100644 --- a/api/core/rag/datasource/vdb/vector_factory.py +++ b/api/core/rag/datasource/vdb/vector_factory.py @@ -1,8 +1,7 @@ from abc import ABC, abstractmethod from typing import Any -from flask import current_app - +from configs import dify_config from core.embedding.cached_embedding import CacheEmbedding from core.model_manager import ModelManager from core.model_runtime.entities.model_entities import ModelType @@ -37,8 +36,7 @@ def __init__(self, dataset: Dataset, attributes: list = None): self._vector_processor = self._init_vector() def _init_vector(self) -> BaseVector: - config = current_app.config - vector_type = config.get('VECTOR_STORE') + vector_type = dify_config.VECTOR_STORE if self._dataset.index_struct_dict: vector_type = self._dataset.index_struct_dict['type'] diff --git a/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py b/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py index b7c5c96a7dff7a..378d55472fc77c 100644 --- a/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py +++ b/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py @@ -4,9 +4,9 @@ import requests import weaviate -from flask import current_app from pydantic import BaseModel, model_validator +from configs import dify_config from core.rag.datasource.entity.embedding import Embeddings from core.rag.datasource.vdb.field import Field from core.rag.datasource.vdb.vector_base import BaseVector @@ -281,9 +281,9 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings return WeaviateVector( collection_name=collection_name, config=WeaviateConfig( - endpoint=current_app.config.get('WEAVIATE_ENDPOINT'), - api_key=current_app.config.get('WEAVIATE_API_KEY'), - batch_size=int(current_app.config.get('WEAVIATE_BATCH_SIZE')) + endpoint=dify_config.WEAVIATE_ENDPOINT, + api_key=dify_config.WEAVIATE_API_KEY, + batch_size=dify_config.WEAVIATE_BATCH_SIZE ), attributes=attributes ) diff --git a/api/core/rag/extractor/extract_processor.py b/api/core/rag/extractor/extract_processor.py index 909bfdc137ff71..d01cf48fac57eb 100644 --- a/api/core/rag/extractor/extract_processor.py +++ b/api/core/rag/extractor/extract_processor.py @@ -5,8 +5,8 @@ from urllib.parse import unquote import requests -from flask import current_app +from configs import dify_config from core.rag.extractor.csv_extractor import CSVExtractor from core.rag.extractor.entity.datasource_type import DatasourceType from core.rag.extractor.entity.extract_setting import ExtractSetting @@ -94,9 +94,9 @@ def extract(cls, extract_setting: ExtractSetting, is_automatic: bool = False, storage.download(upload_file.key, file_path) input_file = Path(file_path) file_extension = input_file.suffix.lower() - etl_type = current_app.config['ETL_TYPE'] - unstructured_api_url = current_app.config['UNSTRUCTURED_API_URL'] - unstructured_api_key = current_app.config['UNSTRUCTURED_API_KEY'] + etl_type = dify_config.ETL_TYPE + unstructured_api_url = dify_config.UNSTRUCTURED_API_URL + unstructured_api_key = dify_config.UNSTRUCTURED_API_KEY if etl_type == 'Unstructured': if file_extension == '.xlsx' or file_extension == '.xls': extractor = ExcelExtractor(file_path) diff --git a/api/core/rag/extractor/notion_extractor.py b/api/core/rag/extractor/notion_extractor.py index 7c6101010edcb7..9535455909dd9c 100644 --- a/api/core/rag/extractor/notion_extractor.py +++ b/api/core/rag/extractor/notion_extractor.py @@ -3,8 +3,8 @@ from typing import Any, Optional import requests -from flask import current_app +from configs import dify_config from core.rag.extractor.extractor_base import BaseExtractor from core.rag.models.document import Document from extensions.ext_database import db @@ -49,7 +49,7 @@ def __init__( self._notion_access_token = self._get_access_token(tenant_id, self._notion_workspace_id) if not self._notion_access_token: - integration_token = current_app.config.get('NOTION_INTEGRATION_TOKEN') + integration_token = dify_config.NOTION_INTEGRATION_TOKEN if integration_token is None: raise ValueError( "Must specify `integration_token` or set environment " diff --git a/api/core/rag/extractor/word_extractor.py b/api/core/rag/extractor/word_extractor.py index 9045966da92a84..ac4a56319b5142 100644 --- a/api/core/rag/extractor/word_extractor.py +++ b/api/core/rag/extractor/word_extractor.py @@ -8,8 +8,8 @@ import requests from docx import Document as DocxDocument -from flask import current_app +from configs import dify_config from core.rag.extractor.extractor_base import BaseExtractor from core.rag.models.document import Document from extensions.ext_database import db @@ -96,10 +96,9 @@ def _extract_images_from_docx(self, doc, image_folder): storage.save(file_key, rel.target_part.blob) # save file to db - config = current_app.config upload_file = UploadFile( tenant_id=self.tenant_id, - storage_type=config['STORAGE_TYPE'], + storage_type=dify_config.STORAGE_TYPE, key=file_key, name=file_key, size=0, @@ -114,7 +113,7 @@ def _extract_images_from_docx(self, doc, image_folder): db.session.add(upload_file) db.session.commit() - image_map[rel.target_part] = f"![image]({current_app.config.get('CONSOLE_API_URL')}/files/{upload_file.id}/image-preview)" + image_map[rel.target_part] = f"![image]({dify_config.CONSOLE_API_URL}/files/{upload_file.id}/image-preview)" return image_map diff --git a/api/core/rag/index_processor/index_processor_base.py b/api/core/rag/index_processor/index_processor_base.py index edc16c821ae613..33e78ce8c5ccb0 100644 --- a/api/core/rag/index_processor/index_processor_base.py +++ b/api/core/rag/index_processor/index_processor_base.py @@ -2,8 +2,7 @@ from abc import ABC, abstractmethod from typing import Optional -from flask import current_app - +from configs import dify_config from core.model_manager import ModelInstance from core.rag.extractor.entity.extract_setting import ExtractSetting from core.rag.models.document import Document @@ -48,7 +47,7 @@ def _get_splitter(self, processing_rule: dict, # The user-defined segmentation rule rules = processing_rule['rules'] segmentation = rules["segmentation"] - max_segmentation_tokens_length = int(current_app.config['INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH']) + max_segmentation_tokens_length = dify_config.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH if segmentation["max_tokens"] < 50 or segmentation["max_tokens"] > max_segmentation_tokens_length: raise ValueError(f"Custom segment length should be between 50 and {max_segmentation_tokens_length}.") From 218930c89763f669dc06dd322e5a14963b5d0c75 Mon Sep 17 00:00:00 2001 From: Songyawn Date: Thu, 18 Jul 2024 18:55:48 +0800 Subject: [PATCH 33/62] fix tool icon get failed (#6375) Co-authored-by: songyawen --- api/core/tools/tool_manager.py | 2 +- api/services/app_service.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/core/tools/tool_manager.py b/api/core/tools/tool_manager.py index 3342300eb4e480..d7ddb40e6be310 100644 --- a/api/core/tools/tool_manager.py +++ b/api/core/tools/tool_manager.py @@ -574,7 +574,7 @@ def get_tool_icon(cls, tenant_id: str, provider_type: str, provider_id: str) -> provider: ApiToolProvider = db.session.query(ApiToolProvider).filter( ApiToolProvider.tenant_id == tenant_id, ApiToolProvider.id == provider_id - ) + ).first() return json.loads(provider.icon) except: return { diff --git a/api/services/app_service.py b/api/services/app_service.py index 069f766793e704..f88e824b07ae5c 100644 --- a/api/services/app_service.py +++ b/api/services/app_service.py @@ -346,7 +346,7 @@ def get_app_meta(self, app_model: App) -> dict: try: provider: ApiToolProvider = db.session.query(ApiToolProvider).filter( ApiToolProvider.id == provider_id - ) + ).first() meta['tool_icons'][tool_name] = json.loads(provider.icon) except: meta['tool_icons'][tool_name] = { From ba181197c2fb6f433eb7dd7c5f031bb64c3fad9b Mon Sep 17 00:00:00 2001 From: themanforfree Date: Thu, 18 Jul 2024 18:58:46 +0800 Subject: [PATCH 34/62] feat: api_key support for xinference (#6417) Signed-off-by: themanforfree --- .../model_providers/xinference/llm/llm.py | 4 ++- .../xinference/rerank/rerank.py | 28 +++++++++------ .../xinference/speech2text/speech2text.py | 35 +++++++++++-------- .../text_embedding/text_embedding.py | 17 ++++----- .../xinference/xinference.yaml | 9 +++++ 5 files changed, 60 insertions(+), 33 deletions(-) diff --git a/api/core/model_runtime/model_providers/xinference/llm/llm.py b/api/core/model_runtime/model_providers/xinference/llm/llm.py index 0ef63f8e23491f..988bb0ce4432df 100644 --- a/api/core/model_runtime/model_providers/xinference/llm/llm.py +++ b/api/core/model_runtime/model_providers/xinference/llm/llm.py @@ -453,9 +453,11 @@ def _generate(self, model: str, credentials: dict, prompt_messages: list[PromptM if credentials['server_url'].endswith('/'): credentials['server_url'] = credentials['server_url'][:-1] + api_key = credentials.get('api_key') or "abc" + client = OpenAI( base_url=f'{credentials["server_url"]}/v1', - api_key='abc', + api_key=api_key, max_retries=3, timeout=60, ) diff --git a/api/core/model_runtime/model_providers/xinference/rerank/rerank.py b/api/core/model_runtime/model_providers/xinference/rerank/rerank.py index 17b85862c9b0fb..649898f47a1c3f 100644 --- a/api/core/model_runtime/model_providers/xinference/rerank/rerank.py +++ b/api/core/model_runtime/model_providers/xinference/rerank/rerank.py @@ -44,15 +44,23 @@ def _invoke(self, model: str, credentials: dict, docs=[] ) - if credentials['server_url'].endswith('/'): - credentials['server_url'] = credentials['server_url'][:-1] - - handle = RESTfulRerankModelHandle(credentials['model_uid'], credentials['server_url'],auth_headers={}) - response = handle.rerank( - documents=docs, - query=query, - top_n=top_n, - ) + server_url = credentials['server_url'] + model_uid = credentials['model_uid'] + api_key = credentials.get('api_key') + if server_url.endswith('/'): + server_url = server_url[:-1] + auth_headers = {'Authorization': f'Bearer {api_key}'} if api_key else {} + + try: + handle = RESTfulRerankModelHandle(model_uid, server_url, auth_headers) + response = handle.rerank( + documents=docs, + query=query, + top_n=top_n, + ) + except RuntimeError as e: + raise InvokeServerUnavailableError(str(e)) + rerank_documents = [] for idx, result in enumerate(response['results']): @@ -102,7 +110,7 @@ def validate_credentials(self, model: str, credentials: dict) -> None: if not isinstance(xinference_client, RESTfulRerankModelHandle): raise InvokeBadRequestError( 'please check model type, the model you want to invoke is not a rerank model') - + self.invoke( model=model, credentials=credentials, diff --git a/api/core/model_runtime/model_providers/xinference/speech2text/speech2text.py b/api/core/model_runtime/model_providers/xinference/speech2text/speech2text.py index f60d8d34430a00..9ee36213176ef7 100644 --- a/api/core/model_runtime/model_providers/xinference/speech2text/speech2text.py +++ b/api/core/model_runtime/model_providers/xinference/speech2text/speech2text.py @@ -99,9 +99,9 @@ def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]] } def _speech2text_invoke( - self, - model: str, - credentials: dict, + self, + model: str, + credentials: dict, file: IO[bytes], language: Optional[str] = None, prompt: Optional[str] = None, @@ -121,17 +121,24 @@ def _speech2text_invoke( :param temperature: The sampling temperature, between 0 and 1. Higher values like 0.8 will make the output mor e random,while lower values like 0.2 will make it more focused and deterministic.If set to 0, the model wi ll use log probability to automatically increase the temperature until certain thresholds are hit. :return: text for given audio file """ - if credentials['server_url'].endswith('/'): - credentials['server_url'] = credentials['server_url'][:-1] - - handle = RESTfulAudioModelHandle(credentials['model_uid'],credentials['server_url'],auth_headers={}) - response = handle.transcriptions( - audio=file, - language = language, - prompt = prompt, - response_format = response_format, - temperature = temperature - ) + server_url = credentials['server_url'] + model_uid = credentials['model_uid'] + api_key = credentials.get('api_key') + if server_url.endswith('/'): + server_url = server_url[:-1] + auth_headers = {'Authorization': f'Bearer {api_key}'} if api_key else {} + + try: + handle = RESTfulAudioModelHandle(model_uid, server_url, auth_headers) + response = handle.transcriptions( + audio=file, + language=language, + prompt=prompt, + response_format=response_format, + temperature=temperature + ) + except RuntimeError as e: + raise InvokeServerUnavailableError(str(e)) return response["text"] diff --git a/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py index e8429cecd4e065..11f1e29cb39f81 100644 --- a/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py +++ b/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py @@ -43,16 +43,17 @@ def _invoke(self, model: str, credentials: dict, """ server_url = credentials['server_url'] model_uid = credentials['model_uid'] - + api_key = credentials.get('api_key') if server_url.endswith('/'): server_url = server_url[:-1] + auth_headers = {'Authorization': f'Bearer {api_key}'} if api_key else {} try: - handle = RESTfulEmbeddingModelHandle(model_uid, server_url, auth_headers={}) + handle = RESTfulEmbeddingModelHandle(model_uid, server_url, auth_headers) embeddings = handle.create_embedding(input=texts) except RuntimeError as e: - raise InvokeServerUnavailableError(e) - + raise InvokeServerUnavailableError(str(e)) + """ for convenience, the response json is like: class Embedding(TypedDict): @@ -106,7 +107,7 @@ def validate_credentials(self, model: str, credentials: dict) -> None: try: if "/" in credentials['model_uid'] or "?" in credentials['model_uid'] or "#" in credentials['model_uid']: raise CredentialsValidateFailedError("model_uid should not contain /, ?, or #") - + server_url = credentials['server_url'] model_uid = credentials['model_uid'] extra_args = XinferenceHelper.get_xinference_extra_parameter(server_url=server_url, model_uid=model_uid) @@ -117,7 +118,7 @@ def validate_credentials(self, model: str, credentials: dict) -> None: server_url = server_url[:-1] client = Client(base_url=server_url) - + try: handle = client.get_model(model_uid=model_uid) except RuntimeError as e: @@ -151,7 +152,7 @@ def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]] KeyError ] } - + def _calc_response_usage(self, model: str, credentials: dict, tokens: int) -> EmbeddingUsage: """ Calculate response usage @@ -186,7 +187,7 @@ def get_customizable_model_schema(self, model: str, credentials: dict) -> AIMode """ used to define customizable model schema """ - + entity = AIModelEntity( model=model, label=I18nObject( diff --git a/api/core/model_runtime/model_providers/xinference/xinference.yaml b/api/core/model_runtime/model_providers/xinference/xinference.yaml index 28ffc0389e6b99..9496c66fdde7d4 100644 --- a/api/core/model_runtime/model_providers/xinference/xinference.yaml +++ b/api/core/model_runtime/model_providers/xinference/xinference.yaml @@ -46,3 +46,12 @@ model_credential_schema: placeholder: zh_Hans: 在此输入您的Model UID en_US: Enter the model uid + - variable: api_key + label: + zh_Hans: API密钥 + en_US: API key + type: text-input + required: false + placeholder: + zh_Hans: 在此输入您的API密钥 + en_US: Enter the api key From 929c22a4e8c4a6a270df16a7ea14a0d7f57f5a5b Mon Sep 17 00:00:00 2001 From: Carson <331094718@qq.com> Date: Thu, 18 Jul 2024 19:02:23 +0800 Subject: [PATCH 35/62] fix: tools edit modal schema edit issue (#6396) --- .../components/tools/edit-custom-collection-modal/index.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/app/components/tools/edit-custom-collection-modal/index.tsx b/web/app/components/tools/edit-custom-collection-modal/index.tsx index c354bb919aa736..5fcf6fb0248705 100644 --- a/web/app/components/tools/edit-custom-collection-modal/index.tsx +++ b/web/app/components/tools/edit-custom-collection-modal/index.tsx @@ -87,9 +87,9 @@ const EditCustomCollectionModal: FC = ({ return } (async () => { - const customCollection = getCustomCollection() try { const { parameters_schema, schema_type } = await parseParamsSchema(debouncedSchema) + const customCollection = getCustomCollection() const newCollection = produce(customCollection, (draft) => { draft.schema_type = schema_type }) @@ -97,6 +97,7 @@ const EditCustomCollectionModal: FC = ({ setParamsSchemas(parameters_schema) } catch (e) { + const customCollection = getCustomCollection() const newCollection = produce(customCollection, (draft) => { draft.schema_type = '' }) From c0ec40e48382bfd6393fc9918c1294fd6408bd18 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Thu, 18 Jul 2024 19:23:18 +0800 Subject: [PATCH 36/62] fix(api/core/tools/provider/builtin/spider/tools/scraper_crawler.yaml): Fix wrong placeholder config in scraper crawler tool. (#6432) --- .../provider/builtin/spider/tools/scraper_crawler.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/core/tools/provider/builtin/spider/tools/scraper_crawler.yaml b/api/core/tools/provider/builtin/spider/tools/scraper_crawler.yaml index 4ebdce61ff300e..5b20c2fc2f70ad 100644 --- a/api/core/tools/provider/builtin/spider/tools/scraper_crawler.yaml +++ b/api/core/tools/provider/builtin/spider/tools/scraper_crawler.yaml @@ -74,7 +74,8 @@ parameters: human_description: en_US: blacklist a set of paths that you do not want to crawl. you can use regex patterns to help with the list. zh_Hans: 指定一组不想爬取的路径。您可以使用正则表达式模式来帮助定义列表。 - placeholder: /blog/*, /about + placeholder: + en_US: /blog/*, /about form: form - name: whitelist type: string @@ -85,7 +86,8 @@ parameters: human_description: en_US: Whitelist a set of paths that you want to crawl, ignoring all other routes that do not match the patterns. You can use regex patterns to help with the list. zh_Hans: 指定一组要爬取的路径，忽略所有不匹配模式的其他路由。您可以使用正则表达式模式来帮助定义列表。 - placeholder: /blog/*, /about + placeholder: + en_US: /blog/*, /about form: form - name: readability type: boolean From dc847ba14544c146901809bcbe9f081f5975f40b Mon Sep 17 00:00:00 2001 From: leoterry <754236623@qq.com> Date: Thu, 18 Jul 2024 19:25:41 +0800 Subject: [PATCH 37/62] Fix the vector retrieval sorting issue (#6431) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: weifj <“weifj@tuyuansu.com.cn”> --- api/core/rag/datasource/vdb/chroma/chroma_vector.py | 3 ++- api/core/rag/datasource/vdb/qdrant/qdrant_vector.py | 2 ++ api/core/rag/datasource/vdb/weaviate/weaviate_vector.py | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/api/core/rag/datasource/vdb/chroma/chroma_vector.py b/api/core/rag/datasource/vdb/chroma/chroma_vector.py index 1d85fa78c0437c..3629887b448aeb 100644 --- a/api/core/rag/datasource/vdb/chroma/chroma_vector.py +++ b/api/core/rag/datasource/vdb/chroma/chroma_vector.py @@ -111,7 +111,8 @@ def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Doc metadata=metadata, ) docs.append(doc) - + # Sort the documents by score in descending order + docs = sorted(docs, key=lambda x: x.metadata['score'], reverse=True) return docs def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: diff --git a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py index c7c0b7f6f47152..f9a93898683cfa 100644 --- a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py +++ b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py @@ -362,6 +362,8 @@ def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Doc metadata=metadata, ) docs.append(doc) + # Sort the documents by score in descending order + docs = sorted(docs, key=lambda x: x.metadata['score'], reverse=True) return docs def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: diff --git a/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py b/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py index 378d55472fc77c..87fc5ff158e2d3 100644 --- a/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py +++ b/api/core/rag/datasource/vdb/weaviate/weaviate_vector.py @@ -216,7 +216,8 @@ def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Doc if score > score_threshold: doc.metadata['score'] = score docs.append(doc) - + # Sort the documents by score in descending order + docs = sorted(docs, key=lambda x: x.metadata['score'], reverse=True) return docs def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: From 4a026fa352dff40643ddf13960cc05a5fa27d27c Mon Sep 17 00:00:00 2001 From: ybalbert001 <120714773+ybalbert001@users.noreply.github.com> Date: Thu, 18 Jul 2024 19:32:31 +0800 Subject: [PATCH 38/62] Enhancement: add model provider - Amazon Sagemaker (#6255) Co-authored-by: Yuanbo Li Co-authored-by: crazywoola <427733928@qq.com> --- .../model_providers/sagemaker/__init__.py | 0 .../sagemaker/_assets/icon_l_en.png | Bin 0 -> 9395 bytes .../sagemaker/_assets/icon_s_en.png | Bin 0 -> 9720 bytes .../model_providers/sagemaker/llm/__init__.py | 0 .../model_providers/sagemaker/llm/llm.py | 238 ++++++++++++++++++ .../sagemaker/rerank/__init__.py | 0 .../sagemaker/rerank/rerank.py | 190 ++++++++++++++ .../model_providers/sagemaker/sagemaker.py | 17 ++ .../model_providers/sagemaker/sagemaker.yaml | 125 +++++++++ .../sagemaker/text_embedding/__init__.py | 0 .../text_embedding/text_embedding.py | 214 ++++++++++++++++ .../model_runtime/sagemaker/__init__.py | 0 .../model_runtime/sagemaker/test_provider.py | 19 ++ .../model_runtime/sagemaker/test_rerank.py | 55 ++++ .../sagemaker/test_text_embedding.py | 55 ++++ 15 files changed, 913 insertions(+) create mode 100644 api/core/model_runtime/model_providers/sagemaker/__init__.py create mode 100644 api/core/model_runtime/model_providers/sagemaker/_assets/icon_l_en.png create mode 100644 api/core/model_runtime/model_providers/sagemaker/_assets/icon_s_en.png create mode 100644 api/core/model_runtime/model_providers/sagemaker/llm/__init__.py create mode 100644 api/core/model_runtime/model_providers/sagemaker/llm/llm.py create mode 100644 api/core/model_runtime/model_providers/sagemaker/rerank/__init__.py create mode 100644 api/core/model_runtime/model_providers/sagemaker/rerank/rerank.py create mode 100644 api/core/model_runtime/model_providers/sagemaker/sagemaker.py create mode 100644 api/core/model_runtime/model_providers/sagemaker/sagemaker.yaml create mode 100644 api/core/model_runtime/model_providers/sagemaker/text_embedding/__init__.py create mode 100644 api/core/model_runtime/model_providers/sagemaker/text_embedding/text_embedding.py create mode 100644 api/tests/integration_tests/model_runtime/sagemaker/__init__.py create mode 100644 api/tests/integration_tests/model_runtime/sagemaker/test_provider.py create mode 100644 api/tests/integration_tests/model_runtime/sagemaker/test_rerank.py create mode 100644 api/tests/integration_tests/model_runtime/sagemaker/test_text_embedding.py diff --git a/api/core/model_runtime/model_providers/sagemaker/__init__.py b/api/core/model_runtime/model_providers/sagemaker/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/core/model_runtime/model_providers/sagemaker/_assets/icon_l_en.png b/api/core/model_runtime/model_providers/sagemaker/_assets/icon_l_en.png new file mode 100644 index 0000000000000000000000000000000000000000..0abe07a78ff776018db04d84b7cc0ceb1c9ae81f GIT binary patch literal 9395 zcmai)WmsEFwD$uo5Ue;9in|tCpm=e2cLK$o;0{Gv2<{YjFD^wxDG&+;iWj#6MS{ED zlylDeKF|GfKP1_EXYH9aGi%nKJ->gXnu;to1{nqb0Kk@)lhObHkV_HAAAzWdZ?@^f z7laapHGjSiJ(WauH znX3hb#6i<4EOjqRsbqeQBcRqg-{O0hg~XB)v8o&}06oFbhAUveW?OQmb{1VDzrAYr z6ERHf>-n?AX=4wiA_}iM8%xE$W>Zhiv(>B@pt!9iw~;$OFRL!9=lJ&f3!+;&ej^w6 zBL}1Tp`lLz-CkXhqyS3*3c#HDTR&wmYw&S+){=G;Ork3^hX2daA;qixA1^w?CFrS~{(SY!O*+De_ zG5^o5kLA&v=0Cf&;I+jes~NWTt(ovxh3EyNZEq?*Y|kw;u`i+!J(bDKYHq+dWxM9 zJ)fyv9BDh!8&Y8Q6e-+C0$A z(I6z@hXUVrS)s+kBjtr>Jx{IG{~6yk9&x45#e8h{NU|nAYN2^aut|DTwqYHue`>?( z&^g`w$)0!q#aWkJf3pk<_J&%=OZ+572 zuUXe|*`u#L)e-!6Xygx#IB^2~F)j-)5(!G2_FICdyn-LDYvBG|RVsTS_YJddn4< z1}hCtTXrvYHNc-9q^9M2N#25-0(IPUf6z*%{Tm(VLvFT0G%y1)M(Q%dwfie+ymBfu zL^l<`0mf*Hu(AGiaDeMjVot^8u6;E$z9Vs&N|ZF4vpnfn(D{#~^`;BU#6U^204PKC z%?95r&0|c16_9}wHCEatC5PPTA54Gi%;(4My)6jhnyGnczWisl zap#y5q!(grKKexSHE)9lAj_C2q+=v*^!JX)+~|}_q=74k`k#%vyv>Atw#;)kOX9lf zCs&-mmcruSXOzK^^o+aeGd%;+IHdm8AU1?*MPM*j%CBru|HIy)KQbxabaVzQIOO&> z;XJa~lVI#T)V%4&-Aso*t)&UlKC5=+b|SbX67Mm&kX5A&>%f{)XPaXOFT8-z!{b5?~vFEu_?HyG%&}8dv zOf)$$++#;r$R3OE-BM1x8GeZmpr(OAuCz9J=ve$l8?T!u6Z}Q)hpXUgRw#IxZfz0WlIDsyfoX(?b z+0P4^J7lnaT)~b(C*GODE0(G;==_yrsFC z7;k*Mv^ns!_LSJgt=MZ8163~TTDia+DNuHxwt$X0im~!p{X3NyWg*h>^y{gYS@3E_R4n?iUje@z2oxDS>iBsp#`#M$P;0V@QvB~t*$jE^lu$DdA8NSU%m(_wF0 zEU5JNSGm*5Yz|z|Yx7)I!P+Gi1D;FRK6?c-B4CjxLnlI~4ge@x`rl@-Ia zyCzBX+D~fMiRhGu0Lno&<#rPdhbI6SGB7S%N@1MiCT$B9%dQki8{76aPX7f7fk)>PbN^MQn4yPvh< z2o=yTf=^Ql_He{&Jg~aaNZE9;lN4amCQIgqOHU>kM%jJ0z6mfUoxtR!-wK%}HN_!a zGMjHGpaxhPHN$t2a^5RTX-UhU#|$COQ?tQ>`R{g0H{6N~$kzo|N0$>AW1Re8DHf|$AJ zc*gxTD!YHB*OjI-hy*^pgfnw5#lm`Fq&Pf2IyUVc?n%U(<_ zEGogHZEKQ{bnpE%-~yCacl(D)jHmBCOD5U8(GR^=0|Ssu+u{4TTsuCqEbJpJ3(0d` zS#s+_Z%Vqd54t$4!j_p^$ooP^Y1RBNP9IPu*}B@dT@=}+A+nyEgIH;#^~ z4H@rIHn~|eZfH)VUm_ZtMPs;{<4RnAL27}jfS?e?P{Q-+1AKPQ! ze^FM5w%3hUERvXn@4ovaBB$juL|pJADvmQeO3zsob$aQDB;Xm{$MB#Xst19?(jauB zFBrGq>ksRcdn$I4UEsx34x)6+)OVDp6>H1cSat^GTuu(e^*X}1JU59L{nA5Wpw77d8515k3gYL-E4xbd@qb~Mt z??UdRJF&VL;tWG0#uq)%cOfOUzmn#ssmF62$o3_@R=`!yn{W7yQD;eKRRX6OIKlNS zXiAzd?HkowPNvZ5XOlt$r>2@wt_)KQrAf!wp9m&w@aj@8MDHaHils!S48d?6jXwC` z4y9)Wa}HJ^6Fs+3KlJ&qa96)M@r=P6jkfj1YS4{ z#}&N3qxw~@eP!PyRYwKhS5|s2D8nQHle2P4RVS?mv+yTP*#dmzz@+B2_ z6hon2VGqVOvHooF5@Q;)g~nf}Ve7|Go}jq=a=R2t`n%gDrx)4IxvtcwiUYaB zdaa*;Cvv_r8*kuk9B9#KkN|HP`?mACNS7hupg+VTWID;pwy#cwoThhMN~ZCLyE7AE z0h-ufj2p-Xkh3=J*X)ezx$<8IYJf)%XhHk`EtS);nF%j57#9c{t7``gU1ccRn z$g`b^Iq@2VSE4UiQ{w?$A z=b_IBkUSq6F3S}X&g@un{G{<&BH3JJ!9=U0tfHS#6px2k95**NZ_l0LuszjREA<|C zU6*5jOSaid+MK+t;3&jU?bama^$+I<^iU>~GPx z{hRzy_5dAe$1;VdPA7xQ5(yNb=|4g|C9bSOR!Y@XvlX|~?XFTTeMYR0%?~EJspat1 zzB17+N+- zm{jlf7&;9GC{PnVdd2)fK{m%i$$W~G1t1}y{36}G*L)ZD&+b>XyNb^fH3TZFm9R-nt-oA%6RYs&|f4eFBY;Ba} zN$au58_>8z7TRW4KSTDjm@_xFwhpb8bHERjtPZmbt6bn=1U;(?4zO6bn~&&Kp}%A3 zU)SZmrojbssip z)P&!^zmUvG!4M=0yS*l#H->>`37d* zasjF-(_hNLT7(7T+~OWT_KxfPWQz6&Z?qGLJ-3QSid>b^G1ywo9IuQ9rHtl&#r#>S ztSSvGXr5W0;px%U#ZX)_Wm)pMQeN|+6b8>N=^C<8+=fJ3jh1PuE_S`VHPL=s!n7G& z+#-cLDH6AFl}QBdAb@V;mx!J%EkzX)&{|FgI`Ucwh4& za-fYw{`u2F%G^+uRvGOOyK$lXHkQ7x^+Q`~vdoCbI{G`$+yb5#+E{(I>^c)X(I6z| ze9=(nClg6p8Y->o6lNJBOiv3*tT`cA02SIG&gmNTv2p?z51lc@mJ<+f+g~Ys`-o*# zfy?hYa_D{1kD%@O=*zhDo?wd7nI4|p#P`Mr_ys4vY-3XYdb$n^@g`H+^^9DoeCd2347Yq5 z-?YZZP~+h5*2LS)Tj>0~JA1vjzW1>Jz|y4H*tb4nsr)XySpWSMWMQ=iy- zuXWcb_?f#>5|srsDu@u5totL!^bOS;IdwkIQ%&Z*PFufm7`ox6dvK!B%S*)$Qe07r z%RGh0Lzkmhvj#KPYVn$;CS7jlk}vp>rAG?K^3Qeq=X$`36!doZqT1Nk8Kj9O5numQXHP2o^;NIxXjgD z0VC8>qKeL_)e-}+n+a#ObXm#>rlfS%*Jp{cN#d$G=~(kOwF8vTD?ch@WLDQpzeL6aIWcejEpo`!KFlbbS@in0;N|S<|vOhQ4a=x6{#< zL9~5+I4lPD2l3th=@`VSKPWFwS6d@2A?{(C#bJAck-Ls-ub{lb-kdOn!c?kLV16HPYcD>FqTGnVhu@u zzBy}9>0@j*YE=pV@$j&fUaE&b)Gt2oJqFKi8Nql_@87%K#m|UP%D4{&l=Uv?4Ayg& zVYqiB*11}84t+L4^Z&)@qKXrpuT)ENF{t@5(f$lk}VQU z$#mx^@9gLN1f^1;oEB;+6V`shHi^U{{#lm@KxG45-#~IOSgw#YA*K}o=-{%`wiO8Y z?55iJ+btVcDOi>*qB$Wvfb2=OSASP#%J(<#j^b64e9B{-d_n20VFtT$haeA$<#}XA zYwOgwA{r#bQSF+gCle7(iTZt`6i<~)S=%IZx#8?RVHpw2K9qTA8{c@n+AksPrqaB& z>NYU7w=Oz`jTe6Hns!uKHrB6>Ice8F*!g+P`sEPt9s>JSZh(Fr*gweoyV+BAIX!{X zbG|*1WobBW zfjJA16}Li?UBxl{k5?6BBrKJWkaF>7odjCIn!Pb&I3#O8`{q_6xbG()ntH7Yoj*ef z8>Q0|-k5b~#Hz5Ku%5dfB%@7Q0TiipCa1O1H5h&S(6}3=R2RCA zeqOI1sDjv^?!wY~&yv*2QdxJz@1}qME;G1}5mIQ;)#7RX{{8*_6JXS&JC%tw9v&05 zS7ap_)uUt&{RU5C!**IbxCo0c^`)9x%PT^YP|E=&%TUmA>En?$0Q6lKytV=biW|q>PG`Nb#Jc z8ltN8ReThhKJsFsUYfVE-@qL%O8-ZXs|TW)&37nHL~F+QmCsbCiwhf>7jhCH6rzOu zZIB4tYzG*Zg1dI1AhiY?7mgDFLOV&NR@%4lX>x0an72OWab`PH=MWK1buLcwcv`d% zJ63m%>GlX~fPJA1?12@3t~9s8e@+fBe-dG;s&)bm5x5onuzqee>dkZ;S1FOqBDgiS zL;_z@eTgkXSZZkIrlhc1KvqEjweUSa;;vEnoi3G@q)X zPAV9Cb$MvzU%OKeL;DBKmkR z{c&(vDo5%zNf!#a&^8^GD(2AEFr=j{LxfHt!M8X=@HNf38S5b9R_Wb6CjsZ5MS6W$J~MR+2d7j%oIYc?uzPtS+zYJfE(E4I=KTDVsOQJdNBY zE9Cs$+4T1;|s5JeRC zkqV2H}CU97-UmLq2$0cAYCG&3uREQ)wc? zR(tO~R;hCgMGPhF!?YyjeAMn#D9*gq z$^GbvZuK!|wNN z;H5~rvAMDoESF;kezEm{di#gd_vE*-{&;Ocqr?2Y;ueSt_$wn?Gs&fyK9 zJ9)`f3*qQq6Rpkb=(L~) zmO}O+iuL}HGohN&*hv%ealt!s;>PR!I@8(c&ABLGcUk%VgLifi6QAx{&2#C$)UpmU z1PwAAi(RNk`%QVzN9DjDL)ZrWK@F;)hS&`EGC|Fu8k={nq+zdaO9yIFzqKz?7^nYC2=}Kns1G&B06pOhyNXy+gmB^f>_fEH58mJ}69SG}~T{j|QO&Q`Hh7 zLg}hTSYY!TMZyS6gtnXO$a1~^O`BH=2ZA<y2uP2<**)v_3SuX)YRCebg!HdW~0P3u>}f@YgFJHReP`L`pw!!?DO18@j46v{tTKACAFo8Tgcv_FLNr750v3X3W`fTn|~p3D6)VT z()?nUp&_76zz@&+RCMeMrLc{9I-+0Vk0O6m$^-js4St@n4d)Q_Xw4wcJ3`yS0ZN}9 z22HQ~5ZpL21r?PG=`BqFhf2`#MRy$Z02;$85XAIwA>gv>+{W3IDRZ=oKNEj3TLl9%bz zzY4;N-1VU^+P>dX>@OgbRqoX1TpVOX8XKp7M4;vmGI`U~igupKA5K{p+tBcaL7{Tu zeFMl6*Xmh0f{2yc`3KzMw)-5gzSes6k`w}yr>GHwsR4ke`hT?m5=|I+gQ?UOz}|A! zKigxm5#4dD!16jqscMpwsHKprO^nd|M}UZGkkx$Aj;Xcy`&QlUrcLgI$4}25t+4+vZN3ci~0^e!r5H$(0(0nSJkQ5yxbC#DHp$387r%c~PR&!E^J3 z^rcY=0Vp%=4Tb$~XnTP_>+U$gU;L<+W#`rmygg~!*G|Z?bQ5LV>U}HdmPaYX;xFZ_ zGwwxzVBr^|4cw2|zDOmD@)H2Tz}kAtp2;$WF`p@7dbraX&s4aL`~`mqERTNZg2umk zau+GEVt;_1JKmoG1=)Jav#(!ZGH(4?Vg090#%}m-_nz&UCxU~IHX2F=o@ZT}q*x8E zF4tUieYGA1ULY^Jbp>Q+KjMAD(mFRNN>wLYYEvA`&d)e<;)H8x6R;|X#2c)2X+t?? zR1znMW{3NjMRh@y-$L8hvh-~s|9XTmIpuZwITEIV%n2=rKAd*}^LC2Qz!q9YnXxIq z1h!0mJ3dl#QeW^60*K>fQxzYS4eCyPWs}>v)Vg%z@(}tdV*S_gc=3&&gOzA#wu1TV znmyysc|wf&20>f=^DlZJwr!33DI=^q>%GJOALdJPs8D1r5>rE}UIK94;6$Yqm_(=|Xy~)Yi{Jq(=Wzv* z$+=liHlfG8kZTvF^c6VTKUxU6@eZyCk~6jIr2em+G4~2+(tjt|xT-LW@lmz%|CA*E zDO3I#kH4J%u{ps1Lviw-kmWzS|AZ`I|0+|klK2T>7!&u zPPNyTv@tQEbT;W9nGC%?*msrn4bpEGwMfPEP0N3UH9A?y-QktBNO}MrW{r;q$<=^b hi=clrH(3v84ipNJtpSyN2o(}QURp(}TEaBse*pS#84mye literal 0 HcmV?d00001 diff --git a/api/core/model_runtime/model_providers/sagemaker/_assets/icon_s_en.png b/api/core/model_runtime/model_providers/sagemaker/_assets/icon_s_en.png new file mode 100644 index 0000000000000000000000000000000000000000..6b88942a5ce27cb57e27ce876cecedf3c5dfce3b GIT binary patch literal 9720 zcmZ{KWmFy8vh~J&Gi0z?1+fFdU=ss3iQ-j+5z?Ax`=Z1wk>fwmS?5(5D0 z6Oo?GVBYS@EM(P{002K~03aj+0C;!{g&YC^o~!`CktqPcp9KKmx_sdk|AI{^>`fIrpVJb*j|;lF%!2-<%zkN`lGEdcr-jQ*SbbI87} zKb*fU6d2;)6~U1ILTiDc{^e``p|x}Jn!XtX7g;@b000T|&w>Ete8d9)AV+OAbv<;I z6!|Tj9huGEIh$KD`#8G%VF3hv_}^SdOAj+LA4dl#cYYrs&|e7tH~&wV1w{52#KT?) zq^qP#CgJR6Nyg30%FGH9MkFI66LfoL#jh?Y{SW=^P6%Y<;o-v1!s6}i&Fsy=?CfUE z!p6tP$HL0a!p_e0hG25{b@DLtVRCY(_`Ase){(Syw{Wv{@vwDvBKuR<%-q@2LkI-= zBlNH1?>;?jt^Td#V`i`LFPQvvmKR|F_J42LGF&;%004_8|Z2 zi|wDb{xkL;_&?L}tJ?ZlI_OH;I$Aop|Cxo2omG(K|IGY1Q^MK7*-gX6%)(OmP3SMm ze}w)U{g;m3f9m`%>Ay4oFa=ruyzT#Z`+sk(zoKt@OBhj*L6 zG<_gW0)3OTmV6IZP6O>(s!%3?;uz6VjN-C3P~?MI)Ypp z=@sSjKN7PwClsk0=ZooG%>_24sT6FaG8~cs5a#HQAkpaN(9%MaZtqt2$)_&&EyqqL z{jyvqRrbgLs=nkPK|IK z@j>&b7kN33i8?>3vdnN_-|K!YnFwYx8~-KZz#OO6VjOM!TV+MSPx-9zY$Z@TGXJ*Q zd0b(?k#qX$oApXyhDc>**W8zHXYo@HnV-DVyus{xd>z0vf;Ah)f(=58mBCYrDDIQj z5pR=#GnJ*QGoddpB24WT*S8Yjnp9nPX4NxMBZ9K_wJCMeb~;@5%@L$1<)<&*?fy)! zwZZeQEm?$IV#mwpSTk#~DA*8HzSM0u5$c&W`OlbYs!UWCJ%mnB{F<~PBU8P4aXa$z z+!eL~dZs2I#AnL>lP`o406Yu^Sw-_6QD-PW3tFHd;&O+P?RINx!6qwY)54ZAMO=@g zstlfLMU9$JJHkS+SNcGRO+nN4bD> zyW7Kfju;T&312~>cw8uyd4z_Sz7SruOa1QAP?UYtBJC3>Zr0;oX-Y`^h_F%6jf-J zIX)|ymUZXoR7ewHH1*gtH!(hINc>$c>p!kO$r6WF%#yD2drE^{N|oHs<{G@t2uk#& zksyc-^M#9cZ&inWLY1o8V26!C9JP&*ZLzBoTiU7~SX$69Z7=LkJ$PN_HH2KW3;Xa* zTCr%iI99-WcUJq~bk?{JpWcJ@hk&B4_@3GVD7ikL%a{Vc&2iui6Xicfq^5mT%B%d^ zV#~h#+*0!^fN3OZ1H4*lpwP@itl1h9(osj5oyjxvVe7W($*Lc1CCu`HG`X~AsWR@DP(bsYxI3| z#muRdGFz-$%>-Z1q)z;+G`n0-IBHO;;YK?%ohCComoIs35s<^N#tm!J0M7bBJ}Tx3 z68Akdj~=I(-FLx zJC;tH&3^jYuO8V&jtU`>T7)MX;7_+Ea!(`Pv4<}#6)xlN#6r*yzb^M7jddUTVxFO0 z-0xBw-evVsf3sCrN0cGvx7rD1V)LoLkXADL@nYSXAHyLD3{-7WgE(AI;mYSZA<^5o z6el9~#ZB7+6_0NS8jro+dWSJ1*9~IC#EFiUag?v_CG3N~p5Z6P=Y9{t(8Gj&`1y%{ z`SOaAuX!rXjVVk?RIfGYywHEQPFxM}6+DQ* zq5VLWQ~4=3?$=S1*|9E3MlZ(sXsEajy+ZKveSPun`BHFZnY)ZbOb@ybGlDqz-i70; zfd4Ed)2`37ptNjS=(~{H59KXC`*PV;2Fz@}U%{EGrWkJD%TiVrW5mP+a}O6hx3kQF+&CwQq3~uQH$%er31v5z&_uXlUM8m;nmOn(-H?tu@Zz%Pf|pIZ3{R^ z7VT;0WtQ@g*S*h$!hSy_8qN839Q}oz^lvkhg?)PrdoeX*N2Be8eoO?arY{AoeL`Ud zEhb+(w?!ipY92+y4ssgDwmNqadPS_xCCDiRS{cGbrw8ZlF;Lh69>@p# ziyqNU#r-6NPX1gmUfs!9c6o;N-+oHqXJV3ejQ*b8N?q_LQD?%7BiniX9QBDd27gbt z=pE1O@9@z=k|IvBBV+XUdKSB`bNCG0a&x}l$|?_b_$C8qi^%RZ5rfxW6`nA^c$^=S z`Ush-P+F?iN-|t{@;KgcL)zYf7X`%TSK4Cp`JBExz9!u%%@xmbRb53+RJ3j-%vRCH z!ybC_k7Twv5RCQ3eumfVYcOkSn}9`?WYRRqA8W+RHIEm?n0*+q^6E;Cpt;2>c}mNf z;Z=$!=1@C7RmeEmK!d}y{y;R+JTDq{8vl-fP1Wc#kx4y-~bAaq{SY$7cA z@8EITtPk{w{u7Qroq`>;P!^U9-g~$@wwMXxv#CbSGMPs@32`%@6apc!O|f;OCBP0k z$;aYQD_n{jTwr5UUMBrH@CNM(Ksk8_ytT@uQjqIxF0; zb9o?w%;{wqbV0EOtz3qTTv``zC%S6?frPPxQXSot#Ou62*W+T@i&)%#w)w*~AN-Pi z*y492OCW99Gcr)?L6PyLnq`cF9=i$h2?ts;5FZ*SO^4S zZofWK4CLH19*7AI&u>Q>2mClm5-kZ$$tq@$W-(;g!MG5_dw<1AJCui3>bX-~KJ&ah z0&4a*Z-!|5o)f$yfaJ(JxSOA1Dj3`1a+5u3^$uevXBLSSUu-4z5FM1v2>Jap?%4USmKURzz z7hksBmi2={!87zzxvzEajn7JZJh?k9T!#=afB6agklE^m)sUR9NnEDhF1} z4hEu5l1%FpG6$-2UX*5%`8$^ldt&fLjUzPQ56QJ(HB}4WeKB4ajLba3WVD*hRI1+E zUGNSf(NQ^JgLxll%8HrM%xg188cML_aJlVX(c*%A;|{STBVQ6Z#Of7H+W!AnRxo* za6~XNIBpN7A`v&YUBtF!+B#viZFWTP(m9abA8K*M{(%syOi~FQl0;hR$FG^%J?QL- zqY(baO@m<4`p#od2yLoq8@q&0z=BQtCrmhx+0}2mi>`;PC(L+4N@&OnJDc(RPZOvX zzQ2St22wN!`Wq57a^;47uOyvHCqoOc{L#|~r76Ju{*9;UrPHH!dDfzyBpQ^<$!*Zs zY1GcDF-KP)K58*@G;FcYSjw~#$eEihwyKnW#hn*p32q4V>q0|o z(BOfC4ZN?4F}{n9_V#WFaEyWdtjBx1BcrQB_JtEt3fNU~NL?j4+lM&)DH zdJWQjY=kDPXj=^piD5gREF7DzF)T9c*&C|E4}IX3q`KAZyqzl!coz%I3{=w+E$qrv z5i0em+;%He%z5Yv>fP?2A>%#3L?Kiv$6 z?v0)XL#PEu~@H5pypH_4Ik%!PQ4=MM>%JmK9yg!!t~su^*)!_Ti+DLoP-Wi*~5 zU-C#4P}8zQ#CI-3Frd6gJTdtb$b9R7IKnq5EcyI8I7K;JLzzF1G7d0(_mS6Wz;?A{ zgF}xF_j{AHeuu{u9fz)odlo5zeuYqdE_OR~)K$SnL0`OmcBq?LH6p$}WymM}uCeH%3zF zyafu%;Ns&#P5jEbTb|R4{a+0_%H$Q*;Z-y-UmjU(1RN8pdRfI==F^}cN9SbX%DrYO zi+j|@(xQh_)@a(YJ|ON@sOes=$jiCMj~~>DO37@LiuC|gV^QEkQDQdl8#1twk!Q$= z#**F}E%um5a*AS#LPap!Sd>-paY3x{=cm}M+_PaO>GX5!?$Z&<_biahGrac1er8H) zo)Ae4TrKZ}6a-`E`acfU2i=Vs54O98Ov%7;^-HJU=9a_Q(J9S~T$*r+>1REIf6Hdv z-9cfR@HxA;UrxW9u9`eHawgIYi_XBLBVtoXIFaikuC@2|Yi0f+gQOgZ5Mo}vm?9Okhwy?vAHK=*auU-3FrBD^1Jf!x{+Su|(XNp%KE-Z9*Z*He9 z{jbCYomYC0L>!sCu=g-d!D}O}P4;jq^i$h$98*scXk;hzLbhlcp22ceTrF^qp)1tq z!0W2AvifCWs%U$US%b`#Zmi(im3N5{4m+fX;>Qz$k^k$pB+;4!;(B;+xlzwP@>-AY zCu=6$1Zz1k%{!?`a(m) zFS#ja0ywU$Ht05~M~A|1O1N5gwWF-$VMFSqJ};NPUn9OH5x(E}FkRj%(W6Slc=_!R z4hlRFP8>3^>w}P^>0;-PB~CKAFRvN>!dhom`qm%{lvD`H$d(UMqPpn$#B&EO(*L*0LCVUbat5Pj~rIig9ff+uL8{@XbWs&T>2U=Z&Gh zeBRP1z~UFS)B0Q?A`1c*Yt%nIU3Yov# zQ4(;9geO`eeSjgxnO`}!*MGMZ@oc6RQPkA`rmI?^WJcXkxFm1@$pdPB({+&siX+tz zY5f*!Zc}gtMTj~G#iToKw`nP@pr^)!%5b;jLc`Z_sWv5lgM~l@m{*abVWaCg$a><# zPiM+t3zm@AyZZ{!=ZY92^hE(WHea>EHg20W0QxbgWNg_~lB6c-o8(i10}>@HzM^?~O#W}+h;4pa z0e3}QEq6xuW?&a~o1P+FdutW8z+!gkGhz6cO2C=`dtknPG$L_K$Lfq-$(&>qH(@xf z`1I+u-RYx~D%eU&G}P`Gi2x}}hcJE~OZ%m#k%Vwz(mpph#V4Iv`&Q_1Hn z)!6L-gI9g0RosY3fK9-ESn~7DO2F%63P;_d*~z@+*N#Iu6_qZ96um+6#4-$HJF^tQ zI(n7icS3~g>6#ivo;*rrDdyJH{Z-5S+KRIFWwpNV#D@dwF0AW2UHb6LYm*skmcE0< zO7x2JIA_MU&ck*4uoX|S=C+kDPK`_V4!P9=AK_d}jz4OT6{KE#d^Bq&*xj2q*2>6e zQM!HRH!Sd|pB_jV>i*QnhgxRX^^sY@P5>s%Nq8i8quskK)tJxD{VT7(|4F(aB0uNI zB};k!ph_9Yye3D3Zcxi9;bdk{`C?m{Zd216(KDW35dxz@Yj~sf@N#f2k|TNo;x|4;+OTh!x}6SW>`5ZTplyg)5Z(C&R7`_XPk@9d zv|j{C&drz+cvCj zN=NF6ELR{@DnW$9F~q%_RU8(z8KvvFLal-8&Mufra^_{&ilrJo8u~gobbCm`p0C)h z(5x=S=`&DBzdAfjqVTkIs~3hDJW{+eOOXsD$Xj`_GTeCn0Acc4%%Yi}5?}q^9DQzY z#=GcF;j@zC^VR@PFV?;}A<29v+V2|ge|q@&X^I|BUN#$_%*v?wJdlC35Xo*}wE-Eua-G#n za@Tl@egRgswoVoHk-uGdZFRxw{pBwx74Jh(Fcf5BnZvghjqgd5-{Wa}eU>o=jNmFc z@~!uJGp7rGO`im=*3Ii(f0+CF8?M&Adv(I?g*KlzihR1_!2R=$`g4)@@`}I`_oZtU zuJe;(Vd7a&%?E7)u@k22k%TB8{7Ll8Yr1=3I=1C6X5sQEGa--OPxNuUc{GYO>&&jw zqYdR(mQ(#xz3OjYt}BGeVdZ234iSJBc=UjZU0$NdYC|k?FDk8pyr>Lu`sgMid{j=> zA!b17ZudYSdM-?$h<4ZT$_w$A7RWRxZzu2%`$puGk?0V4C4P2Wo7_8+UujN?=*l&W z`V?1yCKX9UMp?FBbr*2Qm_b$|Me1m#2`FfWc!Q<_WchmS>b2PKe($9JEMq(hqxp>X7XpV>*&%KY#e5G5`XXsX1-o~c1XkG)a< zkPNBIDC{m7Rfl|mc}B>O`W_5lghRN%Uwugj4xrQC_S<6b-`swRF?4+GN&{x6&=fHa`FwTO%Iof7egAA zh9KO}01bpMjH})|^F6(;Ga#{%fwT6v>8lnIfXHnf8*zMJZe$^a%)#91Se`E58JA^V zr;)s>6rjb2pU3KlyYGX)EjBzLr)>B4)fTWfCIX4r<6 z&UsfgeMg^=Z!v&=;-%nXBf+yzLNAmug;V}WlU$};SYwifOf33*!fEyJTofrWfOUjG z2&fnuCEuNrE!_$KkbwS5Sx6ApMu-?Ac7M3yOR=ML|G~E4Co`6kI)XAXqlUgHqj;s#T64N9S4qF7FuTSQs zy;01kPBl3c+NE@VXT*^1|(PiK$T=9b~O`bIIXUQ*sy|8TJdr0Gj)bobN_{ zmGG}WxDkQg88JRoSf3hGQ^79DRQ0iP8glML8N|V@RbEk}(Opw_l%&TpIP%8#-25nH z1f)vNtZGG&WA79^G$7&%(`(r_Y2*?vnDECCldON-Gi3x7ktTCGsFvdnqyef{FH?(s zUZSg+{Pe%+PM(eng(irFnU{naP%1*&7k=wlUT}a0jChy~Y`26KbYUF0=B(=Q^;V#b z!SmTc20J;cKoZG^&kZt!@ubu(BoQs^j;dPL+c~gX#K30+_F7ddS zBmZz%Il!Sc!MigfT7k5lJt(7J>fI7^*1Jgy_K&=&sr(45pYEtnS#;^Xd$UkCA5rCg z-a-JQL}hdv9I;kGZ|7p3BZh;150HCskQD% zOFka)1C8bFKzCW)yD-%W&I>z5*4wfAN1=%&IG*gr0<~1t+!HBA$GiC`ueb?aUZJF1 zJ$#T)E)&ZFl8zTVo$nUK)c~zzf;u z$ys zrty(&H^c^J@vyyYDCv#sB;+%r>gcFcS87j6{)b2#xNF?{x{K4L6vO7+AQ>vp&xx#A zo!yFT-$ibdfkMxerZJM@Hk!I~7wcM_E~wUkBL)8bw6vAJyV+F9405}iJI7rs^DnN; z$eR&9eIB8BJ@c26J?bFr*2?}x8;JpR@A}5yA2-8qrAbf4_r?`sj(B0cLNzwaJ!9SC ze2!o|C4KE)ol4U}FuhpOK{=yq&yhC?KW#atCbC7QhJRt%Xl3@)7OX!+J{m?eSGbvO4B2? zKD+GR2V02>1R+5PwS_?;-~BAUX*ZOxt;0#f%Pv7-AB?x&Cu-2H81DUJ%Ou4>0^k~k2xINDO)wL*EG z{5cR{_OTfD!M*#^qV4tqZgDO;6;#c9@cmT;SM(A2!X1k17)A{%dsnuLik`9uR>BhK zvgh61uFt|VP5JI*ef0PMNRN^cQk<8~EFrJ?LwXdYJh{-CTJF+|zzo_coWCzb-LqA! zlMf8cPTh!bdYUW85Zvs0kd1P6f)e;;3loVZzD?D4#y+3IAjJDE*625BtSI}hos9=X z(LBjqgdnr;Zr(Cd#3HSa&Kn|^Fe#nSi2*2q!Zz{!>cCppGm*<{47Fb;!3jVz!$b%S zSv@zh#qM{vdxyWSre+qlwqBwz{ezR<|1SJRsyJSQU;JECaEVIu5-cwf3H#B0(e2Ow OOmb4nk~QL{VgCa)i~r65 literal 0 HcmV?d00001 diff --git a/api/core/model_runtime/model_providers/sagemaker/llm/__init__.py b/api/core/model_runtime/model_providers/sagemaker/llm/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/core/model_runtime/model_providers/sagemaker/llm/llm.py b/api/core/model_runtime/model_providers/sagemaker/llm/llm.py new file mode 100644 index 00000000000000..f8e7757a969f8e --- /dev/null +++ b/api/core/model_runtime/model_providers/sagemaker/llm/llm.py @@ -0,0 +1,238 @@ +import json +import logging +from collections.abc import Generator +from typing import Any, Optional, Union + +import boto3 + +from core.model_runtime.entities.llm_entities import LLMMode, LLMResult +from core.model_runtime.entities.message_entities import ( + AssistantPromptMessage, + PromptMessage, + PromptMessageTool, +) +from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, I18nObject, ModelType +from core.model_runtime.errors.invoke import ( + InvokeAuthorizationError, + InvokeBadRequestError, + InvokeConnectionError, + InvokeError, + InvokeRateLimitError, + InvokeServerUnavailableError, +) +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel + +logger = logging.getLogger(__name__) + + +class SageMakerLargeLanguageModel(LargeLanguageModel): + """ + Model class for Cohere large language model. + """ + sagemaker_client: Any = None + + def _invoke(self, model: str, credentials: dict, + prompt_messages: list[PromptMessage], model_parameters: dict, + tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, + stream: bool = True, user: Optional[str] = None) \ + -> Union[LLMResult, Generator]: + """ + Invoke large language model + + :param model: model name + :param credentials: model credentials + :param prompt_messages: prompt messages + :param model_parameters: model parameters + :param tools: tools for tool calling + :param stop: stop words + :param stream: is stream response + :param user: unique user id + :return: full response or stream response chunk generator result + """ + # get model mode + model_mode = self.get_model_mode(model, credentials) + + if not self.sagemaker_client: + access_key = credentials.get('access_key') + secret_key = credentials.get('secret_key') + aws_region = credentials.get('aws_region') + if aws_region: + if access_key and secret_key: + self.sagemaker_client = boto3.client("sagemaker-runtime", + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=aws_region) + else: + self.sagemaker_client = boto3.client("sagemaker-runtime", region_name=aws_region) + else: + self.sagemaker_client = boto3.client("sagemaker-runtime") + + + sagemaker_endpoint = credentials.get('sagemaker_endpoint') + response_model = self.sagemaker_client.invoke_endpoint( + EndpointName=sagemaker_endpoint, + Body=json.dumps( + { + "inputs": prompt_messages[0].content, + "parameters": { "stop" : stop}, + "history" : [] + } + ), + ContentType="application/json", + ) + + assistant_text = response_model['Body'].read().decode('utf8') + + # transform assistant message to prompt message + assistant_prompt_message = AssistantPromptMessage( + content=assistant_text + ) + + usage = self._calc_response_usage(model, credentials, 0, 0) + + response = LLMResult( + model=model, + prompt_messages=prompt_messages, + message=assistant_prompt_message, + usage=usage + ) + + return response + + def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], + tools: Optional[list[PromptMessageTool]] = None) -> int: + """ + Get number of tokens for given prompt messages + + :param model: model name + :param credentials: model credentials + :param prompt_messages: prompt messages + :param tools: tools for tool calling + :return: + """ + # get model mode + model_mode = self.get_model_mode(model) + + try: + return 0 + except Exception as e: + raise self._transform_invoke_error(e) + + def validate_credentials(self, model: str, credentials: dict) -> None: + """ + Validate model credentials + + :param model: model name + :param credentials: model credentials + :return: + """ + try: + # get model mode + model_mode = self.get_model_mode(model) + except Exception as ex: + raise CredentialsValidateFailedError(str(ex)) + + @property + def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + """ + Map model invoke error to unified error + The key is the error type thrown to the caller + The value is the error type thrown by the model, + which needs to be converted into a unified error type for the caller. + + :return: Invoke error mapping + """ + return { + InvokeConnectionError: [ + InvokeConnectionError + ], + InvokeServerUnavailableError: [ + InvokeServerUnavailableError + ], + InvokeRateLimitError: [ + InvokeRateLimitError + ], + InvokeAuthorizationError: [ + InvokeAuthorizationError + ], + InvokeBadRequestError: [ + InvokeBadRequestError, + KeyError, + ValueError + ] + } + + def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity | None: + """ + used to define customizable model schema + """ + rules = [ + ParameterRule( + name='temperature', + type=ParameterType.FLOAT, + use_template='temperature', + label=I18nObject( + zh_Hans='温度', + en_US='Temperature' + ), + ), + ParameterRule( + name='top_p', + type=ParameterType.FLOAT, + use_template='top_p', + label=I18nObject( + zh_Hans='Top P', + en_US='Top P' + ) + ), + ParameterRule( + name='max_tokens', + type=ParameterType.INT, + use_template='max_tokens', + min=1, + max=credentials.get('context_length', 2048), + default=512, + label=I18nObject( + zh_Hans='最大生成长度', + en_US='Max Tokens' + ) + ) + ] + + completion_type = LLMMode.value_of(credentials["mode"]) + + if completion_type == LLMMode.CHAT: + print(f"completion_type : {LLMMode.CHAT.value}") + + if completion_type == LLMMode.COMPLETION: + print(f"completion_type : {LLMMode.COMPLETION.value}") + + features = [] + + support_function_call = credentials.get('support_function_call', False) + if support_function_call: + features.append(ModelFeature.TOOL_CALL) + + support_vision = credentials.get('support_vision', False) + if support_vision: + features.append(ModelFeature.VISION) + + context_length = credentials.get('context_length', 2048) + + entity = AIModelEntity( + model=model, + label=I18nObject( + en_US=model + ), + fetch_from=FetchFrom.CUSTOMIZABLE_MODEL, + model_type=ModelType.LLM, + features=features, + model_properties={ + ModelPropertyKey.MODE: completion_type, + ModelPropertyKey.CONTEXT_SIZE: context_length + }, + parameter_rules=rules + ) + + return entity diff --git a/api/core/model_runtime/model_providers/sagemaker/rerank/__init__.py b/api/core/model_runtime/model_providers/sagemaker/rerank/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/core/model_runtime/model_providers/sagemaker/rerank/rerank.py b/api/core/model_runtime/model_providers/sagemaker/rerank/rerank.py new file mode 100644 index 00000000000000..0b06f54ef1823f --- /dev/null +++ b/api/core/model_runtime/model_providers/sagemaker/rerank/rerank.py @@ -0,0 +1,190 @@ +import json +import logging +from typing import Any, Optional + +import boto3 + +from core.model_runtime.entities.common_entities import I18nObject +from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelType +from core.model_runtime.entities.rerank_entities import RerankDocument, RerankResult +from core.model_runtime.errors.invoke import ( + InvokeAuthorizationError, + InvokeBadRequestError, + InvokeConnectionError, + InvokeError, + InvokeRateLimitError, + InvokeServerUnavailableError, +) +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.__base.rerank_model import RerankModel + +logger = logging.getLogger(__name__) + +class SageMakerRerankModel(RerankModel): + """ + Model class for Cohere rerank model. + """ + sagemaker_client: Any = None + + def _sagemaker_rerank(self, query_input: str, docs: list[str], rerank_endpoint:str): + inputs = [query_input]*len(docs) + response_model = self.sagemaker_client.invoke_endpoint( + EndpointName=rerank_endpoint, + Body=json.dumps( + { + "inputs": inputs, + "docs": docs + } + ), + ContentType="application/json", + ) + json_str = response_model['Body'].read().decode('utf8') + json_obj = json.loads(json_str) + scores = json_obj['scores'] + return scores if isinstance(scores, list) else [scores] + + + def _invoke(self, model: str, credentials: dict, + query: str, docs: list[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, + user: Optional[str] = None) \ + -> RerankResult: + """ + Invoke rerank model + + :param model: model name + :param credentials: model credentials + :param query: search query + :param docs: docs for reranking + :param score_threshold: score threshold + :param top_n: top n + :param user: unique user id + :return: rerank result + """ + line = 0 + try: + if len(docs) == 0: + return RerankResult( + model=model, + docs=docs + ) + + line = 1 + if not self.sagemaker_client: + access_key = credentials.get('aws_access_key_id') + secret_key = credentials.get('aws_secret_access_key') + aws_region = credentials.get('aws_region') + if aws_region: + if access_key and secret_key: + self.sagemaker_client = boto3.client("sagemaker-runtime", + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=aws_region) + else: + self.sagemaker_client = boto3.client("sagemaker-runtime", region_name=aws_region) + else: + self.sagemaker_client = boto3.client("sagemaker-runtime") + + line = 2 + + sagemaker_endpoint = credentials.get('sagemaker_endpoint') + candidate_docs = [] + + scores = self._sagemaker_rerank(query, docs, sagemaker_endpoint) + for idx in range(len(scores)): + candidate_docs.append({"content" : docs[idx], "score": scores[idx]}) + + sorted(candidate_docs, key=lambda x: x['score'], reverse=True) + + line = 3 + rerank_documents = [] + for idx, result in enumerate(candidate_docs): + rerank_document = RerankDocument( + index=idx, + text=result.get('content'), + score=result.get('score', -100.0) + ) + + if score_threshold is not None: + if rerank_document.score >= score_threshold: + rerank_documents.append(rerank_document) + else: + rerank_documents.append(rerank_document) + + return RerankResult( + model=model, + docs=rerank_documents + ) + + except Exception as e: + logger.exception(f'Exception {e}, line : {line}') + + def validate_credentials(self, model: str, credentials: dict) -> None: + """ + Validate model credentials + + :param model: model name + :param credentials: model credentials + :return: + """ + try: + self._invoke( + model=model, + credentials=credentials, + query="What is the capital of the United States?", + docs=[ + "Carson City is the capital city of the American state of Nevada. At the 2010 United States " + "Census, Carson City had a population of 55,274.", + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean that " + "are a political division controlled by the United States. Its capital is Saipan.", + ], + score_threshold=0.8 + ) + except Exception as ex: + raise CredentialsValidateFailedError(str(ex)) + + @property + def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + """ + Map model invoke error to unified error + The key is the error type thrown to the caller + The value is the error type thrown by the model, + which needs to be converted into a unified error type for the caller. + + :return: Invoke error mapping + """ + return { + InvokeConnectionError: [ + InvokeConnectionError + ], + InvokeServerUnavailableError: [ + InvokeServerUnavailableError + ], + InvokeRateLimitError: [ + InvokeRateLimitError + ], + InvokeAuthorizationError: [ + InvokeAuthorizationError + ], + InvokeBadRequestError: [ + InvokeBadRequestError, + KeyError, + ValueError + ] + } + + def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity | None: + """ + used to define customizable model schema + """ + entity = AIModelEntity( + model=model, + label=I18nObject( + en_US=model + ), + fetch_from=FetchFrom.CUSTOMIZABLE_MODEL, + model_type=ModelType.RERANK, + model_properties={ }, + parameter_rules=[] + ) + + return entity \ No newline at end of file diff --git a/api/core/model_runtime/model_providers/sagemaker/sagemaker.py b/api/core/model_runtime/model_providers/sagemaker/sagemaker.py new file mode 100644 index 00000000000000..02d05f406c50f7 --- /dev/null +++ b/api/core/model_runtime/model_providers/sagemaker/sagemaker.py @@ -0,0 +1,17 @@ +import logging + +from core.model_runtime.model_providers.__base.model_provider import ModelProvider + +logger = logging.getLogger(__name__) + + +class SageMakerProvider(ModelProvider): + def validate_provider_credentials(self, credentials: dict) -> None: + """ + Validate provider credentials + + if validate failed, raise exception + + :param credentials: provider credentials, credentials form defined in `provider_credential_schema`. + """ + pass diff --git a/api/core/model_runtime/model_providers/sagemaker/sagemaker.yaml b/api/core/model_runtime/model_providers/sagemaker/sagemaker.yaml new file mode 100644 index 00000000000000..290cb0edabee09 --- /dev/null +++ b/api/core/model_runtime/model_providers/sagemaker/sagemaker.yaml @@ -0,0 +1,125 @@ +provider: sagemaker +label: + zh_Hans: Sagemaker + en_US: Sagemaker +icon_small: + en_US: icon_s_en.png +icon_large: + en_US: icon_l_en.png +description: + en_US: Customized model on Sagemaker + zh_Hans: Sagemaker上的私有化部署的模型 +background: "#ECE9E3" +help: + title: + en_US: How to deploy customized model on Sagemaker + zh_Hans: 如何在Sagemaker上的私有化部署的模型 + url: + en_US: https://github.com/aws-samples/dify-aws-tool/blob/main/README.md#how-to-deploy-sagemaker-endpoint + zh_Hans: https://github.com/aws-samples/dify-aws-tool/blob/main/README_ZH.md#%E5%A6%82%E4%BD%95%E9%83%A8%E7%BD%B2sagemaker%E6%8E%A8%E7%90%86%E7%AB%AF%E7%82%B9 +supported_model_types: + - llm + - text-embedding + - rerank +configurate_methods: + - customizable-model +model_credential_schema: + model: + label: + en_US: Model Name + zh_Hans: 模型名称 + placeholder: + en_US: Enter your model name + zh_Hans: 输入模型名称 + credential_form_schemas: + - variable: mode + show_on: + - variable: __model_type + value: llm + label: + en_US: Completion mode + type: select + required: false + default: chat + placeholder: + zh_Hans: 选择对话类型 + en_US: Select completion mode + options: + - value: completion + label: + en_US: Completion + zh_Hans: 补全 + - value: chat + label: + en_US: Chat + zh_Hans: 对话 + - variable: sagemaker_endpoint + label: + en_US: sagemaker endpoint + type: text-input + required: true + placeholder: + zh_Hans: 请输出你的Sagemaker推理端点 + en_US: Enter your Sagemaker Inference endpoint + - variable: aws_access_key_id + required: false + label: + en_US: Access Key (If not provided, credentials are obtained from the running environment.) + zh_Hans: Access Key (如果未提供，凭证将从运行环境中获取。) + type: secret-input + placeholder: + en_US: Enter your Access Key + zh_Hans: 在此输入您的 Access Key + - variable: aws_secret_access_key + required: false + label: + en_US: Secret Access Key + zh_Hans: Secret Access Key + type: secret-input + placeholder: + en_US: Enter your Secret Access Key + zh_Hans: 在此输入您的 Secret Access Key + - variable: aws_region + required: false + label: + en_US: AWS Region + zh_Hans: AWS 地区 + type: select + default: us-east-1 + options: + - value: us-east-1 + label: + en_US: US East (N. Virginia) + zh_Hans: 美国东部 (弗吉尼亚北部) + - value: us-west-2 + label: + en_US: US West (Oregon) + zh_Hans: 美国西部 (俄勒冈州) + - value: ap-southeast-1 + label: + en_US: Asia Pacific (Singapore) + zh_Hans: 亚太地区 (新加坡) + - value: ap-northeast-1 + label: + en_US: Asia Pacific (Tokyo) + zh_Hans: 亚太地区 (东京) + - value: eu-central-1 + label: + en_US: Europe (Frankfurt) + zh_Hans: 欧洲 (法兰克福) + - value: us-gov-west-1 + label: + en_US: AWS GovCloud (US-West) + zh_Hans: AWS GovCloud (US-West) + - value: ap-southeast-2 + label: + en_US: Asia Pacific (Sydney) + zh_Hans: 亚太地区 (悉尼) + - value: cn-north-1 + label: + en_US: AWS Beijing (cn-north-1) + zh_Hans: 中国北京 (cn-north-1) + - value: cn-northwest-1 + label: + en_US: AWS Ningxia (cn-northwest-1) + zh_Hans: 中国宁夏 (cn-northwest-1) diff --git a/api/core/model_runtime/model_providers/sagemaker/text_embedding/__init__.py b/api/core/model_runtime/model_providers/sagemaker/text_embedding/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/core/model_runtime/model_providers/sagemaker/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/sagemaker/text_embedding/text_embedding.py new file mode 100644 index 00000000000000..4b2858b1a28228 --- /dev/null +++ b/api/core/model_runtime/model_providers/sagemaker/text_embedding/text_embedding.py @@ -0,0 +1,214 @@ +import itertools +import json +import logging +import time +from typing import Any, Optional + +import boto3 + +from core.model_runtime.entities.common_entities import I18nObject +from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelPropertyKey, ModelType, PriceType +from core.model_runtime.entities.text_embedding_entities import EmbeddingUsage, TextEmbeddingResult +from core.model_runtime.errors.invoke import ( + InvokeAuthorizationError, + InvokeBadRequestError, + InvokeConnectionError, + InvokeError, + InvokeRateLimitError, + InvokeServerUnavailableError, +) +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel + +BATCH_SIZE = 20 +CONTEXT_SIZE=8192 + +logger = logging.getLogger(__name__) + +def batch_generator(generator, batch_size): + while True: + batch = list(itertools.islice(generator, batch_size)) + if not batch: + break + yield batch + +class SageMakerEmbeddingModel(TextEmbeddingModel): + """ + Model class for Cohere text embedding model. + """ + sagemaker_client: Any = None + + def _sagemaker_embedding(self, sm_client, endpoint_name, content_list:list[str]): + response_model = sm_client.invoke_endpoint( + EndpointName=endpoint_name, + Body=json.dumps( + { + "inputs": content_list, + "parameters": {}, + "is_query" : False, + "instruction" : '' + } + ), + ContentType="application/json", + ) + json_str = response_model['Body'].read().decode('utf8') + json_obj = json.loads(json_str) + embeddings = json_obj['embeddings'] + return embeddings + + def _invoke(self, model: str, credentials: dict, + texts: list[str], user: Optional[str] = None) \ + -> TextEmbeddingResult: + """ + Invoke text embedding model + + :param model: model name + :param credentials: model credentials + :param texts: texts to embed + :param user: unique user id + :return: embeddings result + """ + # get model properties + try: + line = 1 + if not self.sagemaker_client: + access_key = credentials.get('aws_access_key_id') + secret_key = credentials.get('aws_secret_access_key') + aws_region = credentials.get('aws_region') + if aws_region: + if access_key and secret_key: + self.sagemaker_client = boto3.client("sagemaker-runtime", + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=aws_region) + else: + self.sagemaker_client = boto3.client("sagemaker-runtime", region_name=aws_region) + else: + self.sagemaker_client = boto3.client("sagemaker-runtime") + + line = 2 + sagemaker_endpoint = credentials.get('sagemaker_endpoint') + + line = 3 + truncated_texts = [ item[:CONTEXT_SIZE] for item in texts ] + + batches = batch_generator((text for text in truncated_texts), batch_size=BATCH_SIZE) + all_embeddings = [] + + line = 4 + for batch in batches: + embeddings = self._sagemaker_embedding(self.sagemaker_client, sagemaker_endpoint, batch) + all_embeddings.extend(embeddings) + + line = 5 + # calc usage + usage = self._calc_response_usage( + model=model, + credentials=credentials, + tokens=0 # It's not SAAS API, usage is meaningless + ) + line = 6 + + return TextEmbeddingResult( + embeddings=all_embeddings, + usage=usage, + model=model + ) + + except Exception as e: + logger.exception(f'Exception {e}, line : {line}') + + def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + """ + Get number of tokens for given prompt messages + + :param model: model name + :param credentials: model credentials + :param texts: texts to embed + :return: + """ + return 0 + + def validate_credentials(self, model: str, credentials: dict) -> None: + """ + Validate model credentials + + :param model: model name + :param credentials: model credentials + :return: + """ + try: + print("validate_credentials ok....") + except Exception as ex: + raise CredentialsValidateFailedError(str(ex)) + + def _calc_response_usage(self, model: str, credentials: dict, tokens: int) -> EmbeddingUsage: + """ + Calculate response usage + + :param model: model name + :param credentials: model credentials + :param tokens: input tokens + :return: usage + """ + # get input price info + input_price_info = self.get_price( + model=model, + credentials=credentials, + price_type=PriceType.INPUT, + tokens=tokens + ) + + # transform usage + usage = EmbeddingUsage( + tokens=tokens, + total_tokens=tokens, + unit_price=input_price_info.unit_price, + price_unit=input_price_info.unit, + total_price=input_price_info.total_amount, + currency=input_price_info.currency, + latency=time.perf_counter() - self.started_at + ) + + return usage + + @property + def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + return { + InvokeConnectionError: [ + InvokeConnectionError + ], + InvokeServerUnavailableError: [ + InvokeServerUnavailableError + ], + InvokeRateLimitError: [ + InvokeRateLimitError + ], + InvokeAuthorizationError: [ + InvokeAuthorizationError + ], + InvokeBadRequestError: [ + KeyError + ] + } + + def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity | None: + """ + used to define customizable model schema + """ + + entity = AIModelEntity( + model=model, + label=I18nObject( + en_US=model + ), + fetch_from=FetchFrom.CUSTOMIZABLE_MODEL, + model_type=ModelType.TEXT_EMBEDDING, + model_properties={ + ModelPropertyKey.CONTEXT_SIZE: CONTEXT_SIZE, + ModelPropertyKey.MAX_CHUNKS: BATCH_SIZE, + }, + parameter_rules=[] + ) + + return entity diff --git a/api/tests/integration_tests/model_runtime/sagemaker/__init__.py b/api/tests/integration_tests/model_runtime/sagemaker/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/tests/integration_tests/model_runtime/sagemaker/test_provider.py b/api/tests/integration_tests/model_runtime/sagemaker/test_provider.py new file mode 100644 index 00000000000000..639227e7450343 --- /dev/null +++ b/api/tests/integration_tests/model_runtime/sagemaker/test_provider.py @@ -0,0 +1,19 @@ +import os + +import pytest + +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.sagemaker.sagemaker import SageMakerProvider + + +def test_validate_provider_credentials(): + provider = SageMakerProvider() + + with pytest.raises(CredentialsValidateFailedError): + provider.validate_provider_credentials( + credentials={} + ) + + provider.validate_provider_credentials( + credentials={} + ) diff --git a/api/tests/integration_tests/model_runtime/sagemaker/test_rerank.py b/api/tests/integration_tests/model_runtime/sagemaker/test_rerank.py new file mode 100644 index 00000000000000..c67849dd798883 --- /dev/null +++ b/api/tests/integration_tests/model_runtime/sagemaker/test_rerank.py @@ -0,0 +1,55 @@ +import os + +import pytest + +from core.model_runtime.entities.rerank_entities import RerankResult +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.sagemaker.rerank.rerank import SageMakerRerankModel + + +def test_validate_credentials(): + model = SageMakerRerankModel() + + with pytest.raises(CredentialsValidateFailedError): + model.validate_credentials( + model='bge-m3-rerank-v2', + credentials={ + "aws_region": os.getenv("AWS_REGION"), + "aws_access_key": os.getenv("AWS_ACCESS_KEY"), + "aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY") + }, + query="What is the capital of the United States?", + docs=[ + "Carson City is the capital city of the American state of Nevada. At the 2010 United States " + "Census, Carson City had a population of 55,274.", + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean that " + "are a political division controlled by the United States. Its capital is Saipan.", + ], + score_threshold=0.8 + ) + + +def test_invoke_model(): + model = SageMakerRerankModel() + + result = model.invoke( + model='bge-m3-rerank-v2', + credentials={ + "aws_region": os.getenv("AWS_REGION"), + "aws_access_key": os.getenv("AWS_ACCESS_KEY"), + "aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY") + }, + query="What is the capital of the United States?", + docs=[ + "Carson City is the capital city of the American state of Nevada. At the 2010 United States " + "Census, Carson City had a population of 55,274.", + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean that " + "are a political division controlled by the United States. Its capital is Saipan.", + ], + score_threshold=0.8 + ) + + assert isinstance(result, RerankResult) + assert len(result.docs) == 1 + assert result.docs[0].index == 1 + assert result.docs[0].score >= 0.8 diff --git a/api/tests/integration_tests/model_runtime/sagemaker/test_text_embedding.py b/api/tests/integration_tests/model_runtime/sagemaker/test_text_embedding.py new file mode 100644 index 00000000000000..e817e8f04ab67c --- /dev/null +++ b/api/tests/integration_tests/model_runtime/sagemaker/test_text_embedding.py @@ -0,0 +1,55 @@ +import os + +import pytest + +from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.sagemaker.text_embedding.text_embedding import SageMakerEmbeddingModel + + +def test_validate_credentials(): + model = SageMakerEmbeddingModel() + + with pytest.raises(CredentialsValidateFailedError): + model.validate_credentials( + model='bge-m3', + credentials={ + } + ) + + model.validate_credentials( + model='bge-m3-embedding', + credentials={ + } + ) + + +def test_invoke_model(): + model = SageMakerEmbeddingModel() + + result = model.invoke( + model='bge-m3-embedding', + credentials={ + }, + texts=[ + "hello", + "world" + ], + user="abc-123" + ) + + assert isinstance(result, TextEmbeddingResult) + assert len(result.embeddings) == 2 + +def test_get_num_tokens(): + model = SageMakerEmbeddingModel() + + num_tokens = model.get_num_tokens( + model='bge-m3-embedding', + credentials={ + }, + texts=[ + ] + ) + + assert num_tokens == 0 From 7b45a5d4525a3745f2dac1a11b26c88629413aea Mon Sep 17 00:00:00 2001 From: Weishan-0 <29089406+Weishan-0@users.noreply.github.com> Date: Thu, 18 Jul 2024 19:37:04 +0800 Subject: [PATCH 39/62] fix: Unable to display images generated by Dall-E 3 (#6155) --- .../provider/builtin/dalle/tools/dalle3.py | 47 +++++++++++++++++-- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/api/core/tools/provider/builtin/dalle/tools/dalle3.py b/api/core/tools/provider/builtin/dalle/tools/dalle3.py index 61609947faf40b..f985deade55aba 100644 --- a/api/core/tools/provider/builtin/dalle/tools/dalle3.py +++ b/api/core/tools/provider/builtin/dalle/tools/dalle3.py @@ -1,5 +1,5 @@ +import base64 import random -from base64 import b64decode from typing import Any, Union from openai import OpenAI @@ -69,11 +69,50 @@ def _invoke(self, result = [] for image in response.data: - result.append(self.create_blob_message(blob=b64decode(image.b64_json), - meta={'mime_type': 'image/png'}, - save_as=self.VARIABLE_KEY.IMAGE.value)) + mime_type, blob_image = DallE3Tool._decode_image(image.b64_json) + blob_message = self.create_blob_message(blob=blob_image, + meta={'mime_type': mime_type}, + save_as=self.VARIABLE_KEY.IMAGE.value) + result.append(blob_message) return result + @staticmethod + def _decode_image(base64_image: str) -> tuple[str, bytes]: + """ + Decode a base64 encoded image. If the image is not prefixed with a MIME type, + it assumes 'image/png' as the default. + + :param base64_image: Base64 encoded image string + :return: A tuple containing the MIME type and the decoded image bytes + """ + if DallE3Tool._is_plain_base64(base64_image): + return 'image/png', base64.b64decode(base64_image) + else: + return DallE3Tool._extract_mime_and_data(base64_image) + + @staticmethod + def _is_plain_base64(encoded_str: str) -> bool: + """ + Check if the given encoded string is plain base64 without a MIME type prefix. + + :param encoded_str: Base64 encoded image string + :return: True if the string is plain base64, False otherwise + """ + return not encoded_str.startswith('data:image') + + @staticmethod + def _extract_mime_and_data(encoded_str: str) -> tuple[str, bytes]: + """ + Extract MIME type and image data from a base64 encoded string with a MIME type prefix. + + :param encoded_str: Base64 encoded image string with MIME type prefix + :return: A tuple containing the MIME type and the decoded image bytes + """ + mime_type = encoded_str.split(';')[0].split(':')[1] + image_data_base64 = encoded_str.split(',')[1] + decoded_data = base64.b64decode(image_data_base64) + return mime_type, decoded_data + @staticmethod def _generate_random_id(length=8): characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' From e493ce99817fdd5ebfc9be36170431e5fd6de579 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Thu, 18 Jul 2024 20:25:28 +0800 Subject: [PATCH 40/62] update clean embedding cache logic (#6434) --- api/schedule/clean_embedding_cache_task.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/schedule/clean_embedding_cache_task.py b/api/schedule/clean_embedding_cache_task.py index 3d49b487c6dc7d..f68c54600a6012 100644 --- a/api/schedule/clean_embedding_cache_task.py +++ b/api/schedule/clean_embedding_cache_task.py @@ -16,16 +16,14 @@ def clean_embedding_cache_task(): clean_days = int(dify_config.CLEAN_DAY_SETTING) start_at = time.perf_counter() thirty_days_ago = datetime.datetime.now() - datetime.timedelta(days=clean_days) - page = 1 while True: try: embeddings = db.session.query(Embedding).filter(Embedding.created_at < thirty_days_ago) \ - .order_by(Embedding.created_at.desc()).paginate(page=page, per_page=100) + .order_by(Embedding.created_at.desc()).limit(100).all() except NotFound: break for embedding in embeddings: db.session.delete(embedding) db.session.commit() - page += 1 end_at = time.perf_counter() click.echo(click.style('Cleaned embedding cache from db success latency: {}'.format(end_at - start_at), fg='green')) From 284ef52bba8d48fbaf8d857feb79c92dc9c4aa3c Mon Sep 17 00:00:00 2001 From: yoyocircle <61043261+yoyocircle@users.noreply.github.com> Date: Thu, 18 Jul 2024 21:54:16 +0800 Subject: [PATCH 41/62] feat: passing the inputs values using `difyChatbotConfig` (#6376) --- .../base/chat/embedded-chatbot/hooks.tsx | 21 ++++++++ web/app/components/base/chat/utils.ts | 20 ++++++++ web/public/embed.js | 51 +++++++++++++++---- web/public/embed.min.js | 32 +----------- 4 files changed, 82 insertions(+), 42 deletions(-) create mode 100644 web/app/components/base/chat/utils.ts diff --git a/web/app/components/base/chat/embedded-chatbot/hooks.tsx b/web/app/components/base/chat/embedded-chatbot/hooks.tsx index 1d88fbefcb9fe5..39d25f57d194e1 100644 --- a/web/app/components/base/chat/embedded-chatbot/hooks.tsx +++ b/web/app/components/base/chat/embedded-chatbot/hooks.tsx @@ -31,6 +31,7 @@ import type { import { addFileInfos, sortAgentSorts } from '@/app/components/tools/utils' import { useToastContext } from '@/app/components/base/toast' import { changeLanguage } from '@/i18n/i18next-config' +import { getProcessedInputsFromUrlParams } from '@/app/components/base/chat/utils' export const useEmbeddedChatbot = () => { const isInstalledApp = false @@ -109,6 +110,7 @@ export const useEmbeddedChatbot = () => { const { t } = useTranslation() const newConversationInputsRef = useRef>({}) const [newConversationInputs, setNewConversationInputs] = useState>({}) + const [initInputs, setInitInputs] = useState>({}) const handleNewConversationInputsChange = useCallback((newInputs: Record) => { newConversationInputsRef.current = newInputs setNewConversationInputs(newInputs) @@ -116,30 +118,49 @@ export const useEmbeddedChatbot = () => { const inputsForms = useMemo(() => { return (appParams?.user_input_form || []).filter((item: any) => item.paragraph || item.select || item['text-input'] || item.number).map((item: any) => { if (item.paragraph) { + let value = initInputs[item.paragraph.variable] + if (value && item.paragraph.max_length && value.length > item.paragraph.max_length) + value = value.slice(0, item.paragraph.max_length) + return { ...item.paragraph, + default: value || item.default, type: 'paragraph', } } if (item.number) { + const convertedNumber = Number(initInputs[item.number.variable]) ?? undefined return { ...item.number, + default: convertedNumber || item.default, type: 'number', } } if (item.select) { + const isInputInOptions = item.select.options.includes(initInputs[item.select.variable]) return { ...item.select, + default: (isInputInOptions ? initInputs[item.select.variable] : undefined) || item.default, type: 'select', } } + let value = initInputs[item['text-input'].variable] + if (value && item['text-input'].max_length && value.length > item['text-input'].max_length) + value = value.slice(0, item['text-input'].max_length) + return { ...item['text-input'], + default: value || item.default, type: 'text-input', } }) }, [appParams]) + + useEffect(() => { + // init inputs from url params + setInitInputs(getProcessedInputsFromUrlParams()) + }, []) useEffect(() => { const conversationInputs: Record = {} diff --git a/web/app/components/base/chat/utils.ts b/web/app/components/base/chat/utils.ts new file mode 100644 index 00000000000000..3fe5050cc7b34d --- /dev/null +++ b/web/app/components/base/chat/utils.ts @@ -0,0 +1,20 @@ +async function decodeBase64AndDecompress(base64String: string) { + const binaryString = atob(base64String) + const compressedUint8Array = Uint8Array.from(binaryString, char => char.charCodeAt(0)) + const decompressedStream = new Response(compressedUint8Array).body.pipeThrough(new DecompressionStream('gzip')) + const decompressedArrayBuffer = await new Response(decompressedStream).arrayBuffer() + return new TextDecoder().decode(decompressedArrayBuffer) +} + +function getProcessedInputsFromUrlParams(): Record { + const urlParams = new URLSearchParams(window.location.search) + const inputs: Record = {} + urlParams.forEach(async (value, key) => { + inputs[key] = await decodeBase64AndDecompress(decodeURIComponent(value)) + }) + return inputs +} + +export { + getProcessedInputsFromUrlParams, +} diff --git a/web/public/embed.js b/web/public/embed.js index 8bff34d8518e91..14420f0c8ca437 100644 --- a/web/public/embed.js +++ b/web/public/embed.js @@ -24,27 +24,55 @@ }; // Main function to embed the chatbot - function embedChatbot() { + async function embedChatbot() { if (!config || !config.token) { console.error(`${configKey} is empty or token is not provided`); return; } + async function compressAndEncodeBase64(input) { + const uint8Array = new TextEncoder().encode(input); + const compressedStream = new Response( + new Blob([uint8Array]).stream().pipeThrough(new CompressionStream('gzip')) + ).arrayBuffer(); + const compressedUint8Array = new Uint8Array(await compressedStream); + return btoa(String.fromCharCode(...compressedUint8Array)); + } + + async function getCompressedInputsFromConfig() { + const inputs = config?.inputs || {}; + const compressedInputs = {}; + await Promise.all( + Object.entries(inputs).map(async ([key, value]) => { + compressedInputs[key] = await compressAndEncodeBase64(value); + }) + ); + return compressedInputs; + } + + const params = new URLSearchParams(await getCompressedInputsFromConfig()); + const baseUrl = config.baseUrl || `https://${config.isDev ? "dev." : ""}udify.app`; + // pre-check the length of the URL + const iframeUrl = `${baseUrl}/chatbot/${config.token}?${params}`; + if(iframeUrl.length > 2048) { + console.error("The URL is too long, please reduce the number of inputs to prevent the bot from failing to load"); + } + // Function to create the iframe for the chatbot function createIframe() { const iframe = document.createElement("iframe"); iframe.allow = "fullscreen;microphone"; iframe.title = "dify chatbot bubble window"; iframe.id = iframeId; - iframe.src = `${baseUrl}/chatbot/${config.token}`; + iframe.src = iframeUrl; iframe.style.cssText = ` - border: none; position: fixed; flex-direction: column; justify-content: space-between; - box-shadow: rgba(150, 150, 150, 0.2) 0px 10px 30px 0px, rgba(150, 150, 150, 0.2) 0px 0px 0px 1px; - bottom: 5rem; right: 1rem; width: 24rem; max-width: calc(100vw - 2rem); height: 40rem; - max-height: calc(100vh - 6rem); border-radius: 0.75rem; display: flex; z-index: 2147483647; + border: none; position: fixed; flex-direction: column; justify-content: space-between; + box-shadow: rgba(150, 150, 150, 0.2) 0px 10px 30px 0px, rgba(150, 150, 150, 0.2) 0px 0px 0px 1px; + bottom: 5rem; right: 1rem; width: 24rem; max-width: calc(100vw - 2rem); height: 40rem; + max-height: calc(100vh - 6rem); border-radius: 0.75rem; display: flex; z-index: 2147483647; overflow: hidden; left: unset; background-color: #F3F4F6; `; @@ -106,19 +134,19 @@ document.head.appendChild(styleSheet); styleSheet.sheet.insertRule(` #${containerDiv.id} { - position: fixed; + position: fixed; bottom: var(--${containerDiv.id}-bottom, 1rem); right: var(--${containerDiv.id}-right, 1rem); left: var(--${containerDiv.id}-left, unset); top: var(--${containerDiv.id}-top, unset); width: var(--${containerDiv.id}-width, 50px); height: var(--${containerDiv.id}-height, 50px); - border-radius: var(--${containerDiv.id}-border-radius, 25px); + border-radius: var(--${containerDiv.id}-border-radius, 25px); background-color: var(--${containerDiv.id}-bg-color, #155EEF); box-shadow: var(--${containerDiv.id}-box-shadow, rgba(0, 0, 0, 0.2) 0px 4px 8px 0px); cursor: pointer; - z-index: 2147483647; - transition: all 0.2s ease-in-out 0s; + z-index: 2147483647; + transition: all 0.2s ease-in-out 0s; } `); styleSheet.sheet.insertRule(` @@ -154,7 +182,8 @@ } else { document.addEventListener('keydown', handleEscKey); } - + + resetIframePosition(); }); diff --git a/web/public/embed.min.js b/web/public/embed.min.js index e37111755f49c3..ec721a204dfeb6 100644 --- a/web/public/embed.min.js +++ b/web/public/embed.min.js @@ -1,31 +1 @@ -!function(){const t="difyChatbotConfig",c="dify-chatbot-bubble-button",a="dify-chatbot-bubble-window",h=window[t],p={open:` - - `,close:` - - `};function e(){if(h&&h.token){const o=h.baseUrl||`https://${h.isDev?"dev.":""}udify.app`;function i(){var e,t,n,i=document.getElementById(a),o=document.getElementById(c);i&&o&&(o=o.getBoundingClientRect(),e=window.innerHeight-o.bottom,t=window.innerWidth-o.right,n=o.left,i.style.bottom=`${e+o.height+5+i.clientHeight>window.innerHeight?e-i.clientHeight-5:e+o.height+5}px`,i.style.right=`${t+i.clientWidth>window.innerWidth?window.innerWidth-n-i.clientWidth:t}px`)}function e(){const n=document.createElement("div");Object.entries(h.containerProps||{}).forEach(([e,t])=>{"className"===e?n.classList.add(...t.split(" ")):"style"===e?"object"==typeof t?Object.assign(n.style,t):n.style.cssText=t:"function"==typeof t?n.addEventListener(e.replace(/^on/,"").toLowerCase(),t):n[e]=t}),n.id=c;var e=document.createElement("style");document.head.appendChild(e),e.sheet.insertRule(` - #${n.id} { - position: fixed; - bottom: var(--${n.id}-bottom, 1rem); - right: var(--${n.id}-right, 1rem); - left: var(--${n.id}-left, unset); - top: var(--${n.id}-top, unset); - width: var(--${n.id}-width, 50px); - height: var(--${n.id}-height, 50px); - border-radius: var(--${n.id}-border-radius, 25px); - background-color: var(--${n.id}-bg-color, #155EEF); - box-shadow: var(--${n.id}-box-shadow, rgba(0, 0, 0, 0.2) 0px 4px 8px 0px); - cursor: pointer; - z-index: 2147483647; - transition: all 0.2s ease-in-out 0s; - } - `),e.sheet.insertRule(` - #${n.id}:hover { - transform: var(--${n.id}-hover-transform, scale(1.1)); - } - `);const t=document.createElement("div");if(t.style.cssText="display: flex; align-items: center; justify-content: center; width: 100%; height: 100%; z-index: 2147483647;",t.innerHTML=p.open,n.appendChild(t),document.body.appendChild(n),n.addEventListener("click",function(){var e=document.getElementById(a);e?(e.style.display="none"===e.style.display?"block":"none",t.innerHTML="none"===e.style.display?p.open:p.close,"none"===e.style.display?document.removeEventListener("keydown",d):document.addEventListener("keydown",d),i()):((e=document.createElement("iframe")).allow="fullscreen;microphone",e.title="dify chatbot bubble window",e.id=a,e.src=o+"/chatbot/"+h.token,e.style.cssText=` - border: none; position: fixed; flex-direction: column; justify-content: space-between; - box-shadow: rgba(150, 150, 150, 0.2) 0px 10px 30px 0px, rgba(150, 150, 150, 0.2) 0px 0px 0px 1px; - bottom: 5rem; right: 1rem; width: 24rem; max-width: calc(100vw - 2rem); height: 40rem; - max-height: calc(100vh - 6rem); border-radius: 0.75rem; display: flex; z-index: 2147483647; - overflow: hidden; left: unset; background-color: #F3F4F6; - `,document.body.appendChild(e),i(),this.title="Exit (ESC)",t.innerHTML=p.close,document.addEventListener("keydown",d))}),h.draggable){var s=n;var l=h.dragAxis||"both";let o=!1,d,r;s.addEventListener("mousedown",function(e){o=!0,d=e.clientX-s.offsetLeft,r=e.clientY-s.offsetTop}),document.addEventListener("mousemove",function(e){var t,n,i;o&&(s.style.transition="none",s.style.cursor="grabbing",(t=document.getElementById(a))&&(t.style.display="none",s.querySelector("div").innerHTML=p.open),t=e.clientX-d,e=window.innerHeight-e.clientY-r,i=s.getBoundingClientRect(),n=window.innerWidth-i.width,i=window.innerHeight-i.height,"x"!==l&&"both"!==l||s.style.setProperty(`--${c}-left`,Math.max(0,Math.min(t,n))+"px"),"y"!==l&&"both"!==l||s.style.setProperty(`--${c}-bottom`,Math.max(0,Math.min(e,i))+"px"))}),document.addEventListener("mouseup",function(){o=!1,s.style.transition="",s.style.cursor="pointer"})}}document.getElementById(c)||e()}else console.error(t+" is empty or token is not provided")}function d(e){var t;"Escape"===e.key&&(e=document.getElementById(a),t=document.getElementById(c),e)&&"none"!==e.style.display&&(e.style.display="none",t.querySelector("div").innerHTML=p.open)}document.addEventListener("keydown",d),h?.dynamicScript?e():document.body.onload=e}(); \ No newline at end of file +!function(){const e="difyChatbotConfig",t="dify-chatbot-bubble-button",n="dify-chatbot-bubble-window",o=window[e],i={open:'\n \n ',close:'\n \n '};async function d(){if(!o||!o.token)return void console.error(`${e} is empty or token is not provided`);const d=new URLSearchParams(await async function(){const e=o?.inputs||{},t={};return await Promise.all(Object.entries(e).map((async([e,n])=>{t[e]=await async function(e){const t=(new TextEncoder).encode(e),n=new Response(new Blob([t]).stream().pipeThrough(new CompressionStream("gzip"))).arrayBuffer(),o=new Uint8Array(await n);return btoa(String.fromCharCode(...o))}(n)}))),t}()),s=`${o.baseUrl||`https://${o.isDev?"dev.":""}udify.app`}/chatbot/${o.token}?${d}`;function c(){const e=document.getElementById(n),o=document.getElementById(t);if(e&&o){const t=o.getBoundingClientRect(),n=window.innerHeight-t.bottom,i=window.innerWidth-t.right,d=t.left;e.style.bottom=`${n+t.height+5+e.clientHeight>window.innerHeight?n-e.clientHeight-5:n+t.height+5}px`,e.style.right=`${i+e.clientWidth>window.innerWidth?window.innerWidth-d-e.clientWidth:i}px`}}s.length>2048&&console.error("The URL is too long, please reduce the number of inputs to prevent the bot from failing to load"),document.getElementById(t)||function(){const e=document.createElement("div");Object.entries(o.containerProps||{}).forEach((([t,n])=>{"className"===t?e.classList.add(...n.split(" ")):"style"===t?"object"==typeof n?Object.assign(e.style,n):e.style.cssText=n:"function"==typeof n?e.addEventListener(t.replace(/^on/,"").toLowerCase(),n):e[t]=n})),e.id=t;const d=document.createElement("style");document.head.appendChild(d),d.sheet.insertRule(`\n #${e.id} {\n position: fixed;\n bottom: var(--${e.id}-bottom, 1rem);\n right: var(--${e.id}-right, 1rem);\n left: var(--${e.id}-left, unset);\n top: var(--${e.id}-top, unset);\n width: var(--${e.id}-width, 50px);\n height: var(--${e.id}-height, 50px);\n border-radius: var(--${e.id}-border-radius, 25px);\n background-color: var(--${e.id}-bg-color, #155EEF);\n box-shadow: var(--${e.id}-box-shadow, rgba(0, 0, 0, 0.2) 0px 4px 8px 0px);\n cursor: pointer;\n z-index: 2147483647;\n transition: all 0.2s ease-in-out 0s;\n }\n `),d.sheet.insertRule(`\n #${e.id}:hover {\n transform: var(--${e.id}-hover-transform, scale(1.1));\n }\n `);const l=document.createElement("div");l.style.cssText="display: flex; align-items: center; justify-content: center; width: 100%; height: 100%; z-index: 2147483647;",l.innerHTML=i.open,e.appendChild(l),document.body.appendChild(e),e.addEventListener("click",(function(){const e=document.getElementById(n);if(!e)return function(){const e=document.createElement("iframe");e.allow="fullscreen;microphone",e.title="dify chatbot bubble window",e.id=n,e.src=s,e.style.cssText="\n border: none; position: fixed; flex-direction: column; justify-content: space-between;\n box-shadow: rgba(150, 150, 150, 0.2) 0px 10px 30px 0px, rgba(150, 150, 150, 0.2) 0px 0px 0px 1px;\n bottom: 5rem; right: 1rem; width: 24rem; max-width: calc(100vw - 2rem); height: 40rem;\n max-height: calc(100vh - 6rem); border-radius: 0.75rem; display: flex; z-index: 2147483647;\n overflow: hidden; left: unset; background-color: #F3F4F6;\n ",document.body.appendChild(e)}(),c(),this.title="Exit (ESC)",l.innerHTML=i.close,void document.addEventListener("keydown",r);e.style.display="none"===e.style.display?"block":"none",l.innerHTML="none"===e.style.display?i.open:i.close,"none"===e.style.display?document.removeEventListener("keydown",r):document.addEventListener("keydown",r),c()})),o.draggable&&function(e,o){let d,r,s=!1;function c(t){s=!0,d=t.clientX-e.offsetLeft,r=t.clientY-e.offsetTop}function l(c){if(!s)return;e.style.transition="none",e.style.cursor="grabbing";const l=document.getElementById(n);l&&(l.style.display="none",e.querySelector("div").innerHTML=i.open);const a=c.clientX-d,h=window.innerHeight-c.clientY-r,p=e.getBoundingClientRect(),u=window.innerWidth-p.width,m=window.innerHeight-p.height;"x"!==o&&"both"!==o||e.style.setProperty(`--${t}-left`,`${Math.max(0,Math.min(a,u))}px`),"y"!==o&&"both"!==o||e.style.setProperty(`--${t}-bottom`,`${Math.max(0,Math.min(h,m))}px`)}function a(){s=!1,e.style.transition="",e.style.cursor="pointer"}e.addEventListener("mousedown",c),document.addEventListener("mousemove",l),document.addEventListener("mouseup",a)}(e,o.dragAxis||"both")}()}function r(e){if("Escape"===e.key){const e=document.getElementById(n),o=document.getElementById(t);e&&"none"!==e.style.display&&(e.style.display="none",o.querySelector("div").innerHTML=i.open)}}document.addEventListener("keydown",r),o?.dynamicScript?d():document.body.onload=d}(); From dad3fd2dc153b9b6d4b8d6a40a26bcb6d0d2f6d5 Mon Sep 17 00:00:00 2001 From: takatost Date: Fri, 19 Jul 2024 01:53:43 +0800 Subject: [PATCH 42/62] feat: add gpt-4o-mini (#6442) --- .../model_providers/openai/llm/_position.yaml | 2 + .../openai/llm/gpt-4o-mini-2024-07-18.yaml | 44 +++++++++++++++++++ .../openai/llm/gpt-4o-mini.yaml | 44 +++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini-2024-07-18.yaml create mode 100644 api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini.yaml diff --git a/api/core/model_runtime/model_providers/openai/llm/_position.yaml b/api/core/model_runtime/model_providers/openai/llm/_position.yaml index 566055e3f76704..91b9215829b00b 100644 --- a/api/core/model_runtime/model_providers/openai/llm/_position.yaml +++ b/api/core/model_runtime/model_providers/openai/llm/_position.yaml @@ -1,6 +1,8 @@ - gpt-4 - gpt-4o - gpt-4o-2024-05-13 +- gpt-4o-mini +- gpt-4o-mini-2024-07-18 - gpt-4-turbo - gpt-4-turbo-2024-04-09 - gpt-4-turbo-preview diff --git a/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini-2024-07-18.yaml b/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini-2024-07-18.yaml new file mode 100644 index 00000000000000..7304a58affc6f0 --- /dev/null +++ b/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini-2024-07-18.yaml @@ -0,0 +1,44 @@ +model: gpt-4o-mini-2024-07-18 +label: + zh_Hans: gpt-4o-mini-2024-07-18 + en_US: gpt-4o-mini-2024-07-18 +model_type: llm +features: + - multi-tool-call + - agent-thought + - stream-tool-call + - vision +model_properties: + mode: chat + context_size: 128000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: presence_penalty + use_template: presence_penalty + - name: frequency_penalty + use_template: frequency_penalty + - name: max_tokens + use_template: max_tokens + default: 512 + min: 1 + max: 4096 + - name: response_format + label: + zh_Hans: 回复格式 + en_US: response_format + type: string + help: + zh_Hans: 指定模型必须输出的格式 + en_US: specifying the format that the model must output + required: false + options: + - text + - json_object +pricing: + input: '0.15' + output: '0.60' + unit: '0.000001' + currency: USD diff --git a/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini.yaml b/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini.yaml new file mode 100644 index 00000000000000..053cf7d7c10f4e --- /dev/null +++ b/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini.yaml @@ -0,0 +1,44 @@ +model: gpt-4o-mini +label: + zh_Hans: gpt-4o-mini + en_US: gpt-4o-mini +model_type: llm +features: + - multi-tool-call + - agent-thought + - stream-tool-call + - vision +model_properties: + mode: chat + context_size: 128000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: presence_penalty + use_template: presence_penalty + - name: frequency_penalty + use_template: frequency_penalty + - name: max_tokens + use_template: max_tokens + default: 512 + min: 1 + max: 4096 + - name: response_format + label: + zh_Hans: 回复格式 + en_US: response_format + type: string + help: + zh_Hans: 指定模型必须输出的格式 + en_US: specifying the format that the model must output + required: false + options: + - text + - json_object +pricing: + input: '0.15' + output: '0.60' + unit: '0.000001' + currency: USD From 8e49146a3523dcbf781a7f41aa874b23bd338b29 Mon Sep 17 00:00:00 2001 From: Richards Tu <142148415+richards199999@users.noreply.github.com> Date: Fri, 19 Jul 2024 07:38:15 +0800 Subject: [PATCH 43/62] [EMERGENCY] Fix Anthropic header issue (#6445) --- .../model_runtime/model_providers/anthropic/llm/llm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/core/model_runtime/model_providers/anthropic/llm/llm.py b/api/core/model_runtime/model_providers/anthropic/llm/llm.py index 787c3baaaa9992..107efe4867841c 100644 --- a/api/core/model_runtime/model_providers/anthropic/llm/llm.py +++ b/api/core/model_runtime/model_providers/anthropic/llm/llm.py @@ -114,9 +114,9 @@ def _chat_generate(self, model: str, credentials: dict, extra_model_kwargs['system'] = system # Add the new header for claude-3-5-sonnet-20240620 model - headers = {} + extra_headers = {} if model == "claude-3-5-sonnet-20240620": - headers["anthropic-beta"] = "max-tokens-3-5-sonnet-2024-07-15" + extra_headers["anthropic-beta"] = "max-tokens-3-5-sonnet-2024-07-15" if tools: extra_model_kwargs['tools'] = [ @@ -126,7 +126,7 @@ def _chat_generate(self, model: str, credentials: dict, model=model, messages=prompt_message_dicts, stream=stream, - headers=headers, + extra_headers=extra_headers, **model_parameters, **extra_model_kwargs ) @@ -136,7 +136,7 @@ def _chat_generate(self, model: str, credentials: dict, model=model, messages=prompt_message_dicts, stream=stream, - headers=headers, + extra_headers=extra_headers, **model_parameters, **extra_model_kwargs ) From 2ba05b041f45dfba96867f0bcedc908c43b38bc2 Mon Sep 17 00:00:00 2001 From: Waffle <52460705+ox01024@users.noreply.github.com> Date: Fri, 19 Jul 2024 10:57:45 +0800 Subject: [PATCH 44/62] refactor(myscale):Set the default value of the myscale vector db in DifyConfig. (#6441) --- api/configs/middleware/vdb/myscale_config.py | 23 +++++++++---------- .../datasource/vdb/myscale/myscale_vector.py | 13 +++++------ 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/api/configs/middleware/vdb/myscale_config.py b/api/configs/middleware/vdb/myscale_config.py index e513cad0e8a100..895cd6f1769d6e 100644 --- a/api/configs/middleware/vdb/myscale_config.py +++ b/api/configs/middleware/vdb/myscale_config.py @@ -1,4 +1,3 @@ -from typing import Optional from pydantic import BaseModel, Field, PositiveInt @@ -8,32 +7,32 @@ class MyScaleConfig(BaseModel): MyScale configs """ - MYSCALE_HOST: Optional[str] = Field( + MYSCALE_HOST: str = Field( description='MyScale host', - default=None, + default='localhost', ) - MYSCALE_PORT: Optional[PositiveInt] = Field( + MYSCALE_PORT: PositiveInt = Field( description='MyScale port', default=8123, ) - MYSCALE_USER: Optional[str] = Field( + MYSCALE_USER: str = Field( description='MyScale user', - default=None, + default='default', ) - MYSCALE_PASSWORD: Optional[str] = Field( + MYSCALE_PASSWORD: str = Field( description='MyScale password', - default=None, + default='', ) - MYSCALE_DATABASE: Optional[str] = Field( + MYSCALE_DATABASE: str = Field( description='MyScale database name', - default=None, + default='default', ) - MYSCALE_FTS_PARAMS: Optional[str] = Field( + MYSCALE_FTS_PARAMS: str = Field( description='MyScale fts index parameters', - default=None, + default='', ) diff --git a/api/core/rag/datasource/vdb/myscale/myscale_vector.py b/api/core/rag/datasource/vdb/myscale/myscale_vector.py index 33ee8259c524f2..241b5a8414c476 100644 --- a/api/core/rag/datasource/vdb/myscale/myscale_vector.py +++ b/api/core/rag/datasource/vdb/myscale/myscale_vector.py @@ -159,12 +159,11 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings return MyScaleVector( collection_name=collection_name, config=MyScaleConfig( - # TODO: I think setting those values as the default config would be a better option. - host=dify_config.MYSCALE_HOST or "localhost", - port=dify_config.MYSCALE_PORT or 8123, - user=dify_config.MYSCALE_USER or "default", - password=dify_config.MYSCALE_PASSWORD or "", - database=dify_config.MYSCALE_DATABASE or "default", - fts_params=dify_config.MYSCALE_FTS_PARAMS or "", + host=dify_config.MYSCALE_HOST, + port=dify_config.MYSCALE_PORT, + user=dify_config.MYSCALE_USER, + password=dify_config.MYSCALE_PASSWORD, + database=dify_config.MYSCALE_DATABASE, + fts_params=dify_config.MYSCALE_FTS_PARAMS, ), ) From a5fcd91ba5e669806672de54a3be62f0bb2eeea8 Mon Sep 17 00:00:00 2001 From: Sangmin Ahn Date: Fri, 19 Jul 2024 13:54:15 +0900 Subject: [PATCH 45/62] chore: make text generation timeout duration configurable (#6450) --- web/app/components/share/text-generation/result/index.tsx | 3 ++- web/config/index.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/web/app/components/share/text-generation/result/index.tsx b/web/app/components/share/text-generation/result/index.tsx index f924f206f494b9..caa2f9183eddea 100644 --- a/web/app/components/share/text-generation/result/index.tsx +++ b/web/app/components/share/text-generation/result/index.tsx @@ -19,6 +19,7 @@ import { NodeRunningStatus, WorkflowRunningStatus } from '@/app/components/workf import type { WorkflowProcess } from '@/app/components/base/chat/types' import { sleep } from '@/utils' import type { SiteInfo } from '@/models/share' +import { TEXT_GENERATION_TIMEOUT_MS } from '@/config' export type IResultProps = { isWorkflow: boolean @@ -186,7 +187,7 @@ const Result: FC = ({ let isEnd = false let isTimeout = false; (async () => { - await sleep(1000 * 60) // 1min timeout + await sleep(TEXT_GENERATION_TIMEOUT_MS) if (!isEnd) { setRespondingFalse() onCompleted(getCompletionRes(), taskId, false) diff --git a/web/config/index.ts b/web/config/index.ts index 600becf68cadda..9bb4efcd5eba3e 100644 --- a/web/config/index.ts +++ b/web/config/index.ts @@ -245,3 +245,5 @@ Thought: {{agent_scratchpad}} } export const VAR_REGEX = /\{\{(#[a-zA-Z0-9_-]{1,50}(\.[a-zA-Z_][a-zA-Z0-9_]{0,29}){1,10}#)\}\}/gi + +export const TEXT_GENERATION_TIMEOUT_MS = 60000 From ea45496a7453191c8a743563e7729fd885f099d9 Mon Sep 17 00:00:00 2001 From: Weaxs <459312872@qq.com> Date: Fri, 19 Jul 2024 13:08:39 +0800 Subject: [PATCH 46/62] update ernie models (#6454) --- .../wenxin/llm/ernie-3.5-8k-0205.yaml | 1 + ...review => ernie-4.0-turbo-8k-preview.yaml} | 0 .../wenxin/llm/ernie-4.0-turbo-8k.yaml | 40 +++++++++++++++++++ .../wenxin/llm/ernie-character-8k-0321.yaml | 1 + .../wenxin/llm/ernie-character-8k.yaml | 30 ++++++++++++++ .../wenxin/llm/ernie-lite-8k-0308.yaml | 1 + .../wenxin/llm/ernie-lite-8k-0922.yaml | 1 + .../model_providers/wenxin/llm/ernie_bot.py | 6 ++- 8 files changed, 79 insertions(+), 1 deletion(-) rename api/core/model_runtime/model_providers/wenxin/llm/{ernie-4.0-turbo-8k-preview => ernie-4.0-turbo-8k-preview.yaml} (100%) create mode 100644 api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-turbo-8k.yaml create mode 100644 api/core/model_runtime/model_providers/wenxin/llm/ernie-character-8k.yaml diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-3.5-8k-0205.yaml b/api/core/model_runtime/model_providers/wenxin/llm/ernie-3.5-8k-0205.yaml index 34f73dccbba3e9..b308abcb323f3d 100644 --- a/api/core/model_runtime/model_providers/wenxin/llm/ernie-3.5-8k-0205.yaml +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie-3.5-8k-0205.yaml @@ -35,3 +35,4 @@ parameter_rules: zh_Hans: 禁用模型自行进行外部搜索。 en_US: Disable the model to perform external search. required: false +deprecated: true diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-turbo-8k-preview b/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-turbo-8k-preview.yaml similarity index 100% rename from api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-turbo-8k-preview rename to api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-turbo-8k-preview.yaml diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-turbo-8k.yaml b/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-turbo-8k.yaml new file mode 100644 index 00000000000000..2887a510d05157 --- /dev/null +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-turbo-8k.yaml @@ -0,0 +1,40 @@ +model: ernie-4.0-turbo-8k +label: + en_US: Ernie-4.0-turbo-8K +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 8192 +parameter_rules: + - name: temperature + use_template: temperature + min: 0.1 + max: 1.0 + default: 0.8 + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 2 + max: 2048 + - name: presence_penalty + use_template: presence_penalty + default: 1.0 + min: 1.0 + max: 2.0 + - name: frequency_penalty + use_template: frequency_penalty + - name: response_format + use_template: response_format + - name: disable_search + label: + zh_Hans: 禁用搜索 + en_US: Disable Search + type: boolean + help: + zh_Hans: 禁用模型自行进行外部搜索。 + en_US: Disable the model to perform external search. + required: false diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-character-8k-0321.yaml b/api/core/model_runtime/model_providers/wenxin/llm/ernie-character-8k-0321.yaml index 52e1dc832d9ad3..74451ff9e356e6 100644 --- a/api/core/model_runtime/model_providers/wenxin/llm/ernie-character-8k-0321.yaml +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie-character-8k-0321.yaml @@ -28,3 +28,4 @@ parameter_rules: default: 1.0 min: 1.0 max: 2.0 +deprecated: true diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-character-8k.yaml b/api/core/model_runtime/model_providers/wenxin/llm/ernie-character-8k.yaml new file mode 100644 index 00000000000000..4b11b3e895be9f --- /dev/null +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie-character-8k.yaml @@ -0,0 +1,30 @@ +model: ernie-character-8k-0321 +label: + en_US: ERNIE-Character-8K +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 8192 +parameter_rules: + - name: temperature + use_template: temperature + min: 0.1 + max: 1.0 + default: 0.95 + - name: top_p + use_template: top_p + min: 0 + max: 1.0 + default: 0.7 + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 2 + max: 1024 + - name: presence_penalty + use_template: presence_penalty + default: 1.0 + min: 1.0 + max: 2.0 diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0308.yaml b/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0308.yaml index 78325c1d6432e4..97ecb03f87b623 100644 --- a/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0308.yaml +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0308.yaml @@ -28,3 +28,4 @@ parameter_rules: default: 1.0 min: 1.0 max: 2.0 +deprecated: true diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0922.yaml b/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0922.yaml index ebb47417cc94e1..7410ce51df00ef 100644 --- a/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0922.yaml +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-8k-0922.yaml @@ -28,3 +28,4 @@ parameter_rules: default: 1.0 min: 1.0 max: 2.0 +deprecated: true diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py b/api/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py index 9aeab04cd229ca..bc7f29cf6ea538 100644 --- a/api/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py @@ -97,6 +97,7 @@ def get_access_token(api_key: str, secret_key: str) -> 'BaiduAccessToken': baidu_access_tokens_lock.release() return token + class ErnieMessage: class Role(Enum): USER = 'user' @@ -137,7 +138,9 @@ class ErnieBotModel: 'ernie-speed-appbuilder': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ai_apaas', 'ernie-lite-8k-0922': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/eb-instant', 'ernie-lite-8k-0308': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ernie-lite-8k', + 'ernie-character-8k': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ernie-char-8k', 'ernie-character-8k-0321': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ernie-char-8k', + 'ernie-4.0-tutbo-8k': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ernie-4.0-turbo-8k', 'ernie-4.0-tutbo-8k-preview': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ernie-4.0-turbo-8k-preview', } @@ -149,7 +152,8 @@ class ErnieBotModel: 'ernie-3.5-8k-1222', 'ernie-3.5-4k-0205', 'ernie-3.5-128k', - 'ernie-4.0-8k' + 'ernie-4.0-8k', + 'ernie-4.0-turbo-8k', 'ernie-4.0-turbo-8k-preview' ] From 9e168f9d1c22c9b900060db58a99c4c42978f397 Mon Sep 17 00:00:00 2001 From: sino Date: Fri, 19 Jul 2024 13:09:41 +0800 Subject: [PATCH 47/62] feat: support gpt-4o-mini for openrouter provider (#6447) --- .../openrouter/llm/_position.yaml | 1 + .../openrouter/llm/gpt-4o-mini.yaml | 43 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 api/core/model_runtime/model_providers/openrouter/llm/gpt-4o-mini.yaml diff --git a/api/core/model_runtime/model_providers/openrouter/llm/_position.yaml b/api/core/model_runtime/model_providers/openrouter/llm/_position.yaml index 51131249e587fd..fd4ed1109dfff0 100644 --- a/api/core/model_runtime/model_providers/openrouter/llm/_position.yaml +++ b/api/core/model_runtime/model_providers/openrouter/llm/_position.yaml @@ -1,4 +1,5 @@ - openai/gpt-4o +- openai/gpt-4o-mini - openai/gpt-4 - openai/gpt-4-32k - openai/gpt-3.5-turbo diff --git a/api/core/model_runtime/model_providers/openrouter/llm/gpt-4o-mini.yaml b/api/core/model_runtime/model_providers/openrouter/llm/gpt-4o-mini.yaml new file mode 100644 index 00000000000000..0da5bb877ff8fe --- /dev/null +++ b/api/core/model_runtime/model_providers/openrouter/llm/gpt-4o-mini.yaml @@ -0,0 +1,43 @@ +model: openai/gpt-4o-mini +label: + en_US: gpt-4o-mini +model_type: llm +features: + - multi-tool-call + - agent-thought + - stream-tool-call + - vision +model_properties: + mode: chat + context_size: 128000 +parameter_rules: + - name: temperature + use_template: temperature + - name: top_p + use_template: top_p + - name: presence_penalty + use_template: presence_penalty + - name: frequency_penalty + use_template: frequency_penalty + - name: max_tokens + use_template: max_tokens + default: 512 + min: 1 + max: 4096 + - name: response_format + label: + zh_Hans: 回复格式 + en_US: response_format + type: string + help: + zh_Hans: 指定模型必须输出的格式 + en_US: specifying the format that the model must output + required: false + options: + - text + - json_object +pricing: + input: "0.15" + output: "0.60" + unit: "0.000001" + currency: USD From 57729823a081eba2d4e04f8d0de0194b4663539e Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Fri, 19 Jul 2024 13:48:13 +0800 Subject: [PATCH 48/62] fix wrong method using (#6459) --- api/services/dataset_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index e7af975009e2a7..d5a54ba731ca6a 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -845,7 +845,7 @@ def save_document_with_dataset_id( 'only_main_content': website_info.get('only_main_content', False), 'mode': 'crawl', } - if url.length > 255: + if len(url) > 255: document_name = url[:200] + '...' else: document_name = url From 3da854fe403fb785064670f1a54b77e5b2e89232 Mon Sep 17 00:00:00 2001 From: Joel Date: Fri, 19 Jul 2024 16:39:49 +0800 Subject: [PATCH 49/62] chore: some components upgrage to new ui (#6468) --- web/app/components/base/switch/index.tsx | 6 +++--- .../workflow/nodes/_base/components/field.tsx | 6 +++--- .../nodes/_base/components/next-step/add.tsx | 12 ++++++------ .../nodes/_base/components/next-step/index.tsx | 2 +- .../nodes/_base/components/next-step/item.tsx | 4 ++-- .../nodes/_base/components/next-step/line.tsx | 18 +++++++++--------- .../nodes/_base/components/output-vars.tsx | 14 ++++++++------ .../components/workflow/nodes/_base/node.tsx | 10 +++++----- .../components/workflow/nodes/_base/panel.tsx | 18 +++++++++--------- web/app/styles/globals.css | 6 ++++++ 10 files changed, 52 insertions(+), 44 deletions(-) diff --git a/web/app/components/base/switch/index.tsx b/web/app/components/base/switch/index.tsx index 0b025ab38b9b6d..b3d60672048a55 100644 --- a/web/app/components/base/switch/index.tsx +++ b/web/app/components/base/switch/index.tsx @@ -47,8 +47,8 @@ const Switch = ({ onChange, size = 'lg', defaultValue = false, disabled = false, }} className={classNames( wrapStyle[size], - enabled ? 'bg-blue-600' : 'bg-gray-200', - 'relative inline-flex flex-shrink-0 cursor-pointer rounded-full border-2 border-transparent transition-colors duration-200 ease-in-out', + enabled ? 'bg-components-toggle-bg' : 'bg-components-toggle-bg-unchecked', + 'relative inline-flex flex-shrink-0 cursor-pointer rounded-[5px] border-2 border-transparent transition-colors duration-200 ease-in-out', disabled ? '!opacity-50 !cursor-not-allowed' : '', className, )} @@ -58,7 +58,7 @@ const Switch = ({ onChange, size = 'lg', defaultValue = false, disabled = false, className={classNames( circleStyle[size], enabled ? translateLeft[size] : 'translate-x-0', - 'pointer-events-none inline-block transform rounded-full bg-white shadow ring-0 transition duration-200 ease-in-out', + 'pointer-events-none inline-block transform rounded-[3px] bg-components-toggle-knob shadow ring-0 transition duration-200 ease-in-out', )} /> diff --git a/web/app/components/workflow/nodes/_base/components/field.tsx b/web/app/components/workflow/nodes/_base/components/field.tsx index 1301e9f2ed9201..344fc3d7086917 100644 --- a/web/app/components/workflow/nodes/_base/components/field.tsx +++ b/web/app/components/workflow/nodes/_base/components/field.tsx @@ -38,13 +38,13 @@ const Filed: FC = ({ onClick={() => supportFold && toggleFold()} className={cn('flex justify-between items-center', supportFold && 'cursor-pointer')}>
-
{title}
+
{title}
{tooltip && ( {tooltip}
}> - + )} @@ -52,7 +52,7 @@ const Filed: FC = ({
{operations &&
{operations}
} {supportFold && ( - + )}
diff --git a/web/app/components/workflow/nodes/_base/components/next-step/add.tsx b/web/app/components/workflow/nodes/_base/components/next-step/add.tsx index df90fa156b2ae7..0ab0c8e39e2f6e 100644 --- a/web/app/components/workflow/nodes/_base/components/next-step/add.tsx +++ b/web/app/components/workflow/nodes/_base/components/next-step/add.tsx @@ -51,23 +51,23 @@ const Add = ({ return (
{ branchName && (
-
{branchName.toLocaleUpperCase()}
+
{branchName.toLocaleUpperCase()}
) } -
+
{t('workflow.panel.selectNextStep')} diff --git a/web/app/components/workflow/nodes/_base/components/next-step/index.tsx b/web/app/components/workflow/nodes/_base/components/next-step/index.tsx index a6fd940f63effe..261eb3fac71d95 100644 --- a/web/app/components/workflow/nodes/_base/components/next-step/index.tsx +++ b/web/app/components/workflow/nodes/_base/components/next-step/index.tsx @@ -33,7 +33,7 @@ const NextStep = ({ return (
-
+
{ branchName && ( @@ -75,7 +75,7 @@ const Item = ({ toolIcon={toolIcon} className='shrink-0 mr-1.5' /> -
{data.title}
+
{data.title}
{ !nodesReadOnly && ( + - ) @@ -37,8 +37,8 @@ const Line = ({ ) } @@ -47,7 +47,7 @@ const Line = ({ y={index * 48 + 18 - 2} width={1} height={4} - fill='#98A2B3' + className='fill-divider-soild-alt' /> )) diff --git a/web/app/components/workflow/nodes/_base/components/output-vars.tsx b/web/app/components/workflow/nodes/_base/components/output-vars.tsx index 401a5d6a3e5b70..4b7f9fc12e1a51 100644 --- a/web/app/components/workflow/nodes/_base/components/output-vars.tsx +++ b/web/app/components/workflow/nodes/_base/components/output-vars.tsx @@ -3,8 +3,10 @@ import type { FC } from 'react' import React from 'react' import { useTranslation } from 'react-i18next' import { useBoolean } from 'ahooks' +import { + RiArrowDownSLine, +} from '@remixicon/react' import cn from '@/utils/classnames' -import { ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows' type Props = { className?: string @@ -25,9 +27,9 @@ const OutputVars: FC = ({
+ className={cn(className, 'flex justify-between system-sm-semibold-uppercase text-text-secondary cursor-pointer')}>
{title || t('workflow.nodes.common.outputVars')}
- +
{!isFold && (
@@ -57,10 +59,10 @@ export const VarItem: FC = ({ return (
-
{name}
-
{type}
+
{name}
+
{type}
-
+
{description} {subItems && (
diff --git a/web/app/components/workflow/nodes/_base/node.tsx b/web/app/components/workflow/nodes/_base/node.tsx index 6d1e522e663c6f..2cef91f1b16a00 100644 --- a/web/app/components/workflow/nodes/_base/node.tsx +++ b/web/app/components/workflow/nodes/_base/node.tsx @@ -79,7 +79,7 @@ const BaseNode: FC = ({
= ({ className={cn( 'group relative pb-1 shadow-xs', 'border border-transparent rounded-[15px]', - data.type !== BlockEnum.Iteration && 'w-[240px] bg-[#fcfdff]', - data.type === BlockEnum.Iteration && 'flex flex-col w-full h-full bg-[#fcfdff]/80', + data.type !== BlockEnum.Iteration && 'w-[240px] bg-workflow-block-bg', + data.type === BlockEnum.Iteration && 'flex flex-col w-full h-full bg-workflow-block-bg/80', !data._runningStatus && 'hover:shadow-lg', showRunningBorder && '!border-primary-500', showSuccessBorder && '!border-[#12B76A]', @@ -156,7 +156,7 @@ const BaseNode: FC = ({ />
{data.title}
@@ -197,7 +197,7 @@ const BaseNode: FC = ({ } { data.desc && data.type !== BlockEnum.Iteration && ( -
+
{data.desc}
) diff --git a/web/app/components/workflow/nodes/_base/panel.tsx b/web/app/components/workflow/nodes/_base/panel.tsx index 83d05cbff8ffe7..269d8110dca0ae 100644 --- a/web/app/components/workflow/nodes/_base/panel.tsx +++ b/web/app/components/workflow/nodes/_base/panel.tsx @@ -98,21 +98,21 @@ const BasePanel: FC = ({ return (
-
+
-
+
= ({ handleSyncWorkflowDraft(true) }} > - +
) } -
+
handleNodeSelect(id, true)} > - +
@@ -166,10 +166,10 @@ const BasePanel: FC = ({ { !!availableNextBlocks.length && (
-
+
{t('workflow.panel.nextStep').toLocaleUpperCase()}
-
+
{t('workflow.panel.addNextStep')}
diff --git a/web/app/styles/globals.css b/web/app/styles/globals.css index 1c13e7eb5fb58b..9f23c3d473a1d6 100644 --- a/web/app/styles/globals.css +++ b/web/app/styles/globals.css @@ -276,6 +276,10 @@ button:focus-within { line-height: 24px; } +[class*='code-'] { + @apply font-mono; +} + .code-xs-regular { font-size: 12px; font-weight: 400; @@ -563,6 +567,7 @@ button:focus-within { font-weight: 700; line-height: 1.2; } + /* font define end */ /* border radius start */ @@ -625,6 +630,7 @@ button:focus-within { .radius-full { border-radius: 64px; } + /* border radius end */ .link { From 0bb2b285da1cce2d8ccd1ac71cf80d0a37e1f4e7 Mon Sep 17 00:00:00 2001 From: moqimoqidea <39821951+moqimoqidea@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:05:20 +0800 Subject: [PATCH 50/62] Update CONTRIBUTING_JA "installation FAQ" link. (#6469) --- CONTRIBUTING_JA.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING_JA.md b/CONTRIBUTING_JA.md index e8f5456a3c44b7..1ce8436a7870d1 100644 --- a/CONTRIBUTING_JA.md +++ b/CONTRIBUTING_JA.md @@ -82,7 +82,7 @@ Dify はバックエンドとフロントエンドから構成されています まず`cd api/`でバックエンドのディレクトリに移動し、[Backend README](api/README.md)に従ってインストールします。 次に別のターミナルで、`cd web/`でフロントエンドのディレクトリに移動し、[Frontend README](web/README.md)に従ってインストールしてください。 -よくある問題とトラブルシューティングの手順については、[installation FAQ](https://docs.dify.ai/getting-started/faq/install-faq) を確認してください。 +よくある問題とトラブルシューティングの手順については、[installation FAQ](https://docs.dify.ai/v/japanese/learn-more/faq/install-faq) を確認してください。 ### 5. ブラウザで dify にアクセスする From 90372932fe12ecf840da6670c38da46efbf74f24 Mon Sep 17 00:00:00 2001 From: moqimoqidea <39821951+moqimoqidea@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:05:30 +0800 Subject: [PATCH 51/62] Update CONTRIBUTING "installation FAQ" link. (#6471) --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ce7ad7db98d5f9..f810584f24115c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,7 +81,7 @@ Dify requires the following dependencies to build, make sure they're installed o Dify is composed of a backend and a frontend. Navigate to the backend directory by `cd api/`, then follow the [Backend README](api/README.md) to install it. In a separate terminal, navigate to the frontend directory by `cd web/`, then follow the [Frontend README](web/README.md) to install. -Check the [installation FAQ](https://docs.dify.ai/getting-started/faq/install-faq) for a list of common issues and steps to troubleshoot. +Check the [installation FAQ](https://docs.dify.ai/learn-more/faq/self-host-faq) for a list of common issues and steps to troubleshoot. ### 5. Visit dify in your browser From 47e5dc218ab80c875c580311772e9d24e76f2b9d Mon Sep 17 00:00:00 2001 From: moqimoqidea <39821951+moqimoqidea@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:06:32 +0800 Subject: [PATCH 52/62] =?UTF-8?q?Update=20CONTRIBUTING=5FCN=20"=E5=AE=89?= =?UTF-8?q?=E8=A3=85=E5=B8=B8=E8=A7=81=E9=97=AE=E9=A2=98=E8=A7=A3=E7=AD=94?= =?UTF-8?q?"=20link.=20(#6470)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CONTRIBUTING_CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING_CN.md b/CONTRIBUTING_CN.md index 08fd34e117ba61..303c2513f53b9b 100644 --- a/CONTRIBUTING_CN.md +++ b/CONTRIBUTING_CN.md @@ -77,7 +77,7 @@ Dify 依赖以下工具和库： Dify 由后端和前端组成。通过 `cd api/` 导航到后端目录，然后按照 [后端 README](api/README.md) 进行安装。在另一个终端中，通过 `cd web/` 导航到前端目录，然后按照 [前端 README](web/README.md) 进行安装。 -查看 [安装常见问题解答](https://docs.dify.ai/getting-started/faq/install-faq) 以获取常见问题列表和故障排除步骤。 +查看 [安装常见问题解答](https://docs.dify.ai/v/zh-hans/learn-more/faq/install-faq) 以获取常见问题列表和故障排除步骤。 ### 5. 在浏览器中访问 Dify From 4f9f175f258ab6216b91105bfbb93a097a0c6875 Mon Sep 17 00:00:00 2001 From: sino Date: Fri, 19 Jul 2024 18:24:58 +0800 Subject: [PATCH 53/62] fix: correct gpt-4o-mini max token (#6472) Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> --- .../model_providers/openai/llm/gpt-4o-mini-2024-07-18.yaml | 2 +- .../model_runtime/model_providers/openai/llm/gpt-4o-mini.yaml | 2 +- .../model_providers/openrouter/llm/gpt-4o-mini.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini-2024-07-18.yaml b/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini-2024-07-18.yaml index 7304a58affc6f0..6f23e0647d6eec 100644 --- a/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini-2024-07-18.yaml +++ b/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini-2024-07-18.yaml @@ -24,7 +24,7 @@ parameter_rules: use_template: max_tokens default: 512 min: 1 - max: 4096 + max: 16384 - name: response_format label: zh_Hans: 回复格式 diff --git a/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini.yaml b/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini.yaml index 053cf7d7c10f4e..b97fbf8aabcae4 100644 --- a/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini.yaml +++ b/api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini.yaml @@ -24,7 +24,7 @@ parameter_rules: use_template: max_tokens default: 512 min: 1 - max: 4096 + max: 16384 - name: response_format label: zh_Hans: 回复格式 diff --git a/api/core/model_runtime/model_providers/openrouter/llm/gpt-4o-mini.yaml b/api/core/model_runtime/model_providers/openrouter/llm/gpt-4o-mini.yaml index 0da5bb877ff8fe..de0bad413653eb 100644 --- a/api/core/model_runtime/model_providers/openrouter/llm/gpt-4o-mini.yaml +++ b/api/core/model_runtime/model_providers/openrouter/llm/gpt-4o-mini.yaml @@ -23,7 +23,7 @@ parameter_rules: use_template: max_tokens default: 512 min: 1 - max: 4096 + max: 16384 - name: response_format label: zh_Hans: 回复格式 From 48f872a68c061298007c5fd604c4bfa560aa99ba Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Fri, 19 Jul 2024 18:37:42 +0800 Subject: [PATCH 54/62] fix: build error (#6480) --- web/app/(commonLayout)/tools/page.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/web/app/(commonLayout)/tools/page.tsx b/web/app/(commonLayout)/tools/page.tsx index 4e64d8c0dfe8d3..1b08d54ba3e424 100644 --- a/web/app/(commonLayout)/tools/page.tsx +++ b/web/app/(commonLayout)/tools/page.tsx @@ -12,15 +12,16 @@ const Layout: FC = () => { const { isCurrentWorkspaceDatasetOperator } = useAppContext() useEffect(() => { - document.title = `${t('tools.title')} - Dify` + if (typeof window !== 'undefined') + document.title = `${t('tools.title')} - Dify` if (isCurrentWorkspaceDatasetOperator) return router.replace('/datasets') - }, []) + }, [isCurrentWorkspaceDatasetOperator, router, t]) useEffect(() => { if (isCurrentWorkspaceDatasetOperator) return router.replace('/datasets') - }, [isCurrentWorkspaceDatasetOperator]) + }, [isCurrentWorkspaceDatasetOperator, router]) return } From c013086e64033366d1c7baa84498c6ecfdf14965 Mon Sep 17 00:00:00 2001 From: Even Date: Fri, 19 Jul 2024 20:26:11 +0800 Subject: [PATCH 55/62] fix: next suggest question logic problem (#6451) Co-authored-by: evenyan --- api/core/llm_generator/prompts.py | 1 + api/core/memory/token_buffer_memory.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/api/core/llm_generator/prompts.py b/api/core/llm_generator/prompts.py index 170a28432bc0ef..a1737f00c61624 100644 --- a/api/core/llm_generator/prompts.py +++ b/api/core/llm_generator/prompts.py @@ -64,6 +64,7 @@ SUGGESTED_QUESTIONS_AFTER_ANSWER_INSTRUCTION_PROMPT = ( "Please help me predict the three most likely questions that human would ask, " "and keeping each question under 20 characters.\n" + "MAKE SURE your output is the SAME language as the Assistant's latest response(if the main response is written in Chinese, then the language of your output must be using Chinese.)!\n" "The output must be an array in JSON format following the specified schema:\n" "[\"question1\",\"question2\",\"question3\"]\n" ) diff --git a/api/core/memory/token_buffer_memory.py b/api/core/memory/token_buffer_memory.py index 21f1965e93adac..b33d4dd7cb342c 100644 --- a/api/core/memory/token_buffer_memory.py +++ b/api/core/memory/token_buffer_memory.py @@ -103,7 +103,7 @@ def get_history_prompt_messages(self, max_token_limit: int = 2000, if curr_message_tokens > max_token_limit: pruned_memory = [] - while curr_message_tokens > max_token_limit and prompt_messages: + while curr_message_tokens > max_token_limit and len(prompt_messages)>1: pruned_memory.append(prompt_messages.pop(0)) curr_message_tokens = self.model_instance.get_llm_num_tokens( prompt_messages From 49ef9ef225d215454459e149ca0eff59988a69c0 Mon Sep 17 00:00:00 2001 From: Matri Date: Fri, 19 Jul 2024 20:32:42 +0800 Subject: [PATCH 56/62] feat(tool): getimg.ai integration (#6260) --- .../builtin/getimgai/_assets/icon.svg | 1 + .../provider/builtin/getimgai/getimgai.py | 22 +++ .../provider/builtin/getimgai/getimgai.yaml | 29 +++ .../builtin/getimgai/getimgai_appx.py | 59 +++++++ .../builtin/getimgai/tools/text2image.py | 39 ++++ .../builtin/getimgai/tools/text2image.yaml | 167 ++++++++++++++++++ 6 files changed, 317 insertions(+) create mode 100644 api/core/tools/provider/builtin/getimgai/_assets/icon.svg create mode 100644 api/core/tools/provider/builtin/getimgai/getimgai.py create mode 100644 api/core/tools/provider/builtin/getimgai/getimgai.yaml create mode 100644 api/core/tools/provider/builtin/getimgai/getimgai_appx.py create mode 100644 api/core/tools/provider/builtin/getimgai/tools/text2image.py create mode 100644 api/core/tools/provider/builtin/getimgai/tools/text2image.yaml diff --git a/api/core/tools/provider/builtin/getimgai/_assets/icon.svg b/api/core/tools/provider/builtin/getimgai/_assets/icon.svg new file mode 100644 index 00000000000000..6b2513386da458 --- /dev/null +++ b/api/core/tools/provider/builtin/getimgai/_assets/icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/api/core/tools/provider/builtin/getimgai/getimgai.py b/api/core/tools/provider/builtin/getimgai/getimgai.py new file mode 100644 index 00000000000000..c81d5fa333cd5d --- /dev/null +++ b/api/core/tools/provider/builtin/getimgai/getimgai.py @@ -0,0 +1,22 @@ +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.getimgai.tools.text2image import Text2ImageTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + +class GetImgAIProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict) -> None: + try: + # Example validation using the text2image tool + Text2ImageTool().fork_tool_runtime( + runtime={"credentials": credentials} + ).invoke( + user_id='', + tool_parameters={ + "prompt": "A fire egg", + "response_format": "url", + "style": "photorealism", + } + ) + except Exception as e: + raise ToolProviderCredentialValidationError(str(e)) + \ No newline at end of file diff --git a/api/core/tools/provider/builtin/getimgai/getimgai.yaml b/api/core/tools/provider/builtin/getimgai/getimgai.yaml new file mode 100644 index 00000000000000..c9db0a9e22a6c4 --- /dev/null +++ b/api/core/tools/provider/builtin/getimgai/getimgai.yaml @@ -0,0 +1,29 @@ +identity: + author: Matri Qi + name: getimgai + label: + en_US: getimg.ai + zh_CN: getimg.ai + description: + en_US: GetImg API integration for image generation and scraping. + icon: icon.svg + tags: + - image +credentials_for_provider: + getimg_api_key: + type: secret-input + required: true + label: + en_US: getimg.ai API Key + placeholder: + en_US: Please input your getimg.ai API key + help: + en_US: Get your getimg.ai API key from your getimg.ai account settings. If you are using a self-hosted version, you may enter any key at your convenience. + url: https://dashboard.getimg.ai/api-keys + base_url: + type: text-input + required: false + label: + en_US: getimg.ai server's Base URL + placeholder: + en_US: https://api.getimg.ai/v1 diff --git a/api/core/tools/provider/builtin/getimgai/getimgai_appx.py b/api/core/tools/provider/builtin/getimgai/getimgai_appx.py new file mode 100644 index 00000000000000..e28c57649cac4c --- /dev/null +++ b/api/core/tools/provider/builtin/getimgai/getimgai_appx.py @@ -0,0 +1,59 @@ +import logging +import time +from collections.abc import Mapping +from typing import Any + +import requests +from requests.exceptions import HTTPError + +logger = logging.getLogger(__name__) + +class GetImgAIApp: + def __init__(self, api_key: str | None = None, base_url: str | None = None): + self.api_key = api_key + self.base_url = base_url or 'https://api.getimg.ai/v1' + if not self.api_key: + raise ValueError("API key is required") + + def _prepare_headers(self): + headers = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {self.api_key}' + } + return headers + + def _request( + self, + method: str, + url: str, + data: Mapping[str, Any] | None = None, + headers: Mapping[str, str] | None = None, + retries: int = 3, + backoff_factor: float = 0.3, + ) -> Mapping[str, Any] | None: + for i in range(retries): + try: + response = requests.request(method, url, json=data, headers=headers) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + if i < retries - 1 and isinstance(e, HTTPError) and e.response.status_code >= 500: + time.sleep(backoff_factor * (2 ** i)) + else: + raise + return None + + def text2image( + self, mode: str, **kwargs + ): + data = kwargs['params'] + if not data.get('prompt'): + raise ValueError("Prompt is required") + + endpoint = f'{self.base_url}/{mode}/text-to-image' + headers = self._prepare_headers() + logger.debug(f"Send request to {endpoint=} body={data}") + response = self._request('POST', endpoint, data, headers) + if response is None: + raise HTTPError("Failed to initiate getimg.ai after multiple retries") + return response diff --git a/api/core/tools/provider/builtin/getimgai/tools/text2image.py b/api/core/tools/provider/builtin/getimgai/tools/text2image.py new file mode 100644 index 00000000000000..dad7314479a89d --- /dev/null +++ b/api/core/tools/provider/builtin/getimgai/tools/text2image.py @@ -0,0 +1,39 @@ +import json +from typing import Any, Union + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.provider.builtin.getimgai.getimgai_appx import GetImgAIApp +from core.tools.tool.builtin_tool import BuiltinTool + + +class Text2ImageTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + app = GetImgAIApp(api_key=self.runtime.credentials['getimg_api_key'], base_url=self.runtime.credentials['base_url']) + + options = { + 'style': tool_parameters.get('style'), + 'prompt': tool_parameters.get('prompt'), + 'aspect_ratio': tool_parameters.get('aspect_ratio'), + 'output_format': tool_parameters.get('output_format', 'jpeg'), + 'response_format': tool_parameters.get('response_format', 'url'), + 'width': tool_parameters.get('width'), + 'height': tool_parameters.get('height'), + 'steps': tool_parameters.get('steps'), + 'negative_prompt': tool_parameters.get('negative_prompt'), + 'prompt_2': tool_parameters.get('prompt_2'), + } + options = {k: v for k, v in options.items() if v} + + text2image_result = app.text2image( + mode=tool_parameters.get('mode', 'essential-v2'), + params=options, + wait=True + ) + + if not isinstance(text2image_result, str): + text2image_result = json.dumps(text2image_result, ensure_ascii=False, indent=4) + + if not text2image_result: + return self.create_text_message("getimg.ai request failed.") + + return self.create_text_message(text2image_result) diff --git a/api/core/tools/provider/builtin/getimgai/tools/text2image.yaml b/api/core/tools/provider/builtin/getimgai/tools/text2image.yaml new file mode 100644 index 00000000000000..d972186f56d6a6 --- /dev/null +++ b/api/core/tools/provider/builtin/getimgai/tools/text2image.yaml @@ -0,0 +1,167 @@ +identity: + name: text2image + author: Matri Qi + label: + en_US: text2image + icon: icon.svg +description: + human: + en_US: Generate image via getimg.ai. + llm: This tool is used to generate image from prompt or image via https://getimg.ai. +parameters: + - name: prompt + type: string + required: true + label: + en_US: prompt + human_description: + en_US: The text prompt used to generate the image. The getimg.aier will generate an image based on this prompt. + llm_description: this prompt text will be used to generate image. + form: llm + - name: mode + type: select + required: false + label: + en_US: mode + human_description: + en_US: The getimg.ai mode to use. The mode determines the endpoint used to generate the image. + form: form + options: + - value: "essential-v2" + label: + en_US: essential-v2 + - value: stable-diffusion-xl + label: + en_US: stable-diffusion-xl + - value: stable-diffusion + label: + en_US: stable-diffusion + - value: latent-consistency + label: + en_US: latent-consistency + - name: style + type: select + required: false + label: + en_US: style + human_description: + en_US: The style preset to use. The style preset guides the generation towards a particular style. It's just efficient for `Essential V2` mode. + form: form + options: + - value: photorealism + label: + en_US: photorealism + - value: anime + label: + en_US: anime + - value: art + label: + en_US: art + - name: aspect_ratio + type: select + required: false + label: + en_US: "aspect ratio" + human_description: + en_US: The aspect ratio of the generated image. It's just efficient for `Essential V2` mode. + form: form + options: + - value: "1:1" + label: + en_US: "1:1" + - value: "4:5" + label: + en_US: "4:5" + - value: "5:4" + label: + en_US: "5:4" + - value: "2:3" + label: + en_US: "2:3" + - value: "3:2" + label: + en_US: "3:2" + - value: "4:7" + label: + en_US: "4:7" + - value: "7:4" + label: + en_US: "7:4" + - name: output_format + type: select + required: false + label: + en_US: "output format" + human_description: + en_US: The file format of the generated image. + form: form + options: + - value: jpeg + label: + en_US: jpeg + - value: png + label: + en_US: png + - name: response_format + type: select + required: false + label: + en_US: "response format" + human_description: + en_US: The format in which the generated images are returned. Must be one of url or b64. URLs are only valid for 1 hour after the image has been generated. + form: form + options: + - value: url + label: + en_US: url + - value: b64 + label: + en_US: b64 + - name: model + type: string + required: false + label: + en_US: model + human_description: + en_US: Model ID supported by this pipeline and family. It's just efficient for `Stable Diffusion XL`, `Stable Diffusion`, `Latent Consistency` mode. + form: form + - name: negative_prompt + type: string + required: false + label: + en_US: negative prompt + human_description: + en_US: Text input that will not guide the image generation. It's just efficient for `Stable Diffusion XL`, `Stable Diffusion`, `Latent Consistency` mode. + form: form + - name: prompt_2 + type: string + required: false + label: + en_US: prompt2 + human_description: + en_US: Prompt sent to second tokenizer and text encoder. If not defined, prompt is used in both text-encoders. It's just efficient for `Stable Diffusion XL` mode. + form: form + - name: width + type: number + required: false + label: + en_US: width + human_description: + en_US: he width of the generated image in pixels. Width needs to be multiple of 64. + form: form + - name: height + type: number + required: false + label: + en_US: height + human_description: + en_US: he height of the generated image in pixels. Height needs to be multiple of 64. + form: form + - name: steps + type: number + required: false + label: + en_US: steps + human_description: + en_US: The number of denoising steps. More steps usually can produce higher quality images, but take more time to generate. It's just efficient for `Stable Diffusion XL`, `Stable Diffusion`, `Latent Consistency` mode. + form: form From 27e08a8e2efa6c9ba89893126ab4609d83a203ec Mon Sep 17 00:00:00 2001 From: Joe <79627742+ZhouhaoJiang@users.noreply.github.com> Date: Sat, 20 Jul 2024 00:53:31 +0800 Subject: [PATCH 57/62] Fix/extra table tracing app config (#6487) --- ..._remove_extra_tracing_app_config_table .py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 api/migrations/versions/fecff1c3da27_remove_extra_tracing_app_config_table .py diff --git a/api/migrations/versions/fecff1c3da27_remove_extra_tracing_app_config_table .py b/api/migrations/versions/fecff1c3da27_remove_extra_tracing_app_config_table .py new file mode 100644 index 00000000000000..271b2490de1055 --- /dev/null +++ b/api/migrations/versions/fecff1c3da27_remove_extra_tracing_app_config_table .py @@ -0,0 +1,54 @@ +"""remove extra tracing app config table and add idx_dataset_permissions_tenant_id + +Revision ID: fecff1c3da27 +Revises: 408176b91ad3 +Create Date: 2024-07-19 12:03:21.217463 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'fecff1c3da27' +down_revision = '408176b91ad3' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('tracing_app_configs') + + with op.batch_alter_table('trace_app_config', schema=None) as batch_op: + batch_op.drop_index('tracing_app_config_app_id_idx') + + # idx_dataset_permissions_tenant_id + with op.batch_alter_table('dataset_permissions', schema=None) as batch_op: + batch_op.create_index('idx_dataset_permissions_tenant_id', ['tenant_id']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + 'tracing_app_configs', + sa.Column('id', postgresql.UUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False), + sa.Column('app_id', postgresql.UUID(), nullable=False), + sa.Column('tracing_provider', sa.String(length=255), nullable=True), + sa.Column('tracing_config', postgresql.JSON(astext_type=sa.Text()), nullable=True), + sa.Column( + 'created_at', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False + ), + sa.Column( + 'updated_at', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False + ), + sa.PrimaryKeyConstraint('id', name='tracing_app_config_pkey') + ) + + with op.batch_alter_table('trace_app_config', schema=None) as batch_op: + batch_op.create_index('tracing_app_config_app_id_idx', ['app_id']) + + with op.batch_alter_table('dataset_permissions', schema=None) as batch_op: + batch_op.drop_index('idx_dataset_permissions_tenant_id') + # ### end Alembic commands ### From 1e0e573165a40d72c5f401645536b287c2c0d3aa Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Sat, 20 Jul 2024 01:29:25 +0800 Subject: [PATCH 58/62] update clean embedding cache query logic (#6483) --- ...5b_add_embedding_cache_created_at_index.py | 32 +++++++++++++++++++ api/models/dataset.py | 3 +- api/models/model.py | 2 +- api/schedule/clean_embedding_cache_task.py | 15 ++++++--- 4 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 api/migrations/versions/6e957a32015b_add_embedding_cache_created_at_index.py diff --git a/api/migrations/versions/6e957a32015b_add_embedding_cache_created_at_index.py b/api/migrations/versions/6e957a32015b_add_embedding_cache_created_at_index.py new file mode 100644 index 00000000000000..7445f664cd75a1 --- /dev/null +++ b/api/migrations/versions/6e957a32015b_add_embedding_cache_created_at_index.py @@ -0,0 +1,32 @@ +"""add-embedding-cache-created_at_index + +Revision ID: 6e957a32015b +Revises: fecff1c3da27 +Create Date: 2024-07-19 17:21:34.414705 + +""" +from alembic import op + +import models as models + +# revision identifiers, used by Alembic. +revision = '6e957a32015b' +down_revision = 'fecff1c3da27' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('embeddings', schema=None) as batch_op: + batch_op.create_index('created_at_idx', ['created_at'], unique=False) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('embeddings', schema=None) as batch_op: + batch_op.drop_index('created_at_idx') + + # ### end Alembic commands ### diff --git a/api/models/dataset.py b/api/models/dataset.py index d0be005a154878..34dde2dcef737f 100644 --- a/api/models/dataset.py +++ b/api/models/dataset.py @@ -630,7 +630,8 @@ class Embedding(db.Model): __tablename__ = 'embeddings' __table_args__ = ( db.PrimaryKeyConstraint('id', name='embedding_pkey'), - db.UniqueConstraint('model_name', 'hash', 'provider_name', name='embedding_hash_idx') + db.UniqueConstraint('model_name', 'hash', 'provider_name', name='embedding_hash_idx'), + db.Index('created_at_idx', 'created_at') ) id = db.Column(StringUUID, primary_key=True, server_default=db.text('uuid_generate_v4()')) diff --git a/api/models/model.py b/api/models/model.py index 331bb91c29819f..396cd7ec6382c3 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -1383,7 +1383,7 @@ class TraceAppConfig(db.Model): __tablename__ = 'trace_app_config' __table_args__ = ( db.PrimaryKeyConstraint('id', name='tracing_app_config_pkey'), - db.Index('tracing_app_config_app_id_idx', 'app_id'), + db.Index('trace_app_config_app_id_idx', 'app_id'), ) id = db.Column(StringUUID, server_default=db.text('uuid_generate_v4()')) diff --git a/api/schedule/clean_embedding_cache_task.py b/api/schedule/clean_embedding_cache_task.py index f68c54600a6012..49b1be3a339b8a 100644 --- a/api/schedule/clean_embedding_cache_task.py +++ b/api/schedule/clean_embedding_cache_task.py @@ -2,6 +2,7 @@ import time import click +from sqlalchemy import text from werkzeug.exceptions import NotFound import app @@ -18,12 +19,18 @@ def clean_embedding_cache_task(): thirty_days_ago = datetime.datetime.now() - datetime.timedelta(days=clean_days) while True: try: - embeddings = db.session.query(Embedding).filter(Embedding.created_at < thirty_days_ago) \ + embedding_ids = db.session.query(Embedding.id).filter(Embedding.created_at < thirty_days_ago) \ .order_by(Embedding.created_at.desc()).limit(100).all() + embedding_ids = [embedding_id[0] for embedding_id in embedding_ids] except NotFound: break - for embedding in embeddings: - db.session.delete(embedding) - db.session.commit() + if embedding_ids: + db.session.execute(text( + "DELETE FROM embeddings WHERE id in :embedding_ids" + ), {'embedding_ids': tuple(embedding_ids)}) + + db.session.commit() + else: + break end_at = time.perf_counter() click.echo(click.style('Cleaned embedding cache from db success latency: {}'.format(end_at - start_at), fg='green')) From f73a3a58ae61b31e32984a902560a93b909aa639 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Sat, 20 Jul 2024 09:04:21 +0800 Subject: [PATCH 59/62] update delete embeddings by id (#6489) --- api/schedule/clean_embedding_cache_task.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/api/schedule/clean_embedding_cache_task.py b/api/schedule/clean_embedding_cache_task.py index 49b1be3a339b8a..ccc1062266a02f 100644 --- a/api/schedule/clean_embedding_cache_task.py +++ b/api/schedule/clean_embedding_cache_task.py @@ -25,9 +25,10 @@ def clean_embedding_cache_task(): except NotFound: break if embedding_ids: - db.session.execute(text( - "DELETE FROM embeddings WHERE id in :embedding_ids" - ), {'embedding_ids': tuple(embedding_ids)}) + for embedding_id in embedding_ids: + db.session.execute(text( + "DELETE FROM embeddings WHERE id = :embedding_id" + ), {'embedding_id': embedding_id}) db.session.commit() else: From c57b3931d524ac8cdd34bd04e1c520496f3f3f0c Mon Sep 17 00:00:00 2001 From: Shoya SHIRAKI Date: Sun, 21 Jul 2024 02:11:40 +0900 Subject: [PATCH 60/62] refactor(api): switch to dify_config in controllers/console (#6485) --- api/controllers/console/datasets/datasets.py | 7 ++++--- api/controllers/console/datasets/file.py | 11 ++++++----- api/controllers/console/explore/parameter.py | 4 ++-- api/controllers/console/init_validate.py | 5 +++-- api/controllers/console/setup.py | 7 ++++--- api/controllers/console/version.py | 11 ++++++----- api/controllers/console/workspace/account.py | 7 ++++--- api/controllers/console/workspace/members.py | 4 ++-- api/controllers/console/workspace/tool_providers.py | 5 +++-- api/controllers/console/wraps.py | 7 ++++--- 10 files changed, 38 insertions(+), 30 deletions(-) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 9166372df50517..934b6413aeeb91 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -1,10 +1,11 @@ import flask_restful -from flask import current_app, request +from flask import request from flask_login import current_user from flask_restful import Resource, marshal, marshal_with, reqparse from werkzeug.exceptions import Forbidden, NotFound import services +from configs import dify_config from controllers.console import api from controllers.console.apikey import api_key_fields, api_key_list from controllers.console.app.error import ProviderNotInitializeError @@ -530,7 +531,7 @@ class DatasetApiBaseUrlApi(Resource): @account_initialization_required def get(self): return { - 'api_base_url': (current_app.config['SERVICE_API_URL'] if current_app.config['SERVICE_API_URL'] + 'api_base_url': (dify_config.SERVICE_API_URL if dify_config.SERVICE_API_URL else request.host_url.rstrip('/')) + '/v1' } @@ -540,7 +541,7 @@ class DatasetRetrievalSettingApi(Resource): @login_required @account_initialization_required def get(self): - vector_type = current_app.config['VECTOR_STORE'] + vector_type = dify_config.VECTOR_STORE match vector_type: case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE: return { diff --git a/api/controllers/console/datasets/file.py b/api/controllers/console/datasets/file.py index c13bd45abb02fe..3b2083bcc3351c 100644 --- a/api/controllers/console/datasets/file.py +++ b/api/controllers/console/datasets/file.py @@ -1,8 +1,9 @@ -from flask import current_app, request +from flask import request from flask_login import current_user from flask_restful import Resource, marshal_with import services +from configs import dify_config from controllers.console import api from controllers.console.datasets.error import ( FileTooLargeError, @@ -26,9 +27,9 @@ class FileApi(Resource): @account_initialization_required @marshal_with(upload_config_fields) def get(self): - file_size_limit = current_app.config.get("UPLOAD_FILE_SIZE_LIMIT") - batch_count_limit = current_app.config.get("UPLOAD_FILE_BATCH_LIMIT") - image_file_size_limit = current_app.config.get("UPLOAD_IMAGE_FILE_SIZE_LIMIT") + file_size_limit = dify_config.UPLOAD_FILE_SIZE_LIMIT + batch_count_limit = dify_config.UPLOAD_FILE_BATCH_LIMIT + image_file_size_limit = dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT return { 'file_size_limit': file_size_limit, 'batch_count_limit': batch_count_limit, @@ -76,7 +77,7 @@ class FileSupportTypeApi(Resource): @login_required @account_initialization_required def get(self): - etl_type = current_app.config['ETL_TYPE'] + etl_type = dify_config.ETL_TYPE allowed_extensions = UNSTRUCTURED_ALLOWED_EXTENSIONS if etl_type == 'Unstructured' else ALLOWED_EXTENSIONS return {'allowed_extensions': allowed_extensions} diff --git a/api/controllers/console/explore/parameter.py b/api/controllers/console/explore/parameter.py index 45255edb3a6887..0a168d6306c83e 100644 --- a/api/controllers/console/explore/parameter.py +++ b/api/controllers/console/explore/parameter.py @@ -1,7 +1,7 @@ -from flask import current_app from flask_restful import fields, marshal_with +from configs import dify_config from controllers.console import api from controllers.console.app.error import AppUnavailableError from controllers.console.explore.wraps import InstalledAppResource @@ -78,7 +78,7 @@ def get(self, installed_app: InstalledApp): "transfer_methods": ["remote_url", "local_file"] }}), 'system_parameters': { - 'image_file_size_limit': current_app.config.get('UPLOAD_IMAGE_FILE_SIZE_LIMIT') + 'image_file_size_limit': dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT } } diff --git a/api/controllers/console/init_validate.py b/api/controllers/console/init_validate.py index b319f706b4dc38..6feb1003a975f6 100644 --- a/api/controllers/console/init_validate.py +++ b/api/controllers/console/init_validate.py @@ -1,8 +1,9 @@ import os -from flask import current_app, session +from flask import session from flask_restful import Resource, reqparse +from configs import dify_config from libs.helper import str_len from models.model import DifySetup from services.account_service import TenantService @@ -40,7 +41,7 @@ def post(self): return {'result': 'success'}, 201 def get_init_validate_status(): - if current_app.config['EDITION'] == 'SELF_HOSTED': + if dify_config.EDITION == 'SELF_HOSTED': if os.environ.get('INIT_PASSWORD'): return session.get('is_init_validated') or DifySetup.query.first() diff --git a/api/controllers/console/setup.py b/api/controllers/console/setup.py index def50212a18b82..ef7cc6bc03ce3a 100644 --- a/api/controllers/console/setup.py +++ b/api/controllers/console/setup.py @@ -1,8 +1,9 @@ from functools import wraps -from flask import current_app, request +from flask import request from flask_restful import Resource, reqparse +from configs import dify_config from libs.helper import email, get_remote_ip, str_len from libs.password import valid_password from models.model import DifySetup @@ -17,7 +18,7 @@ class SetupApi(Resource): def get(self): - if current_app.config['EDITION'] == 'SELF_HOSTED': + if dify_config.EDITION == 'SELF_HOSTED': setup_status = get_setup_status() if setup_status: return { @@ -77,7 +78,7 @@ def decorated(*args, **kwargs): def get_setup_status(): - if current_app.config['EDITION'] == 'SELF_HOSTED': + if dify_config.EDITION == 'SELF_HOSTED': return DifySetup.query.first() else: return True diff --git a/api/controllers/console/version.py b/api/controllers/console/version.py index faf36c4f40020f..1fcf4bdc00e5b1 100644 --- a/api/controllers/console/version.py +++ b/api/controllers/console/version.py @@ -3,9 +3,10 @@ import logging import requests -from flask import current_app from flask_restful import Resource, reqparse +from configs import dify_config + from . import api @@ -15,16 +16,16 @@ def get(self): parser = reqparse.RequestParser() parser.add_argument('current_version', type=str, required=True, location='args') args = parser.parse_args() - check_update_url = current_app.config['CHECK_UPDATE_URL'] + check_update_url = dify_config.CHECK_UPDATE_URL result = { - 'version': current_app.config['CURRENT_VERSION'], + 'version': dify_config.CURRENT_VERSION, 'release_date': '', 'release_notes': '', 'can_auto_update': False, 'features': { - 'can_replace_logo': current_app.config['CAN_REPLACE_LOGO'], - 'model_load_balancing_enabled': current_app.config['MODEL_LB_ENABLED'] + 'can_replace_logo': dify_config.CAN_REPLACE_LOGO, + 'model_load_balancing_enabled': dify_config.MODEL_LB_ENABLED } } diff --git a/api/controllers/console/workspace/account.py b/api/controllers/console/workspace/account.py index 0b5c84c2a3d24d..1056d5eb62faf2 100644 --- a/api/controllers/console/workspace/account.py +++ b/api/controllers/console/workspace/account.py @@ -1,10 +1,11 @@ import datetime import pytz -from flask import current_app, request +from flask import request from flask_login import current_user from flask_restful import Resource, fields, marshal_with, reqparse +from configs import dify_config from constants.languages import supported_language from controllers.console import api from controllers.console.setup import setup_required @@ -36,7 +37,7 @@ def post(self): parser = reqparse.RequestParser() - if current_app.config['EDITION'] == 'CLOUD': + if dify_config.EDITION == 'CLOUD': parser.add_argument('invitation_code', type=str, location='json') parser.add_argument( @@ -45,7 +46,7 @@ def post(self): required=True, location='json') args = parser.parse_args() - if current_app.config['EDITION'] == 'CLOUD': + if dify_config.EDITION == 'CLOUD': if not args['invitation_code']: raise ValueError('invitation_code is required') diff --git a/api/controllers/console/workspace/members.py b/api/controllers/console/workspace/members.py index 0e756778ab0e85..34e9da384106a7 100644 --- a/api/controllers/console/workspace/members.py +++ b/api/controllers/console/workspace/members.py @@ -1,8 +1,8 @@ -from flask import current_app from flask_login import current_user from flask_restful import Resource, abort, marshal_with, reqparse import services +from configs import dify_config from controllers.console import api from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check @@ -48,7 +48,7 @@ def post(self): inviter = current_user invitation_results = [] - console_web_url = current_app.config.get("CONSOLE_WEB_URL") + console_web_url = dify_config.CONSOLE_WEB_URL for invitee_email in invitee_emails: try: token = RegisterService.invite_new_member(inviter.current_tenant, invitee_email, interface_language, role=invitee_role, inviter=inviter) diff --git a/api/controllers/console/workspace/tool_providers.py b/api/controllers/console/workspace/tool_providers.py index 6e3f78d4e2e55b..bafeabb08ae2c7 100644 --- a/api/controllers/console/workspace/tool_providers.py +++ b/api/controllers/console/workspace/tool_providers.py @@ -1,10 +1,11 @@ import io -from flask import current_app, send_file +from flask import send_file from flask_login import current_user from flask_restful import Resource, reqparse from werkzeug.exceptions import Forbidden +from configs import dify_config from controllers.console import api from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required @@ -104,7 +105,7 @@ class ToolBuiltinProviderIconApi(Resource): @setup_required def get(self, provider): icon_bytes, mimetype = BuiltinToolManageService.get_builtin_tool_provider_icon(provider) - icon_cache_max_age = current_app.config.get('TOOL_ICON_CACHE_MAX_AGE') + icon_cache_max_age = dify_config.TOOL_ICON_CACHE_MAX_AGE return send_file(io.BytesIO(icon_bytes), mimetype=mimetype, max_age=icon_cache_max_age) class ToolApiProviderAddApi(Resource): diff --git a/api/controllers/console/wraps.py b/api/controllers/console/wraps.py index 7c8ad110784ee0..3baf69acfd2d5f 100644 --- a/api/controllers/console/wraps.py +++ b/api/controllers/console/wraps.py @@ -1,9 +1,10 @@ import json from functools import wraps -from flask import abort, current_app, request +from flask import abort, request from flask_login import current_user +from configs import dify_config from controllers.console.workspace.error import AccountNotInitializedError from services.feature_service import FeatureService from services.operation_service import OperationService @@ -26,7 +27,7 @@ def decorated(*args, **kwargs): def only_edition_cloud(view): @wraps(view) def decorated(*args, **kwargs): - if current_app.config['EDITION'] != 'CLOUD': + if dify_config.EDITION != 'CLOUD': abort(404) return view(*args, **kwargs) @@ -37,7 +38,7 @@ def decorated(*args, **kwargs): def only_edition_self_hosted(view): @wraps(view) def decorated(*args, **kwargs): - if current_app.config['EDITION'] != 'SELF_HOSTED': + if dify_config.EDITION != 'SELF_HOSTED': abort(404) return view(*args, **kwargs) From f38034e4557e0c621b62e081c3d6e11361b3ef58 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Sun, 21 Jul 2024 15:09:09 +0800 Subject: [PATCH 61/62] clean vector collection redis cache (#6494) --- .../vdb/analyticdb/analyticdb_vector.py | 21 +++++++++++-------- .../datasource/vdb/tencent/tencent_vector.py | 2 -- api/core/rag/datasource/vdb/vector_base.py | 4 ++++ api/core/rag/datasource/vdb/vector_factory.py | 5 +++++ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py b/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py index 123b93fcd5aa2d..442d71293f632d 100644 --- a/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py +++ b/api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py @@ -293,15 +293,18 @@ def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: return documents def delete(self) -> None: - from alibabacloud_gpdb20160503 import models as gpdb_20160503_models - request = gpdb_20160503_models.DeleteCollectionRequest( - collection=self._collection_name, - dbinstance_id=self.config.instance_id, - namespace=self.config.namespace, - namespace_password=self.config.namespace_password, - region_id=self.config.region_id, - ) - self._client.delete_collection(request) + try: + from alibabacloud_gpdb20160503 import models as gpdb_20160503_models + request = gpdb_20160503_models.DeleteCollectionRequest( + collection=self._collection_name, + dbinstance_id=self.config.instance_id, + namespace=self.config.namespace, + namespace_password=self.config.namespace_password, + region_id=self.config.region_id, + ) + self._client.delete_collection(request) + except Exception as e: + raise e class AnalyticdbVectorFactory(AbstractVectorFactory): def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings): diff --git a/api/core/rag/datasource/vdb/tencent/tencent_vector.py b/api/core/rag/datasource/vdb/tencent/tencent_vector.py index cdcc22aec99367..3325a1028ece52 100644 --- a/api/core/rag/datasource/vdb/tencent/tencent_vector.py +++ b/api/core/rag/datasource/vdb/tencent/tencent_vector.py @@ -198,8 +198,6 @@ def delete(self) -> None: self._db.drop_collection(name=self._collection_name) - - class TencentVectorFactory(AbstractVectorFactory): def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings) -> TencentVector: diff --git a/api/core/rag/datasource/vdb/vector_base.py b/api/core/rag/datasource/vdb/vector_base.py index dbd8b6284bfc78..17768ab042dd93 100644 --- a/api/core/rag/datasource/vdb/vector_base.py +++ b/api/core/rag/datasource/vdb/vector_base.py @@ -67,3 +67,7 @@ def _filter_duplicate_texts(self, texts: list[Document]) -> list[Document]: def _get_uuids(self, texts: list[Document]) -> list[str]: return [text.metadata['doc_id'] for text in texts] + + @property + def collection_name(self): + return self._collection_name diff --git a/api/core/rag/datasource/vdb/vector_factory.py b/api/core/rag/datasource/vdb/vector_factory.py index 949a4b58471a07..256abd28afbe53 100644 --- a/api/core/rag/datasource/vdb/vector_factory.py +++ b/api/core/rag/datasource/vdb/vector_factory.py @@ -9,6 +9,7 @@ from core.rag.datasource.vdb.vector_base import BaseVector from core.rag.datasource.vdb.vector_type import VectorType from core.rag.models.document import Document +from extensions.ext_redis import redis_client from models.dataset import Dataset @@ -134,6 +135,10 @@ def search_by_full_text( def delete(self) -> None: self._vector_processor.delete() + # delete collection redis cache + if self._vector_processor.collection_name: + collection_exist_cache_key = 'vector_indexing_{}'.format(self._vector_processor.collection_name) + redis_client.delete(collection_exist_cache_key) def _get_embeddings(self) -> Embeddings: model_manager = ModelManager() From dfb6f4fec6f6474a0ba1e65f68f7718fd7ed8126 Mon Sep 17 00:00:00 2001 From: sino Date: Mon, 22 Jul 2024 07:43:18 +0800 Subject: [PATCH 62/62] fix: extract tool calls correctly while arguments is empty (#6503) --- api/core/agent/fc_agent_runner.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/api/core/agent/fc_agent_runner.py b/api/core/agent/fc_agent_runner.py index bec76e7a246d12..7019b5e39fdfb9 100644 --- a/api/core/agent/fc_agent_runner.py +++ b/api/core/agent/fc_agent_runner.py @@ -342,10 +342,14 @@ def extract_tool_calls(self, llm_result_chunk: LLMResultChunk) -> Union[None, li """ tool_calls = [] for prompt_message in llm_result_chunk.delta.message.tool_calls: + args = {} + if prompt_message.function.arguments != '': + args = json.loads(prompt_message.function.arguments) + tool_calls.append(( prompt_message.id, prompt_message.function.name, - json.loads(prompt_message.function.arguments), + args, )) return tool_calls @@ -359,10 +363,14 @@ def extract_blocking_tool_calls(self, llm_result: LLMResult) -> Union[None, list """ tool_calls = [] for prompt_message in llm_result.message.tool_calls: + args = {} + if prompt_message.function.arguments != '': + args = json.loads(prompt_message.function.arguments) + tool_calls.append(( prompt_message.id, prompt_message.function.name, - json.loads(prompt_message.function.arguments), + args, )) return tool_calls