diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index cff98db7e42b9d..4af4daadeba3a8 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -107,7 +107,9 @@ jobs: api/poetry.lock - name: Poetry check - run: poetry check -C api + run: | + poetry check -C api + poetry show -C api - name: Install dependencies run: poetry install -C api --with dev diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 048f4cd942632d..89d301c4fd8a6e 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -17,7 +17,7 @@ env: jobs: build-and-push: runs-on: ubuntu-latest - if: github.event.pull_request.draft == false + if: github.repository == 'langgenius/dify' strategy: matrix: include: diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 160641fdf0bc0d..cb5400b918b98e 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -100,6 +100,7 @@ jobs: **.yaml **.yml Dockerfile + dev/** - name: Super-linter uses: super-linter/super-linter/slim@v6 diff --git a/CONTRIBUTING_JA.md b/CONTRIBUTING_JA.md index c9329d610230f4..e8f5456a3c44b7 100644 --- a/CONTRIBUTING_JA.md +++ b/CONTRIBUTING_JA.md @@ -4,7 +4,7 @@ Dify にコントリビュートしたいとお考えなのですね。それは 私たちは現状を鑑み、機敏かつ迅速に開発をする必要がありますが、同時にあなたのようなコントリビューターの方々に、可能な限りスムーズな貢献体験をしていただきたいと思っています。そのためにこのコントリビュートガイドを作成しました。 コードベースやコントリビュータの方々と私たちがどのように仕事をしているのかに慣れていただき、楽しいパートにすぐに飛び込めるようにすることが目的です。 -このガイドは Dify そのものと同様に、継続的に改善されています。実際のプロジェクトに遅れをとることがあるかもしれませんが、ご理解をお願いします。 +このガイドは Dify そのものと同様に、継続的に改善されています。実際のプロジェクトに遅れをとることがあるかもしれませんが、ご理解のほどよろしくお願いいたします。 ライセンスに関しては、私たちの短い[ライセンスおよびコントリビューター規約](./LICENSE)をお読みください。また、コミュニティは[行動規範](https://github.com/langgenius/.github/blob/main/CODE_OF_CONDUCT.md)を遵守しています。 @@ -14,7 +14,7 @@ Dify にコントリビュートしたいとお考えなのですね。それは ### 機能リクエスト -* 新しい機能要望を出す場合は、提案する機能が何を実現するものなのかを説明し、可能な限り多くの文脈を含めてください。[@perzeusss](https://github.com/perzeuss)は、あなたの要望を書き出すのに役立つ [Feature Request Copilot](https://udify.app/chat/MK2kVSnw1gakVwMX) を作ってくれました。気軽に試してみてください。 +* 新しい機能要望を出す場合は、提案する機能が何を実現するものなのかを説明し、可能な限り多くのコンテキストを含めてください。[@perzeusss](https://github.com/perzeuss)は、あなたの要望を書き出すのに役立つ [Feature Request Copilot](https://udify.app/chat/MK2kVSnw1gakVwMX) を作ってくれました。気軽に試してみてください。 * 既存の課題から 1 つ選びたい場合は、その下にコメントを書いてください。 @@ -54,7 +54,7 @@ Dify にコントリビュートしたいとお考えなのですね。それは ## インストール -Dify を開発用にセットアップする手順は以下の通りです。 +以下の手順で 、Difyのセットアップをしてください。 ### 1. このリポジトリをフォークする @@ -120,7 +120,7 @@ Dify のバックエンドは[Flask](https://flask.palletsprojects.com/en/3.0.x/ ### フロントエンド -このウェブサイトは、Typescript の[Next.js](https://nextjs.org/)ボイラープレートでブートストラップされており、スタイリングには[Tailwind CSS](https://tailwindcss.com/)を使用しています。国際化には[React-i18next](https://react.i18next.com/)を使用しています。 +このウェブサイトは、Typescriptベースの[Next.js](https://nextjs.org/)テンプレートを使ってブートストラップされ、[Tailwind CSS](https://tailwindcss.com/)を使ってスタイリングされています。国際化には[React-i18next](https://react.i18next.com/)を使用しています。 ``` [web/] diff --git a/README.md b/README.md index d81e5c76720078..f5e06ce0fb3536 100644 --- a/README.md +++ b/README.md @@ -185,10 +185,11 @@ After running, you can access the Dify dashboard in your browser at [http://loca If you need to customize the configuration, please refer to the comments in our [docker-compose.yml](docker/docker-compose.yaml) file and manually set the environment configuration. After making the changes, please run `docker-compose up -d` again. You can see the full list of environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments). -If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) which allow Dify to be deployed on Kubernetes. +If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) and YAML files which allow Dify to be deployed on Kubernetes. - [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify) - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm) +- [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes) ## Contributing diff --git a/README_AR.md b/README_AR.md index 9716346fd2b313..1b0127b9e6be3d 100644 --- a/README_AR.md +++ b/README_AR.md @@ -167,10 +167,11 @@ docker compose up -d إذا كنت بحاجة إلى تخصيص التكوين، يرجى الرجوع إلى التعليقات في ملف [docker-compose.yml](docker/docker-compose.yaml) لدينا وتعيين التكوينات البيئية يدويًا. بعد إجراء التغييرات، يرجى تشغيل `docker-compose up -d` مرة أخرى. يمكنك رؤية قائمة كاملة بالمتغيرات البيئية [هنا](https://docs.dify.ai/getting-started/install-self-hosted/environments). -إذا كنت ترغب في تكوين إعداد متوفر بشكل عالي، فهناك [رسوم بيانية Helm](https://helm.sh/) المساهمة من المجتمع تسمح بنشر Dify على Kubernetes. +يوجد مجتمع خاص بـ [Helm Charts](https://helm.sh/) وملفات YAML التي تسمح بتنفيذ Dify على Kubernetes للنظام من الإيجابيات العلوية. - [رسم بياني Helm من قبل @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify) - [رسم بياني Helm من قبل @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm) +- [ملف YAML من قبل @Winson-030](https://github.com/Winson-030/dify-kubernetes) ## المساهمة diff --git a/README_CN.md b/README_CN.md index c6e81b532ad81e..141dc152ec14df 100644 --- a/README_CN.md +++ b/README_CN.md @@ -186,10 +186,11 @@ docker compose up -d #### 使用 Helm Chart 部署 -使用 [Helm Chart](https://helm.sh/) 版本，可以在 Kubernetes 上部署 Dify。 +使用 [Helm Chart](https://helm.sh/) 版本或者 YAML 文件，可以在 Kubernetes 上部署 Dify。 - [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify) - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm) +- [YAML 文件 by @Winson-030](https://github.com/Winson-030/dify-kubernetes) ### 配置 diff --git a/README_ES.md b/README_ES.md index efc1bdfd419cb3..a26719a2f283cd 100644 --- a/README_ES.md +++ b/README_ES.md @@ -192,10 +192,11 @@ Si necesitas personalizar la configuración, consulta los comentarios en nuestro . Después de realizar los cambios, ejecuta `docker-compose up -d` nuevamente. Puedes ver la lista completa de variables de entorno [aquí](https://docs.dify.ai/getting-started/install-self-hosted/environments). -Si deseas configurar una instalación altamente disponible, hay [Gráficos Helm](https://helm.sh/) contribuidos por la comunidad que permiten implementar Dify en Kubernetes. +Si desea configurar una configuración de alta disponibilidad, la comunidad proporciona [Gráficos Helm](https://helm.sh/) y archivos YAML, a través de los cuales puede desplegar Dify en Kubernetes. - [Gráfico Helm por @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify) - [Gráfico Helm por @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm) +- [Ficheros YAML por @Winson-030](https://github.com/Winson-030/dify-kubernetes) ## Contribuir diff --git a/README_FR.md b/README_FR.md index 4f12f3788e2da0..b754ffaef7e66f 100644 --- a/README_FR.md +++ b/README_FR.md @@ -192,10 +192,11 @@ Si vous devez personnaliser la configuration, veuillez vous référer aux commentaires dans notre fichier [docker-compose.yml](docker/docker-compose.yaml) et définir manuellement la configuration de l'environnement. Après avoir apporté les modifications, veuillez exécuter à nouveau `docker-compose up -d`. Vous pouvez voir la liste complète des variables d'environnement [ici](https://docs.dify.ai/getting-started/install-self-hosted/environments). -Si vous souhaitez configurer une installation hautement disponible, il existe des [Helm Charts](https://helm.sh/) contribués par la communauté qui permettent de déployer Dify sur Kubernetes. +Si vous souhaitez configurer une configuration haute disponibilité, la communauté fournit des [Helm Charts](https://helm.sh/) et des fichiers YAML, à travers lesquels vous pouvez déployer Dify sur Kubernetes. - [Helm Chart par @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify) - [Helm Chart par @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm) +- [Fichier YAML par @Winson-030](https://github.com/Winson-030/dify-kubernetes) ## Contribuer diff --git a/README_JA.md b/README_JA.md index 11de404c7dc2a8..2d78992eb32383 100644 --- a/README_JA.md +++ b/README_JA.md @@ -2,9 +2,9 @@

Dify Cloud · - セルフホスト · + セルフホスティング · ドキュメント · - デモのスケジュール + デモの予約

@@ -44,37 +44,37 @@ langgenius%2Fdify | Trendshift

-DifyはオープンソースのLLMアプリケーション開発プラットフォームです。直感的なインターフェースには、AIワークフロー、RAGパイプライン、エージェント機能、モデル管理、観測機能などが組み合わさっており、プロトタイプから本番までの移行を迅速に行うことができます。以下は、主要機能のリストです： +DifyはオープンソースのLLMアプリケーション開発プラットフォームです。直感的なインターフェイスには、AIワークフロー、RAGパイプライン、エージェント機能、モデル管理、観測機能などが組み合わさっており、プロトタイプから生産まで迅速に進めることができます。以下の機能が含まれます：

**1. ワークフロー**: - ビジュアルキャンバス上で強力なAIワークフローを構築してテストし、以下の機能を活用してプロトタイプを超えることができます。 + 強力なAIワークフローをビジュアルキャンバス上で構築し、テストできます。すべての機能、および以下の機能を使用できます。 https://github.com/langgenius/dify/assets/13230914/356df23e-1604-483d-80a6-9517ece318aa -**2. 包括的なモデルサポート**: - 数百のプロプライエタリ/オープンソースのLLMと、数十の推論プロバイダーおよびセルフホスティングソリューションとのシームレスな統合を提供します。GPT、Mistral、Llama3、およびOpenAI API互換のモデルをカバーします。サポートされているモデルプロバイダーの完全なリストは[こちら](https://docs.dify.ai/getting-started/readme/model-providers)をご覧ください。 +**2. 総合的なモデルサポート**: + 数百ものプロプライエタリ/オープンソースのLLMと、数十もの推論プロバイダーおよびセルフホスティングソリューションとのシームレスな統合を提供します。GPT、Mistral、Llama3、OpenAI APIと互換性のあるすべてのモデルを統合されています。サポートされているモデルプロバイダーの完全なリストは[こちら](https://docs.dify.ai/getting-started/readme/model-providers)をご覧ください。 ![providers-v5](https://github.com/langgenius/dify/assets/13230914/5a17bdbe-097a-4100-8363-40255b70f6e3) **3. プロンプトIDE**: - チャットベースのアプリにテキスト読み上げなどの追加機能を追加するプロンプトを作成し、モデルのパフォーマンスを比較する直感的なインターフェース。 + プロンプトの作成、モデルパフォーマンスの比較が行え、チャットベースのアプリに音声合成などの機能も追加できます。 **4. RAGパイプライン**: - 文書の取り込みから取得までをカバーする幅広いRAG機能で、PDF、PPTなどの一般的なドキュメント形式からのテキスト抽出に対するアウトオブボックスのサポートを提供します。 + ドキュメントの取り込みから検索までをカバーする広範なRAG機能ができます。ほかにもPDF、PPT、その他の一般的なドキュメントフォーマットからのテキスト抽出のサーポイントも提供します。 **5. エージェント機能**: - LLM関数呼び出しまたはReActに基づいてエージェントを定義し、エージェント向けの事前構築済みまたはカスタムのツールを追加できます。Difyには、Google検索、DELL·E、Stable Diffusion、WolframAlphaなどのAIエージェント用の50以上の組み込みツールが用意されています。 + LLM Function CallingやReActに基づくエージェントの定義が可能で、AIエージェント用のプリビルトまたはカスタムツールを追加できます。Difyには、Google検索、DELL·E、Stable Diffusion、WolframAlphaなどのAIエージェント用の50以上の組み込みツールが提供します。 **6. LLMOps**: - アプリケーションログとパフォーマンスを時間の経過とともにモニタリングおよび分析します。本番データと注釈に基づいて、プロンプト、データセット、およびモデルを継続的に改善できます。 + アプリケーションのログやパフォーマンスを監視と分析し、生産のデータと注釈に基づいて、プロンプト、データセット、モデルを継続的に改善できます。 **7. Backend-as-a-Service**: - Difyのすべての提供には、それに対応するAPIが付属しており、独自のビジネスロジックにDifyをシームレスに統合できます。 + すべての機能はAPIを提供されており、Difyを自分のビジネスロジックに簡単に統合できます。 ## 機能比較 @@ -95,9 +95,9 @@ DifyはオープンソースのLLMアプリケーション開発プラットフ サポートされているLLM - バリエーション豊富 - バリエーション豊富 - バリエーション豊富 + バラエティ豊か + バラエティ豊か + バラエティ豊か OpenAIのみ @@ -147,15 +147,15 @@ DifyはオープンソースのLLMアプリケーション開発プラットフ ## Difyの使用方法 - **クラウド
** -[こちら](https://dify.ai)のDify Cloudサービスを利用して、セットアップ不要で試すことができます。サンドボックスプランには、200回の無料のGPT-4呼び出しが含まれています。 +[こちら](https://dify.ai)のDify Cloudサービスを利用して、セットアップ不要で試すことができます。サンドボックスプランには、200回のGPT-4呼び出しが無料で含まれています。 - **Dify Community Editionのセルフホスティング
** -この[スターターガイド](#quick-start)を使用して、ローカル環境でDifyを簡単に実行できます。 -さらなる参考資料や詳細な手順については、[ドキュメント](https://docs.dify.ai)をご覧ください。 +この[スタートガイド](#quick-start)を使用して、ローカル環境でDifyを簡単に実行できます。 +詳しくは[ドキュメント](https://docs.dify.ai)をご覧ください。 -- **エンタープライズ/組織向けのDify
** -追加のエンタープライズ向け機能を提供しています。[こちらからミーティングを予約](https://cal.com/guchenhe/30min)したり、[メールを送信](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry)してエンタープライズのニーズについて相談してください。
- > AWSを使用しているスタートアップや中小企業の場合は、[AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6)のDify Premiumをチェックして、ワンクリックで独自のAWS VPCにデプロイできます。カスタムロゴとブランディングでアプリを作成するオプションを備えた手頃な価格のAMIオファリングです。 +- **企業/組織向けのDify
** +企業中心の機能を提供しています。[こちらからミーティングを予約](https://cal.com/guchenhe/30min)したり、[メールを送信](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry)して企業のニーズについて相談してください。
+ > AWSを使用しているスタートアップ企業や中小企業の場合は、[AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6)のDify Premiumをチェックして、ワンクリックで自分のAWS VPCにデプロイできます。さらに、手頃な価格のAMIオファリングどして、ロゴやブランディングをカスタマイズしてアプリケーションを作成するオプションがあります。 ## 最新の情報を入手 @@ -189,10 +189,11 @@ docker compose up -d 環境設定をカスタマイズする場合は、[docker-compose.yml](docker/docker-compose.yaml)ファイル内のコメントを参照して、環境設定を手動で設定してください。変更を加えた後は、再び `docker-compose up -d` を実行してください。環境変数の完全なリストは[こちら](https://docs.dify.ai/getting-started/install-self-hosted/environments)をご覧ください。 -高可用性のセットアップを構成する場合は、コミュニティによって提供されている[Helm Charts](https://helm.sh/)があり、これによりKubernetes上にDifyを展開できます。 +高可用性設定を設定する必要がある場合、コミュニティは[Helm Charts](https://helm.sh/)とYAMLファイルにより、DifyをKubernetesにデプロイすることができます。 - [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify) - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm) +- [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes) ## 貢献 @@ -212,7 +213,7 @@ docker compose up -d ## コミュニティ & お問い合わせ * [Github Discussion](https://github.com/langgenius/dify/discussions). 主に: フィードバックの共有や質問。 -* [GitHub Issues](https://github.com/langgenius/dify/issues). 主に: Dify.AIの使用中に遭遇したバグや機能提案。 +* [GitHub Issues](https://github.com/langgenius/dify/issues). 主に: Dify.AIを使用する際に発生するエラーや問題については、[貢献ガイド](CONTRIBUTING_JA.md)を参照してください * [Email](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). 主に: Dify.AIの使用に関する質問。 * [Discord](https://discord.gg/FngNHpbcY7). 主に: アプリケーションの共有やコミュニティとの交流。 * [Twitter](https://twitter.com/dify_ai). 主に: アプリケーションの共有やコミュニティとの交流。 diff --git a/README_KL.md b/README_KL.md index b1eb5073f69653..033c73fb99e380 100644 --- a/README_KL.md +++ b/README_KL.md @@ -190,11 +190,11 @@ After running, you can access the Dify dashboard in your browser at [http://loca If you need to customize the configuration, please refer to the comments in our [docker-compose.yml](docker/docker-compose.yaml) file and manually set the environment configuration. After making the changes, please run `docker-compose up -d` again. You can see the full list of environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments). -If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) which allow Dify to be deployed on Kubernetes. +If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) and YAML files which allow Dify to be deployed on Kubernetes. - [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify) - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm) - +- [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes) ## Contributing diff --git a/README_KR.md b/README_KR.md index 9c809fa017a8e3..4c48faa7fb323a 100644 --- a/README_KR.md +++ b/README_KR.md @@ -184,11 +184,11 @@ docker compose up -d 구성 커스터마이징이 필요한 경우, [docker-compose.yml](docker/docker-compose.yaml) 파일의 코멘트를 참조하여 환경 구성을 수동으로 설정하십시오. 변경 후 `docker-compose up -d` 를 다시 실행하십시오. 환경 변수의 전체 목록은 [여기](https://docs.dify.ai/getting-started/install-self-hosted/environments)에서 확인할 수 있습니다. -고가용성 설정을 구성하려면 Dify를 Kubernetes에 배포할 수 있는 커뮤니티 제공 [Helm Charts](https://helm.sh/)가 있습니다. +Dify를 Kubernetes에 배포하고 프리미엄 스케일링 설정을 구성했다는 커뮤니티가 제공하는 [Helm Charts](https://helm.sh/)와 YAML 파일이 존재합니다. - [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify) - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm) - +- [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes) ## 기여 diff --git a/api/.env.example b/api/.env.example index 5486cdd5f448b7..bf43af47c56790 100644 --- a/api/.env.example +++ b/api/.env.example @@ -42,6 +42,7 @@ DB_DATABASE=dify # storage type: local, s3, azure-blob STORAGE_TYPE=local STORAGE_LOCAL_PATH=storage +S3_USE_AWS_MANAGED_IAM=false S3_ENDPOINT=https://your-bucket-name.storage.s3.clooudflare.com S3_BUCKET_NAME=your-bucket-name S3_ACCESS_KEY=your-access-key @@ -98,6 +99,15 @@ RELYT_USER=postgres RELYT_PASSWORD=postgres RELYT_DATABASE=postgres +# Tencent configuration +TENCENT_VECTOR_DB_URL=http://127.0.0.1 +TENCENT_VECTOR_DB_API_KEY=dify +TENCENT_VECTOR_DB_TIMEOUT=30 +TENCENT_VECTOR_DB_USERNAME=dify +TENCENT_VECTOR_DB_DATABASE=dify +TENCENT_VECTOR_DB_SHARD=1 +TENCENT_VECTOR_DB_REPLICAS=2 + # PGVECTO_RS configuration PGVECTO_RS_HOST=localhost PGVECTO_RS_PORT=5431 diff --git a/api/README.md b/api/README.md index 4ae9875228c8c9..e32df089afa62a 100644 --- a/api/README.md +++ b/api/README.md @@ -18,58 +18,121 @@ sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env ``` 4. Create environment. - - Anaconda - If you use Anaconda, create a new environment and activate it - ```bash - conda create --name dify python=3.10 - conda activate dify - ``` - - Poetry - If you use Poetry, you don't need to manually create the environment. You can execute `poetry shell` to activate the environment. -5. Install dependencies - - Anaconda - ```bash - pip install -r requirements.txt - ``` - - Poetry + + Dify API service uses [Poetry](https://python-poetry.org/docs/) to manage dependencies. You can execute `poetry shell` to activate the environment. + + > Using pip can be found [below](#usage-with-pip). + +6. Install dependencies + ```bash poetry install ``` + In case of contributors missing to update dependencies for `pyproject.toml`, you can perform the following shell instead. - ```base + + ```bash poetry shell # activate current environment poetry add $(cat requirements.txt) # install dependencies of production and update pyproject.toml poetry add $(cat requirements-dev.txt) --group dev # install dependencies of development and update pyproject.toml ``` -6. Run migrate + +7. Run migrate Before the first launch, migrate the database to the latest version. + + ```bash + poetry run python -m flask db upgrade + ``` +8. Start backend + ```bash - flask db upgrade + poetry run python -m flask run --host 0.0.0.0 --port=5001 --debug ``` - ⚠️ If you encounter problems with jieba, for example +9. Start Dify [web](../web) service. +10. Setup your application by visiting `http://localhost:3000`... +11. If you need to debug local async processing, please start the worker service. + + ```bash + poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail + ``` + + The started celery app handles the async tasks, e.g. dataset importing and documents indexing. + + +## Testing +1. Install dependencies for both the backend and the test environment + + ```bash + poetry install --with dev + ``` + +2. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml` + + ```bash + cd ../ + poetry run -C api bash dev/pytest/pytest_all_tests.sh ``` - > flask db upgrade - Error: While importing 'app', an ImportError was raised: + + +## Usage with pip + +> [!NOTE] +> In the next version, we will deprecate pip as the primary package management tool for dify api service, currently Poetry and pip coexist. + +1. Start the docker-compose stack + + The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`. + + ```bash + cd ../docker + docker-compose -f docker-compose.middleware.yaml -p dify up -d + cd ../api ``` + +2. Copy `.env.example` to `.env` +3. Generate a `SECRET_KEY` in the `.env` file. - Please run the following command instead. + ```bash + sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env + ``` +4. Create environment. + + If you use Anaconda, create a new environment and activate it + + ```bash + conda create --name dify python=3.10 + conda activate dify + ``` + +6. Install dependencies + + ```bash + pip install -r requirements.txt ``` - pip install -r requirements.txt --upgrade --force-reinstall + +7. Run migrate + + Before the first launch, migrate the database to the latest version. + + ```bash + flask db upgrade ``` -7. Start backend: +8. Start backend: ```bash flask run --host 0.0.0.0 --port=5001 --debug ``` -8. Setup your application by visiting http://localhost:5001/console/api/setup or other apis... -9. If you need to debug local async processing, please start the worker service by running -`celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail`. -The started celery app handles the async tasks, e.g. dataset importing and documents indexing. +9. Setup your application by visiting http://localhost:5001/console/api/setup or other apis... +10. If you need to debug local async processing, please start the worker service. + ```bash + celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail + ``` + The started celery app handles the async tasks, e.g. dataset importing and documents indexing. ## Testing @@ -83,3 +146,4 @@ The started celery app handles the async tasks, e.g. dataset importing and docum ```bash dev/pytest/pytest_all_tests.sh ``` + diff --git a/api/commands.py b/api/commands.py index da3f7416d434f1..7a8d8000a0f443 100644 --- a/api/commands.py +++ b/api/commands.py @@ -309,6 +309,14 @@ def migrate_knowledge_vector_database(): "vector_store": {"class_prefix": collection_name} } dataset.index_struct = json.dumps(index_struct_dict) + elif vector_type == VectorType.TENCENT: + dataset_id = dataset.id + collection_name = Dataset.gen_collection_name_by_id(dataset_id) + index_struct_dict = { + "type": VectorType.TENCENT, + "vector_store": {"class_prefix": collection_name} + } + dataset.index_struct = json.dumps(index_struct_dict) elif vector_type == VectorType.PGVECTOR: dataset_id = dataset.id collection_name = Dataset.gen_collection_name_by_id(dataset_id) diff --git a/api/config.py b/api/config.py index 15eb30004fc353..1e83965c11f20c 100644 --- a/api/config.py +++ b/api/config.py @@ -24,6 +24,7 @@ 'APP_WEB_URL': 'https://udify.app', 'FILES_URL': '', 'FILES_ACCESS_TIMEOUT': 300, + 'S3_USE_AWS_MANAGED_IAM': 'False', 'S3_ADDRESS_STYLE': 'auto', 'STORAGE_TYPE': 'local', 'STORAGE_LOCAL_PATH': 'storage', @@ -226,6 +227,7 @@ def __init__(self): self.STORAGE_LOCAL_PATH = get_env('STORAGE_LOCAL_PATH') # S3 Storage settings + self.S3_USE_AWS_MANAGED_IAM = get_bool_env('S3_USE_AWS_MANAGED_IAM') self.S3_ENDPOINT = get_env('S3_ENDPOINT') self.S3_BUCKET_NAME = get_env('S3_BUCKET_NAME') self.S3_ACCESS_KEY = get_env('S3_ACCESS_KEY') @@ -286,6 +288,16 @@ def __init__(self): self.RELYT_PASSWORD = get_env('RELYT_PASSWORD') self.RELYT_DATABASE = get_env('RELYT_DATABASE') + + # tencent settings + self.TENCENT_VECTOR_DB_URL = get_env('TENCENT_VECTOR_DB_URL') + self.TENCENT_VECTOR_DB_API_KEY = get_env('TENCENT_VECTOR_DB_API_KEY') + self.TENCENT_VECTOR_DB_TIMEOUT = get_env('TENCENT_VECTOR_DB_TIMEOUT') + self.TENCENT_VECTOR_DB_USERNAME = get_env('TENCENT_VECTOR_DB_USERNAME') + self.TENCENT_VECTOR_DB_DATABASE = get_env('TENCENT_VECTOR_DB_DATABASE') + self.TENCENT_VECTOR_DB_SHARD = get_env('TENCENT_VECTOR_DB_SHARD') + self.TENCENT_VECTOR_DB_REPLICAS = get_env('TENCENT_VECTOR_DB_REPLICAS') + # pgvecto rs settings self.PGVECTO_RS_HOST = get_env('PGVECTO_RS_HOST') self.PGVECTO_RS_PORT = get_env('PGVECTO_RS_PORT') diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index 21660e00dea72c..4b852c1a49a9b1 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -68,8 +68,8 @@ def post(self): parser.add_argument('icon_background', type=str, location='json') args = parser.parse_args() - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() if 'mode' not in args or args['mode'] is None: @@ -89,8 +89,8 @@ class AppImportApi(Resource): @cloud_edition_billing_resource_check('apps') def post(self): """Import app""" - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() parser = reqparse.RequestParser() @@ -147,7 +147,7 @@ def put(self, app_model): @get_app_model def delete(self, app_model): """Delete app""" - if not current_user.is_admin_or_owner: + if not current_user.is_editor: raise Forbidden() app_service = AppService() @@ -164,8 +164,8 @@ class AppCopyApi(Resource): @marshal_with(app_detail_fields_with_site) def post(self, app_model): """Copy app""" - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() parser = reqparse.RequestParser() @@ -238,6 +238,9 @@ class AppSiteStatus(Resource): @get_app_model @marshal_with(app_detail_fields) def post(self, app_model): + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: + raise Forbidden() parser = reqparse.RequestParser() parser.add_argument('enable_site', type=bool, required=True, location='json') args = parser.parse_args() @@ -255,6 +258,9 @@ class AppApiStatus(Resource): @get_app_model @marshal_with(app_detail_fields) def post(self, app_model): + # The role of the current user in the ta table must be admin or owner + if not current_user.is_admin_or_owner: + raise Forbidden() parser = reqparse.RequestParser() parser.add_argument('enable_api', type=bool, required=True, location='json') args = parser.parse_args() diff --git a/api/controllers/console/app/conversation.py b/api/controllers/console/app/conversation.py index 71b52d5cebe840..64751504052afe 100644 --- a/api/controllers/console/app/conversation.py +++ b/api/controllers/console/app/conversation.py @@ -6,7 +6,7 @@ from flask_restful.inputs import int_range from sqlalchemy import func, or_ from sqlalchemy.orm import joinedload -from werkzeug.exceptions import NotFound +from werkzeug.exceptions import Forbidden, NotFound from controllers.console import api from controllers.console.app.wraps import get_app_model @@ -33,6 +33,8 @@ class CompletionConversationApi(Resource): @get_app_model(mode=AppMode.COMPLETION) @marshal_with(conversation_pagination_fields) def get(self, app_model): + if not current_user.is_admin_or_owner: + raise Forbidden() parser = reqparse.RequestParser() parser.add_argument('keyword', type=str, location='args') parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args') @@ -106,6 +108,8 @@ class CompletionConversationDetailApi(Resource): @get_app_model(mode=AppMode.COMPLETION) @marshal_with(conversation_message_detail_fields) def get(self, app_model, conversation_id): + if not current_user.is_admin_or_owner: + raise Forbidden() conversation_id = str(conversation_id) return _get_conversation(app_model, conversation_id) @@ -115,6 +119,8 @@ def get(self, app_model, conversation_id): @account_initialization_required @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]) def delete(self, app_model, conversation_id): + if not current_user.is_admin_or_owner: + raise Forbidden() conversation_id = str(conversation_id) conversation = db.session.query(Conversation) \ @@ -137,6 +143,8 @@ class ChatConversationApi(Resource): @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]) @marshal_with(conversation_with_summary_pagination_fields) def get(self, app_model): + if not current_user.is_admin_or_owner: + raise Forbidden() parser = reqparse.RequestParser() parser.add_argument('keyword', type=str, location='args') parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args') @@ -225,6 +233,8 @@ class ChatConversationDetailApi(Resource): @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]) @marshal_with(conversation_detail_fields) def get(self, app_model, conversation_id): + if not current_user.is_admin_or_owner: + raise Forbidden() conversation_id = str(conversation_id) return _get_conversation(app_model, conversation_id) @@ -234,6 +244,8 @@ def get(self, app_model, conversation_id): @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]) @account_initialization_required def delete(self, app_model, conversation_id): + if not current_user.is_admin_or_owner: + raise Forbidden() conversation_id = str(conversation_id) conversation = db.session.query(Conversation) \ diff --git a/api/controllers/console/app/site.py b/api/controllers/console/app/site.py index 592009fd88bc13..ff832ac5daebb4 100644 --- a/api/controllers/console/app/site.py +++ b/api/controllers/console/app/site.py @@ -40,8 +40,8 @@ class AppSite(Resource): def post(self, app_model): args = parse_app_site_args() - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be editor, admin, or owner + if not current_user.is_editor: raise Forbidden() site = db.session.query(Site). \ @@ -65,13 +65,6 @@ def post(self, app_model): if value is not None: setattr(site, attr_name, value) - if attr_name == 'title': - app_model.name = value - elif attr_name == 'icon': - app_model.icon = value - elif attr_name == 'icon_background': - app_model.icon_background = value - db.session.commit() return site diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index abd18be234b3d5..cb14abe9231d16 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -8,7 +8,7 @@ from controllers.console import api from controllers.console.apikey import api_key_fields, api_key_list from controllers.console.app.error import ProviderNotInitializeError -from controllers.console.datasets.error import DatasetNameDuplicateError +from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError @@ -107,8 +107,8 @@ def post(self): help='Invalid indexing technique.') args = parser.parse_args() - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() try: @@ -195,8 +195,8 @@ def patch(self, dataset_id): parser.add_argument('retrieval_model', type=dict, location='json', help='Invalid retrieval model.') args = parser.parse_args() - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() dataset = DatasetService.update_dataset( @@ -213,14 +213,17 @@ def patch(self, dataset_id): def delete(self, dataset_id): dataset_id_str = str(dataset_id) - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() - if DatasetService.delete_dataset(dataset_id_str, current_user): - return {'result': 'success'}, 204 - else: - raise NotFound("Dataset not found.") + try: + if DatasetService.delete_dataset(dataset_id_str, current_user): + return {'result': 'success'}, 204 + else: + raise NotFound("Dataset not found.") + except services.errors.dataset.DatasetInUseError: + raise DatasetInUseError() class DatasetQueryApi(Resource): @@ -493,9 +496,8 @@ class DatasetRetrievalSettingApi(Resource): @account_initialization_required def get(self): vector_type = current_app.config['VECTOR_STORE'] - match vector_type: - case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA: + case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT: return { 'retrieval_method': [ 'semantic_search' @@ -517,7 +519,7 @@ class DatasetRetrievalSettingMockApi(Resource): @account_initialization_required def get(self, vector_type): match vector_type: - case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA: + case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCEN: return { 'retrieval_method': [ 'semantic_search' diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index 0ddd749639d19f..976b7df6292431 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -226,8 +226,8 @@ def post(self, dataset_id): if not dataset: raise NotFound('Dataset not found.') - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() try: @@ -278,8 +278,8 @@ class DatasetInitApi(Resource): @marshal_with(dataset_and_document_fields) @cloud_edition_billing_resource_check('vector_space') def post(self): - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() parser = reqparse.RequestParser() @@ -646,8 +646,8 @@ def patch(self, dataset_id, document_id, action): document_id = str(document_id) document = self.get_document(dataset_id, document_id) - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() if action == "pause": @@ -710,8 +710,8 @@ def put(self, dataset_id, document_id): doc_type = req_data.get('doc_type') doc_metadata = req_data.get('doc_metadata') - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() if doc_type is None or doc_metadata is None: @@ -757,8 +757,8 @@ def patch(self, dataset_id, document_id, action): document = self.get_document(dataset_id, document_id) - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() indexing_cache_key = 'document_{}_indexing'.format(document.id) diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.py index 0a88a0d8d44d7b..a189aac3f13761 100644 --- a/api/controllers/console/datasets/datasets_segments.py +++ b/api/controllers/console/datasets/datasets_segments.py @@ -126,8 +126,8 @@ def patch(self, dataset_id, segment_id, action): raise NotFound('Dataset not found.') # check user's model setting DatasetService.check_dataset_model_setting(dataset) - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() try: @@ -302,8 +302,8 @@ def patch(self, dataset_id, document_id, segment_id): ).first() if not segment: raise NotFound('Segment not found.') - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() try: DatasetService.check_dataset_permission(dataset, current_user) diff --git a/api/controllers/console/datasets/error.py b/api/controllers/console/datasets/error.py index 3fb190a53725b1..71476764aaa155 100644 --- a/api/controllers/console/datasets/error.py +++ b/api/controllers/console/datasets/error.py @@ -77,3 +77,9 @@ class WebsiteCrawlError(BaseHTTPException): error_code = 'crawl_failed' description = "{message}" code = 500 + + +class DatasetInUseError(BaseHTTPException): + error_code = 'dataset_in_use' + description = "The dataset is being used by some apps. Please remove the dataset from the apps before deleting it." + code = 409 diff --git a/api/controllers/console/tag/tags.py b/api/controllers/console/tag/tags.py index 5dcba301cb48be..55b212358d90de 100644 --- a/api/controllers/console/tag/tags.py +++ b/api/controllers/console/tag/tags.py @@ -35,8 +35,8 @@ def get(self): @login_required @account_initialization_required def post(self): - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() parser = reqparse.RequestParser() @@ -67,8 +67,8 @@ class TagUpdateDeleteApi(Resource): @account_initialization_required def patch(self, tag_id): tag_id = str(tag_id) - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() parser = reqparse.RequestParser() @@ -94,8 +94,8 @@ def patch(self, tag_id): @account_initialization_required def delete(self, tag_id): tag_id = str(tag_id) - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() TagService.delete_tag(tag_id) @@ -109,8 +109,8 @@ class TagBindingCreateApi(Resource): @login_required @account_initialization_required def post(self): - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() parser = reqparse.RequestParser() @@ -134,8 +134,8 @@ class TagBindingDeleteApi(Resource): @login_required @account_initialization_required def post(self): - # The role of the current user in the ta table must be admin or owner - if not current_user.is_admin_or_owner: + # The role of the current user in the ta table must be admin, owner, or editor + if not current_user.is_editor: raise Forbidden() parser = reqparse.RequestParser() diff --git a/api/controllers/console/workspace/members.py b/api/controllers/console/workspace/members.py index 922b95cde58f2e..f404ca7efc4d2d 100644 --- a/api/controllers/console/workspace/members.py +++ b/api/controllers/console/workspace/members.py @@ -43,7 +43,7 @@ def post(self): invitee_emails = args['emails'] invitee_role = args['role'] interface_language = args['language'] - if invitee_role not in [TenantAccountRole.ADMIN, TenantAccountRole.NORMAL]: + if not TenantAccountRole.is_non_owner_role(invitee_role): return {'code': 'invalid-role', 'message': 'Invalid role'}, 400 inviter = current_user @@ -114,7 +114,7 @@ def put(self, member_id): args = parser.parse_args() new_role = args['role'] - if new_role not in ['admin', 'normal', 'owner']: + if not TenantAccountRole.is_valid_role(new_role): return {'code': 'invalid-role', 'message': 'Invalid role'}, 400 member = Account.query.get(str(member_id)) diff --git a/api/controllers/console/workspace/models.py b/api/controllers/console/workspace/models.py index 76ae6a4ab9e0ee..69f2253e97ea40 100644 --- a/api/controllers/console/workspace/models.py +++ b/api/controllers/console/workspace/models.py @@ -11,7 +11,6 @@ from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.utils.encoders import jsonable_encoder from libs.login import login_required -from models.account import TenantAccountRole from services.model_load_balancing_service import ModelLoadBalancingService from services.model_provider_service import ModelProviderService @@ -43,6 +42,9 @@ def get(self): @login_required @account_initialization_required def post(self): + if not current_user.is_admin_or_owner: + raise Forbidden() + parser = reqparse.RequestParser() parser.add_argument('model_settings', type=list, required=True, nullable=False, location='json') args = parser.parse_args() @@ -96,7 +98,7 @@ def get(self, provider): @login_required @account_initialization_required def post(self, provider: str): - if not TenantAccountRole.is_privileged_role(current_user.current_tenant.current_role): + if not current_user.is_admin_or_owner: raise Forbidden() tenant_id = current_user.current_tenant_id @@ -162,7 +164,7 @@ def post(self, provider: str): @login_required @account_initialization_required def delete(self, provider: str): - if not TenantAccountRole.is_privileged_role(current_user.current_tenant.current_role): + if not current_user.is_admin_or_owner: raise Forbidden() tenant_id = current_user.current_tenant_id diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py index dcbfa88d99487d..8dd16c0787cbca 100644 --- a/api/controllers/service_api/dataset/dataset.py +++ b/api/controllers/service_api/dataset/dataset.py @@ -4,7 +4,7 @@ import services.dataset_service from controllers.service_api import api -from controllers.service_api.dataset.error import DatasetNameDuplicateError +from controllers.service_api.dataset.error import DatasetInUseError, DatasetNameDuplicateError from controllers.service_api.wraps import DatasetApiResource from core.model_runtime.entities.model_entities import ModelType from core.provider_manager import ProviderManager @@ -113,10 +113,13 @@ def delete(self, _, dataset_id): dataset_id_str = str(dataset_id) - if DatasetService.delete_dataset(dataset_id_str, current_user): - return {'result': 'success'}, 204 - else: - raise NotFound("Dataset not found.") + try: + if DatasetService.delete_dataset(dataset_id_str, current_user): + return {'result': 'success'}, 204 + else: + raise NotFound("Dataset not found.") + except services.errors.dataset.DatasetInUseError: + raise DatasetInUseError() api.add_resource(DatasetListApi, '/datasets') api.add_resource(DatasetApi, '/datasets/') diff --git a/api/controllers/service_api/dataset/error.py b/api/controllers/service_api/dataset/error.py index 29142b80e627f3..e77693b6c9495c 100644 --- a/api/controllers/service_api/dataset/error.py +++ b/api/controllers/service_api/dataset/error.py @@ -71,3 +71,9 @@ class InvalidMetadataError(BaseHTTPException): error_code = 'invalid_metadata' description = "The metadata content is incorrect. Please check and verify." code = 400 + + +class DatasetInUseError(BaseHTTPException): + error_code = 'dataset_in_use' + description = "The dataset is being used by some apps. Please remove the dataset from the apps before deleting it." + code = 409 diff --git a/api/core/model_runtime/model_providers/bedrock/bedrock.yaml b/api/core/model_runtime/model_providers/bedrock/bedrock.yaml index 09cd73c99ea5f9..88ec712bdf206c 100644 --- a/api/core/model_runtime/model_providers/bedrock/bedrock.yaml +++ b/api/core/model_runtime/model_providers/bedrock/bedrock.yaml @@ -21,16 +21,16 @@ configurate_methods: provider_credential_schema: credential_form_schemas: - variable: aws_access_key_id - required: true + required: false label: - en_US: Access Key + en_US: Access Key (If not provided, credentials are obtained from your running environment. e.g. IAM role) zh_Hans: Access Key type: secret-input placeholder: en_US: Enter your Access Key zh_Hans: 在此输入您的 Access Key - variable: aws_secret_access_key - required: true + required: false label: en_US: Secret Access Key zh_Hans: Secret Access Key diff --git a/api/core/model_runtime/model_providers/bedrock/llm/llm.py b/api/core/model_runtime/model_providers/bedrock/llm/llm.py index 1386d680a43166..d78dadbf7652f0 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/llm.py +++ b/api/core/model_runtime/model_providers/bedrock/llm/llm.py @@ -95,8 +95,8 @@ def _generate_anthropic(self, model: str, credentials: dict, prompt_messages: li # - https://docs.anthropic.com/claude/reference/claude-on-amazon-bedrock # - https://github.com/anthropics/anthropic-sdk-python client = AnthropicBedrock( - aws_access_key=credentials["aws_access_key_id"], - aws_secret_key=credentials["aws_secret_access_key"], + aws_access_key=credentials.get("aws_access_key_id", None), + aws_secret_key=credentials.get("aws_secret_access_key", None), aws_region=credentials["aws_region"], ) @@ -568,8 +568,8 @@ def _generate(self, model: str, credentials: dict, runtime_client = boto3.client( service_name='bedrock-runtime', config=client_config, - aws_access_key_id=credentials["aws_access_key_id"], - aws_secret_access_key=credentials["aws_secret_access_key"] + aws_access_key_id=credentials.get("aws_access_key_id", None), + aws_secret_access_key=credentials.get("aws_secret_access_key", None) ) model_prefix = model.split('.')[0] @@ -826,4 +826,4 @@ def _map_client_to_invoke_error(self, error_code: str, error_msg: str) -> type[I elif error_code == "ModelStreamErrorException": return InvokeConnectionError(error_msg) - return InvokeError(error_msg) \ No newline at end of file + return InvokeError(error_msg) diff --git a/api/core/model_runtime/model_providers/novita/_assets/icon_l_en.svg b/api/core/model_runtime/model_providers/novita/_assets/icon_l_en.svg new file mode 100644 index 00000000000000..5c92cdbc6d8466 --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/_assets/icon_l_en.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/api/core/model_runtime/model_providers/novita/_assets/icon_s_en.svg b/api/core/model_runtime/model_providers/novita/_assets/icon_s_en.svg new file mode 100644 index 00000000000000..798c1d63485221 --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/_assets/icon_s_en.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/api/core/model_runtime/model_providers/novita/llm/Nous-Hermes-2-Mixtral-8x7B-DPO.yaml b/api/core/model_runtime/model_providers/novita/llm/Nous-Hermes-2-Mixtral-8x7B-DPO.yaml new file mode 100644 index 00000000000000..8b19316473f7e4 --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/Nous-Hermes-2-Mixtral-8x7B-DPO.yaml @@ -0,0 +1,36 @@ +model: Nous-Hermes-2-Mixtral-8x7B-DPO +label: + zh_Hans: Nous-Hermes-2-Mixtral-8x7B-DPO + en_US: Nous-Hermes-2-Mixtral-8x7B-DPO +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 32768 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 diff --git a/api/core/model_runtime/model_providers/novita/llm/llama-3-70b-instruct.yaml b/api/core/model_runtime/model_providers/novita/llm/llama-3-70b-instruct.yaml new file mode 100644 index 00000000000000..5298296de38aab --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/llama-3-70b-instruct.yaml @@ -0,0 +1,36 @@ +model: meta-llama/llama-3-70b-instruct +label: + zh_Hans: meta-llama/llama-3-70b-instruct + en_US: meta-llama/llama-3-70b-instruct +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 8192 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 diff --git a/api/core/model_runtime/model_providers/novita/llm/llama-3-8b-instruct.yaml b/api/core/model_runtime/model_providers/novita/llm/llama-3-8b-instruct.yaml new file mode 100644 index 00000000000000..45e62ee52aee45 --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/llama-3-8b-instruct.yaml @@ -0,0 +1,36 @@ +model: meta-llama/llama-3-8b-instruct +label: + zh_Hans: meta-llama/llama-3-8b-instruct + en_US: meta-llama/llama-3-8b-instruct +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 8192 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 diff --git a/api/core/model_runtime/model_providers/novita/llm/llm.py b/api/core/model_runtime/model_providers/novita/llm/llm.py new file mode 100644 index 00000000000000..c7b223d1b7bdbe --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/llm.py @@ -0,0 +1,48 @@ +from collections.abc import Generator +from typing import Optional, Union + +from core.model_runtime.entities.llm_entities import LLMResult +from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool +from core.model_runtime.entities.model_entities import AIModelEntity +from core.model_runtime.model_providers.openai_api_compatible.llm.llm import OAIAPICompatLargeLanguageModel + + +class NovitaLargeLanguageModel(OAIAPICompatLargeLanguageModel): + + def _update_endpoint_url(self, credentials: dict): + credentials['endpoint_url'] = "https://api.novita.ai/v3/openai" + credentials['extra_headers'] = { 'X-Novita-Source': 'dify.ai' } + return credentials + + def _invoke(self, model: str, credentials: dict, + prompt_messages: list[PromptMessage], model_parameters: dict, + tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, + stream: bool = True, user: Optional[str] = None) \ + -> Union[LLMResult, Generator]: + cred_with_endpoint = self._update_endpoint_url(credentials=credentials) + return super()._invoke(model, cred_with_endpoint, prompt_messages, model_parameters, tools, stop, stream, user) + def validate_credentials(self, model: str, credentials: dict) -> None: + cred_with_endpoint = self._update_endpoint_url(credentials=credentials) + self._add_custom_parameters(credentials, model) + return super().validate_credentials(model, cred_with_endpoint) + + @classmethod + def _add_custom_parameters(cls, credentials: dict, model: str) -> None: + credentials['mode'] = 'chat' + + def _generate(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], model_parameters: dict, + tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, + stream: bool = True, user: Optional[str] = None) -> Union[LLMResult, Generator]: + cred_with_endpoint = self._update_endpoint_url(credentials=credentials) + return super()._generate(model, cred_with_endpoint, prompt_messages, model_parameters, tools, stop, stream, user) + + def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity: + cred_with_endpoint = self._update_endpoint_url(credentials=credentials) + + return super().get_customizable_model_schema(model, cred_with_endpoint) + + def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], + tools: Optional[list[PromptMessageTool]] = None) -> int: + cred_with_endpoint = self._update_endpoint_url(credentials=credentials) + + return super().get_num_tokens(model, cred_with_endpoint, prompt_messages, tools) diff --git a/api/core/model_runtime/model_providers/novita/llm/lzlv_70b.yaml b/api/core/model_runtime/model_providers/novita/llm/lzlv_70b.yaml new file mode 100644 index 00000000000000..0facc0c112b280 --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/lzlv_70b.yaml @@ -0,0 +1,36 @@ +model: lzlv_70b +label: + zh_Hans: lzlv_70b + en_US: lzlv_70b +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 4096 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 diff --git a/api/core/model_runtime/model_providers/novita/llm/mythomax-l2-13b.yaml b/api/core/model_runtime/model_providers/novita/llm/mythomax-l2-13b.yaml new file mode 100644 index 00000000000000..28a8630ff290b0 --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/mythomax-l2-13b.yaml @@ -0,0 +1,36 @@ +model: gryphe/mythomax-l2-13b +label: + zh_Hans: gryphe/mythomax-l2-13b + en_US: gryphe/mythomax-l2-13b +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 4096 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 diff --git a/api/core/model_runtime/model_providers/novita/llm/nous-hermes-llama2-13b.yaml b/api/core/model_runtime/model_providers/novita/llm/nous-hermes-llama2-13b.yaml new file mode 100644 index 00000000000000..ce714a118b3937 --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/nous-hermes-llama2-13b.yaml @@ -0,0 +1,36 @@ +model: nousresearch/nous-hermes-llama2-13b +label: + zh_Hans: nousresearch/nous-hermes-llama2-13b + en_US: nousresearch/nous-hermes-llama2-13b +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 4096 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 diff --git a/api/core/model_runtime/model_providers/novita/llm/openhermes-2.5-mistral-7b.yaml b/api/core/model_runtime/model_providers/novita/llm/openhermes-2.5-mistral-7b.yaml new file mode 100644 index 00000000000000..6cef39f8477824 --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/openhermes-2.5-mistral-7b.yaml @@ -0,0 +1,36 @@ +model: teknium/openhermes-2.5-mistral-7b +label: + zh_Hans: teknium/openhermes-2.5-mistral-7b + en_US: teknium/openhermes-2.5-mistral-7b +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 4096 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 diff --git a/api/core/model_runtime/model_providers/novita/llm/wizardlm-2-8x22b.yaml b/api/core/model_runtime/model_providers/novita/llm/wizardlm-2-8x22b.yaml new file mode 100644 index 00000000000000..b3e3a0369793d6 --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/llm/wizardlm-2-8x22b.yaml @@ -0,0 +1,36 @@ +model: microsoft/wizardlm-2-8x22b +label: + zh_Hans: microsoft/wizardlm-2-8x22b + en_US: microsoft/wizardlm-2-8x22b +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 65535 +parameter_rules: + - name: temperature + use_template: temperature + min: 0 + max: 2 + default: 1 + - name: top_p + use_template: top_p + min: 0 + max: 1 + default: 1 + - name: max_tokens + use_template: max_tokens + min: 1 + max: 2048 + default: 512 + - name: frequency_penalty + use_template: frequency_penalty + min: -2 + max: 2 + default: 0 + - name: presence_penalty + use_template: presence_penalty + min: -2 + max: 2 + default: 0 diff --git a/api/core/model_runtime/model_providers/novita/novita.py b/api/core/model_runtime/model_providers/novita/novita.py new file mode 100644 index 00000000000000..f1b72246057c6d --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/novita.py @@ -0,0 +1,31 @@ +import logging + +from core.model_runtime.entities.model_entities import ModelType +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.__base.model_provider import ModelProvider + +logger = logging.getLogger(__name__) + + +class NovitaProvider(ModelProvider): + def validate_provider_credentials(self, credentials: dict) -> None: + """ + Validate provider credentials + if validate failed, raise exception + + :param credentials: provider credentials, credentials form defined in `provider_credential_schema`. + """ + try: + model_instance = self.get_model_instance(ModelType.LLM) + + # Use `meta-llama/llama-3-8b-instruct` model for validate, + # no matter what model you pass in, text completion model or chat model + model_instance.validate_credentials( + model='meta-llama/llama-3-8b-instruct', + credentials=credentials + ) + except CredentialsValidateFailedError as ex: + raise ex + except Exception as ex: + logger.exception(f'{self.get_provider_schema().provider} credentials validate failed') + raise ex diff --git a/api/core/model_runtime/model_providers/novita/novita.yaml b/api/core/model_runtime/model_providers/novita/novita.yaml new file mode 100644 index 00000000000000..ef6a863569828d --- /dev/null +++ b/api/core/model_runtime/model_providers/novita/novita.yaml @@ -0,0 +1,28 @@ +provider: novita +label: + en_US: novita.ai +icon_small: + en_US: icon_s_en.svg +icon_large: + en_US: icon_l_en.svg +background: "#eadeff" +help: + title: + en_US: Get your API key from novita.ai + zh_Hans: 从 novita.ai 获取 API Key + url: + en_US: https://novita.ai/dashboard/key?utm_source=dify +supported_model_types: + - llm +configurate_methods: + - predefined-model +provider_credential_schema: + credential_form_schemas: + - variable: api_key + required: true + label: + en_US: API Key + type: secret-input + placeholder: + zh_Hans: 在此输入您的 API Key + en_US: Enter your API Key diff --git a/api/core/model_runtime/model_providers/ollama/llm/llm.py b/api/core/model_runtime/model_providers/ollama/llm/llm.py index dd58a563abc0d3..42a588e3dd5df1 100644 --- a/api/core/model_runtime/model_providers/ollama/llm/llm.py +++ b/api/core/model_runtime/model_providers/ollama/llm/llm.py @@ -195,7 +195,7 @@ def _generate( data["options"] = model_parameters or {} if stop: - data["stop"] = "\n".join(stop) + data["options"]["stop"] = stop completion_type = LLMMode.value_of(credentials["mode"]) diff --git a/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py b/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py index b921e4b5aa8c31..f8726c853a2a4c 100644 --- a/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py +++ b/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py @@ -74,7 +74,7 @@ def _invoke(self, model: str, credentials: dict, tools=tools, stop=stop, stream=stream, - user=user + user=user, ) def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], @@ -280,6 +280,12 @@ def _generate(self, model: str, credentials: dict, prompt_messages: list[PromptM 'Content-Type': 'application/json', 'Accept-Charset': 'utf-8', } + extra_headers = credentials.get('extra_headers') + if extra_headers is not None: + headers = { + **headers, + **extra_headers, + } api_key = credentials.get('api_key') if api_key: diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-8k-latest.yaml b/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-8k-latest.yaml new file mode 100644 index 00000000000000..50c82564f1d0f3 --- /dev/null +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie-4.0-8k-latest.yaml @@ -0,0 +1,40 @@ +model: ernie-4.0-8k-Latest +label: + en_US: Ernie-4.0-8K-Latest +model_type: llm +features: + - agent-thought +model_properties: + mode: chat + context_size: 8192 +parameter_rules: + - name: temperature + use_template: temperature + min: 0.1 + max: 1.0 + default: 0.8 + - name: top_p + use_template: top_p + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 2 + max: 2048 + - name: presence_penalty + use_template: presence_penalty + default: 1.0 + min: 1.0 + max: 2.0 + - name: frequency_penalty + use_template: frequency_penalty + - name: response_format + use_template: response_format + - name: disable_search + label: + zh_Hans: 禁用搜索 + en_US: Disable Search + type: boolean + help: + zh_Hans: 禁用模型自行进行外部搜索。 + en_US: Disable the model to perform external search. + required: false diff --git a/api/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py b/api/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py index 4646ba384a2438..305769c1c19cca 100644 --- a/api/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py +++ b/api/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py @@ -57,7 +57,7 @@ def _get_access_token(api_key: str, secret_key: str) -> str: raise RateLimitReachedError(f'Rate limit reached: {resp["error_description"]}') else: raise Exception(f'Unknown error: {resp["error_description"]}') - + return resp['access_token'] @staticmethod @@ -114,7 +114,7 @@ def to_dict(self) -> dict[str, Any]: 'role': self.role, 'content': self.content, } - + def __init__(self, content: str, role: str = 'user') -> None: self.content = content self.role = role @@ -131,6 +131,7 @@ class ErnieBotModel: 'ernie-3.5-4k-0205': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ernie-3.5-4k-0205', 'ernie-3.5-128k': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ernie-3.5-128k', 'ernie-4.0-8k': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/completions_pro', + 'ernie-4.0-8k-latest': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/completions_pro', 'ernie-speed-8k': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ernie_speed', 'ernie-speed-128k': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ernie-speed-128k', 'ernie-speed-appbuilder': 'https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/ai_apaas', @@ -157,7 +158,7 @@ def __init__(self, api_key: str, secret_key: str): self.api_key = api_key self.secret_key = secret_key - def generate(self, model: str, stream: bool, messages: list[ErnieMessage], + def generate(self, model: str, stream: bool, messages: list[ErnieMessage], parameters: dict[str, Any], timeout: int, tools: list[PromptMessageTool], \ stop: list[str], user: str) \ -> Union[Generator[ErnieMessage, None, None], ErnieMessage]: @@ -189,7 +190,7 @@ def generate(self, model: str, stream: bool, messages: list[ErnieMessage], if stream: return self._handle_chat_stream_generate_response(resp) return self._handle_chat_generate_response(resp) - + def _handle_error(self, code: int, msg: str): error_map = { 1: InternalServerError, @@ -234,15 +235,15 @@ def _handle_error(self, code: int, msg: str): def _get_access_token(self) -> str: token = BaiduAccessToken.get_access_token(self.api_key, self.secret_key) return token.access_token - + def _copy_messages(self, messages: list[ErnieMessage]) -> list[ErnieMessage]: return [ErnieMessage(message.content, message.role) for message in messages] - def _check_parameters(self, model: str, parameters: dict[str, Any], + def _check_parameters(self, model: str, parameters: dict[str, Any], tools: list[PromptMessageTool], stop: list[str]) -> None: if model not in self.api_bases: raise BadRequestError(f'Invalid model: {model}') - + # if model not in self.function_calling_supports and tools is not None and len(tools) > 0: # raise BadRequestError(f'Model {model} does not support calling function.') # ErnieBot supports function calling, however, there is lots of limitations. @@ -259,32 +260,32 @@ def _check_parameters(self, model: str, parameters: dict[str, Any], for s in stop: if len(s) > 20: raise BadRequestError('stop item should not exceed 20 characters.') - + def _build_request_body(self, model: str, messages: list[ErnieMessage], stream: bool, parameters: dict[str, Any], tools: list[PromptMessageTool], stop: list[str], user: str) -> dict[str, Any]: # if model in self.function_calling_supports: # return self._build_function_calling_request_body(model, messages, parameters, tools, stop, user) return self._build_chat_request_body(model, messages, stream, parameters, stop, user) - + def _build_function_calling_request_body(self, model: str, messages: list[ErnieMessage], stream: bool, - parameters: dict[str, Any], tools: list[PromptMessageTool], + parameters: dict[str, Any], tools: list[PromptMessageTool], stop: list[str], user: str) \ -> dict[str, Any]: if len(messages) % 2 == 0: raise BadRequestError('The number of messages should be odd.') if messages[0].role == 'function': raise BadRequestError('The first message should be user message.') - + """ TODO: implement function calling """ - def _build_chat_request_body(self, model: str, messages: list[ErnieMessage], stream: bool, + def _build_chat_request_body(self, model: str, messages: list[ErnieMessage], stream: bool, parameters: dict[str, Any], stop: list[str], user: str) \ -> dict[str, Any]: if len(messages) == 0: raise BadRequestError('The number of messages should not be zero.') - + # check if the first element is system, shift it system_message = '' if messages[0].role == 'system': @@ -313,7 +314,7 @@ def _build_chat_request_body(self, model: str, messages: list[ErnieMessage], str body['system'] = system_message return body - + def _handle_chat_generate_response(self, response: Response) -> ErnieMessage: data = response.json() if 'error_code' in data: @@ -349,7 +350,7 @@ def _handle_chat_stream_generate_response(self, response: Response) -> Generator self._handle_error(code, msg) except Exception as e: raise InternalServerError(f'Failed to parse response: {e}') - + if line.startswith('data:'): line = line[5:].strip() else: @@ -361,7 +362,7 @@ def _handle_chat_stream_generate_response(self, response: Response) -> Generator data = loads(line) except Exception as e: raise InternalServerError(f'Failed to parse response: {e}') - + result = data['result'] is_end = data['is_end'] @@ -379,4 +380,4 @@ def _handle_chat_stream_generate_response(self, response: Response) -> Generator yield message else: message = ErnieMessage(content=result, role='assistant') - yield message \ No newline at end of file + yield message diff --git a/api/core/moderation/openai_moderation/openai_moderation.py b/api/core/moderation/openai_moderation/openai_moderation.py index bc868c2d524fc9..fee51007ebeed7 100644 --- a/api/core/moderation/openai_moderation/openai_moderation.py +++ b/api/core/moderation/openai_moderation/openai_moderation.py @@ -41,7 +41,7 @@ def moderation_for_outputs(self, text: str) -> ModerationOutputsResult: return ModerationOutputsResult(flagged=flagged, action=ModerationAction.DIRECT_OUTPUT, preset_response=preset_response) def _is_violated(self, inputs: dict): - text = '\n'.join(inputs.values()) + text = '\n'.join(str(inputs.values())) model_manager = ModelManager() model_instance = model_manager.get_model_instance( tenant_id=self.tenant_id, diff --git a/api/core/rag/datasource/vdb/tencent/__init__.py b/api/core/rag/datasource/vdb/tencent/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/core/rag/datasource/vdb/tencent/tencent_vector.py b/api/core/rag/datasource/vdb/tencent/tencent_vector.py new file mode 100644 index 00000000000000..2372976bad3770 --- /dev/null +++ b/api/core/rag/datasource/vdb/tencent/tencent_vector.py @@ -0,0 +1,227 @@ +import json +from typing import Any, Optional + +from flask import current_app +from pydantic import BaseModel +from tcvectordb import VectorDBClient +from tcvectordb.model import document, enum +from tcvectordb.model import index as vdb_index +from tcvectordb.model.document import Filter + +from core.rag.datasource.entity.embedding import Embeddings +from core.rag.datasource.vdb.vector_base import BaseVector +from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory +from core.rag.datasource.vdb.vector_type import VectorType +from core.rag.models.document import Document +from extensions.ext_redis import redis_client +from models.dataset import Dataset + + +class TencentConfig(BaseModel): + url: str + api_key: Optional[str] + timeout: float = 30 + username: Optional[str] + database: Optional[str] + index_type: str = "HNSW" + metric_type: str = "L2" + shard: int = 1, + replicas: int = 2, + + def to_tencent_params(self): + return { + 'url': self.url, + 'username': self.username, + 'key': self.api_key, + 'timeout': self.timeout + } + + +class TencentVector(BaseVector): + field_id: str = "id" + field_vector: str = "vector" + field_text: str = "text" + field_metadata: str = "metadata" + + def __init__(self, collection_name: str, config: TencentConfig): + super().__init__(collection_name) + self._client_config = config + self._client = VectorDBClient(**self._client_config.to_tencent_params()) + self._db = self._init_database() + + def _init_database(self): + exists = False + for db in self._client.list_databases(): + if db.database_name == self._client_config.database: + exists = True + break + if exists: + return self._client.database(self._client_config.database) + else: + return self._client.create_database(database_name=self._client_config.database) + + def get_type(self) -> str: + return 'tencent' + + def to_index_struct(self) -> dict: + return { + "type": self.get_type(), + "vector_store": {"class_prefix": self._collection_name} + } + + def _has_collection(self) -> bool: + collections = self._db.list_collections() + for collection in collections: + if collection.collection_name == self._collection_name: + return True + return False + + def _create_collection(self, dimension: int) -> None: + lock_name = 'vector_indexing_lock_{}'.format(self._collection_name) + with redis_client.lock(lock_name, timeout=20): + collection_exist_cache_key = 'vector_indexing_{}'.format(self._collection_name) + if redis_client.get(collection_exist_cache_key): + return + + if self._has_collection(): + return + + self.delete() + index_type = None + for k, v in enum.IndexType.__members__.items(): + if k == self._client_config.index_type: + index_type = v + if index_type is None: + raise ValueError("unsupported index_type") + metric_type = None + for k, v in enum.MetricType.__members__.items(): + if k == self._client_config.metric_type: + metric_type = v + if metric_type is None: + raise ValueError("unsupported metric_type") + params = vdb_index.HNSWParams(m=16, efconstruction=200) + index = vdb_index.Index( + vdb_index.FilterIndex( + self.field_id, enum.FieldType.String, enum.IndexType.PRIMARY_KEY + ), + vdb_index.VectorIndex( + self.field_vector, + dimension, + index_type, + metric_type, + params, + ), + vdb_index.FilterIndex( + self.field_text, enum.FieldType.String, enum.IndexType.FILTER + ), + vdb_index.FilterIndex( + self.field_metadata, enum.FieldType.String, enum.IndexType.FILTER + ), + ) + + self._db.create_collection( + name=self._collection_name, + shard=self._client_config.shard, + replicas=self._client_config.replicas, + description="Collection for Dify", + index=index, + ) + redis_client.set(collection_exist_cache_key, 1, ex=3600) + + def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs): + self._create_collection(len(embeddings[0])) + self.add_texts(texts, embeddings) + + def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs): + texts = [doc.page_content for doc in documents] + metadatas = [doc.metadata for doc in documents] + total_count = len(embeddings) + docs = [] + for id in range(0, total_count): + if metadatas is None: + continue + metadata = json.dumps(metadatas[id]) + doc = document.Document( + id=metadatas[id]["doc_id"], + vector=embeddings[id], + text=texts[id], + metadata=metadata, + ) + docs.append(doc) + self._db.collection(self._collection_name).upsert(docs, self._client_config.timeout) + + def text_exists(self, id: str) -> bool: + docs = self._db.collection(self._collection_name).query(document_ids=[id]) + if docs and len(docs) > 0: + return True + return False + + def delete_by_ids(self, ids: list[str]) -> None: + self._db.collection(self._collection_name).delete(document_ids=ids) + + def delete_by_metadata_field(self, key: str, value: str) -> None: + self._db.collection(self._collection_name).delete(filter=Filter(Filter.In(key, [value]))) + + def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]: + + res = self._db.collection(self._collection_name).search(vectors=[query_vector], + params=document.HNSWSearchParams( + ef=kwargs.get("ef", 10)), + retrieve_vector=False, + limit=kwargs.get('top_k', 4), + timeout=self._client_config.timeout, + ) + score_threshold = kwargs.get("score_threshold", .0) if kwargs.get('score_threshold', .0) else 0.0 + return self._get_search_res(res, score_threshold) + + def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: + return [] + + def _get_search_res(self, res, score_threshold): + docs = [] + if res is None or len(res) == 0: + return docs + + for result in res[0]: + meta = result.get(self.field_metadata) + if meta is not None: + meta = json.loads(meta) + score = 1 - result.get("score", 0.0) + if score > score_threshold: + meta["score"] = score + doc = Document(page_content=result.get(self.field_text), metadata=meta) + docs.append(doc) + + return docs + + def delete(self) -> None: + self._db.drop_collection(name=self._collection_name) + + + + +class TencentVectorFactory(AbstractVectorFactory): + def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings) -> TencentVector: + + if dataset.index_struct_dict: + class_prefix: str = dataset.index_struct_dict['vector_store']['class_prefix'] + collection_name = class_prefix.lower() + else: + dataset_id = dataset.id + collection_name = Dataset.gen_collection_name_by_id(dataset_id).lower() + dataset.index_struct = json.dumps( + self.gen_index_struct_dict(VectorType.TIDB_VECTOR, collection_name)) + + config = current_app.config + return TencentVector( + collection_name=collection_name, + config=TencentConfig( + url=config.get('TENCENT_VECTOR_DB_URL'), + api_key=config.get('TENCENT_VECTOR_DB_API_KEY'), + timeout=config.get('TENCENT_VECTOR_DB_TIMEOUT'), + username=config.get('TENCENT_VECTOR_DB_USERNAME'), + database=config.get('TENCENT_VECTOR_DB_DATABASE'), + shard=config.get('TENCENT_VECTOR_DB_SHARD'), + replicas=config.get('TENCENT_VECTOR_DB_REPLICAS'), + ) + ) \ No newline at end of file diff --git a/api/core/rag/datasource/vdb/vector_factory.py b/api/core/rag/datasource/vdb/vector_factory.py index d5c803a14a9f2f..48f18df31f0be2 100644 --- a/api/core/rag/datasource/vdb/vector_factory.py +++ b/api/core/rag/datasource/vdb/vector_factory.py @@ -39,7 +39,6 @@ def __init__(self, dataset: Dataset, attributes: list = None): def _init_vector(self) -> BaseVector: config = current_app.config vector_type = config.get('VECTOR_STORE') - if self._dataset.index_struct_dict: vector_type = self._dataset.index_struct_dict['type'] @@ -76,6 +75,9 @@ def get_vector_factory(vector_type: str) -> type[AbstractVectorFactory]: case VectorType.WEAVIATE: from core.rag.datasource.vdb.weaviate.weaviate_vector import WeaviateVectorFactory return WeaviateVectorFactory + case VectorType.TENCENT: + from core.rag.datasource.vdb.tencent.tencent_vector import TencentVectorFactory + return TencentVectorFactory case _: raise ValueError(f"Vector store {vector_type} is not supported.") diff --git a/api/core/rag/datasource/vdb/vector_type.py b/api/core/rag/datasource/vdb/vector_type.py index 76f2f4ae46b454..aba4f757507329 100644 --- a/api/core/rag/datasource/vdb/vector_type.py +++ b/api/core/rag/datasource/vdb/vector_type.py @@ -10,3 +10,4 @@ class VectorType(str, Enum): RELYT = 'relyt' TIDB_VECTOR = 'tidb_vector' WEAVIATE = 'weaviate' + TENCENT = 'tencent' diff --git a/api/core/tools/provider/_position.yaml b/api/core/tools/provider/_position.yaml index 3b0f78cc76da31..74940e819f23e3 100644 --- a/api/core/tools/provider/_position.yaml +++ b/api/core/tools/provider/_position.yaml @@ -27,4 +27,5 @@ - qrcode - dingtalk - feishu +- feishu_base - slack diff --git a/api/core/tools/provider/builtin/feishu_base/_assets/icon.svg b/api/core/tools/provider/builtin/feishu_base/_assets/icon.svg new file mode 100644 index 00000000000000..2663a0f59ee6a4 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/_assets/icon.svg @@ -0,0 +1,47 @@ + + + + diff --git a/api/core/tools/provider/builtin/feishu_base/feishu_base.py b/api/core/tools/provider/builtin/feishu_base/feishu_base.py new file mode 100644 index 00000000000000..febb769ff83cc9 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/feishu_base.py @@ -0,0 +1,8 @@ +from core.tools.provider.builtin.feishu_base.tools.get_tenant_access_token import GetTenantAccessTokenTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + +class FeishuBaseProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict) -> None: + GetTenantAccessTokenTool() + pass \ No newline at end of file diff --git a/api/core/tools/provider/builtin/feishu_base/feishu_base.yaml b/api/core/tools/provider/builtin/feishu_base/feishu_base.yaml new file mode 100644 index 00000000000000..f3dcbb6136b3a3 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/feishu_base.yaml @@ -0,0 +1,14 @@ +identity: + author: Doug Lea + name: feishu_base + label: + en_US: Feishu Base + zh_Hans: 飞书多维表格 + description: + en_US: Feishu Base + zh_Hans: 飞书多维表格 + icon: icon.svg + tags: + - social + - productivity +credentials_for_provider: diff --git a/api/core/tools/provider/builtin/feishu_base/tools/add_base_record.py b/api/core/tools/provider/builtin/feishu_base/tools/add_base_record.py new file mode 100644 index 00000000000000..be43b43ce47337 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/add_base_record.py @@ -0,0 +1,52 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class AddBaseRecordTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + url = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records" + + access_token = tool_parameters.get('Authorization', '') + if not access_token: + return self.create_text_message('Invalid parameter access_token') + + app_token = tool_parameters.get('app_token', '') + if not app_token: + return self.create_text_message('Invalid parameter app_token') + + table_id = tool_parameters.get('table_id', '') + if not table_id: + return self.create_text_message('Invalid parameter table_id') + + fields = tool_parameters.get('fields', '') + if not fields: + return self.create_text_message('Invalid parameter fields') + + headers = { + 'Content-Type': 'application/json', + 'Authorization': f"Bearer {access_token}", + } + + params = {} + payload = { + "fields": json.loads(fields) + } + + try: + res = httpx.post(url.format(app_token=app_token, table_id=table_id), headers=headers, params=params, + json=payload, timeout=30) + res_json = res.json() + if res.is_success: + return self.create_text_message(text=json.dumps(res_json)) + else: + return self.create_text_message( + f"Failed to add base record, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to add base record. {}".format(e)) diff --git a/api/core/tools/provider/builtin/feishu_base/tools/add_base_record.yaml b/api/core/tools/provider/builtin/feishu_base/tools/add_base_record.yaml new file mode 100644 index 00000000000000..3ce0154efd69dc --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/add_base_record.yaml @@ -0,0 +1,66 @@ +identity: + name: add_base_record + author: Doug Lea + label: + en_US: Add Base Record + zh_Hans: 在多维表格数据表中新增一条记录 +description: + human: + en_US: Add Base Record + zh_Hans: | + 在多维表格数据表中新增一条记录，详细请参考：https://open.larkoffice.com/document/server-docs/docs/bitable-v1/app-table-record/create + llm: Add a new record in the multidimensional table data table. +parameters: + - name: Authorization + type: string + required: true + label: + en_US: token + zh_Hans: 凭证 + human_description: + en_US: API access token parameter, tenant_access_token or user_access_token + zh_Hans: API 的访问凭证参数，tenant_access_token 或 user_access_token + llm_description: API access token parameter, tenant_access_token or user_access_token + form: llm + + - name: app_token + type: string + required: true + label: + en_US: app_token + zh_Hans: 多维表格 + human_description: + en_US: bitable app token + zh_Hans: 多维表格的唯一标识符 app_token + llm_description: bitable app token + form: llm + + - name: table_id + type: string + required: true + label: + en_US: table_id + zh_Hans: 多维表格的数据表 + human_description: + en_US: bitable table id + zh_Hans: 多维表格数据表的唯一标识符 table_id + llm_description: bitable table id + form: llm + + - name: fields + type: string + required: true + label: + en_US: fields + zh_Hans: 数据表的列字段内容 + human_description: + en_US: The fields of the Base data table are the columns of the data table. + zh_Hans: | + 要增加一行多维表格记录，字段结构拼接如下：{"多行文本":"多行文本内容","单选":"选项1","多选":["选项1","选项2"],"复选框":true,"人员":[{"id":"ou_2910013f1e6456f16a0ce75ede950a0a"}],"群组":[{"id":"oc_cd07f55f14d6f4a4f1b51504e7e97f48"}],"电话号码":"13026162666"} + 当前接口支持的字段类型为：多行文本、单选、条码、多选、日期、人员、附件、复选框、超链接、数字、单向关联、双向关联、电话号码、地理位置。 + 不同类型字段的数据结构请参考数据结构概述：https://open.larkoffice.com/document/server-docs/docs/bitable-v1/bitable-structure + llm_description: | + 要增加一行多维表格记录，字段结构拼接如下：{"多行文本":"多行文本内容","单选":"选项1","多选":["选项1","选项2"],"复选框":true,"人员":[{"id":"ou_2910013f1e6456f16a0ce75ede950a0a"}],"群组":[{"id":"oc_cd07f55f14d6f4a4f1b51504e7e97f48"}],"电话号码":"13026162666"} + 当前接口支持的字段类型为：多行文本、单选、条码、多选、日期、人员、附件、复选框、超链接、数字、单向关联、双向关联、电话号码、地理位置。 + 不同类型字段的数据结构请参考数据结构概述：https://open.larkoffice.com/document/server-docs/docs/bitable-v1/bitable-structure + form: llm diff --git a/api/core/tools/provider/builtin/feishu_base/tools/create_base.py b/api/core/tools/provider/builtin/feishu_base/tools/create_base.py new file mode 100644 index 00000000000000..639644e7f0e3ea --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/create_base.py @@ -0,0 +1,43 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class CreateBaseTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + url = "https://open.feishu.cn/open-apis/bitable/v1/apps" + + access_token = tool_parameters.get('Authorization', '') + if not access_token: + return self.create_text_message('Invalid parameter access_token') + + name = tool_parameters.get('name', '') + folder_token = tool_parameters.get('folder_token', '') + + headers = { + 'Content-Type': 'application/json', + 'Authorization': f"Bearer {access_token}", + } + + params = {} + payload = { + "name": name, + "folder_token": folder_token + } + + try: + res = httpx.post(url, headers=headers, params=params, json=payload, timeout=30) + res_json = res.json() + if res.is_success: + return self.create_text_message(text=json.dumps(res_json)) + else: + return self.create_text_message( + f"Failed to create base, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to create base. {}".format(e)) diff --git a/api/core/tools/provider/builtin/feishu_base/tools/create_base.yaml b/api/core/tools/provider/builtin/feishu_base/tools/create_base.yaml new file mode 100644 index 00000000000000..76c76a916d4951 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/create_base.yaml @@ -0,0 +1,47 @@ +identity: + name: create_base + author: Doug Lea + label: + en_US: Create Base + zh_Hans: 创建多维表格 +description: + human: + en_US: Create base + zh_Hans: 在指定目录下创建多维表格 + llm: A tool for create a multidimensional table in the specified directory. +parameters: + - name: Authorization + type: string + required: true + label: + en_US: token + zh_Hans: 凭证 + human_description: + en_US: API access token parameter, tenant_access_token or user_access_token + zh_Hans: API 的访问凭证参数，tenant_access_token 或 user_access_token + llm_description: API access token parameter, tenant_access_token or user_access_token + form: llm + + - name: name + type: string + required: false + label: + en_US: name + zh_Hans: name + human_description: + en_US: Base App Name + zh_Hans: 多维表格App名字 + llm_description: Base App Name + form: llm + + - name: folder_token + type: string + required: false + label: + en_US: folder_token + zh_Hans: 多维表格App归属文件夹 + human_description: + en_US: Base App home folder. The default is empty, indicating that Base will be created in the cloud space root directory. + zh_Hans: 多维表格App归属文件夹。默认为空，表示多维表格将被创建在云空间根目录。 + llm_description: Base App home folder. The default is empty, indicating that Base will be created in the cloud space root directory. + form: llm diff --git a/api/core/tools/provider/builtin/feishu_base/tools/create_base_table.py b/api/core/tools/provider/builtin/feishu_base/tools/create_base_table.py new file mode 100644 index 00000000000000..e9062e8730f9ac --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/create_base_table.py @@ -0,0 +1,52 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class CreateBaseTableTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + url = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables" + + access_token = tool_parameters.get('Authorization', '') + if not access_token: + return self.create_text_message('Invalid parameter access_token') + + app_token = tool_parameters.get('app_token', '') + if not app_token: + return self.create_text_message('Invalid parameter app_token') + + name = tool_parameters.get('name', '') + + fields = tool_parameters.get('fields', '') + if not fields: + return self.create_text_message('Invalid parameter fields') + + headers = { + 'Content-Type': 'application/json', + 'Authorization': f"Bearer {access_token}", + } + + params = {} + payload = { + "table": { + "name": name, + "fields": json.loads(fields) + } + } + + try: + res = httpx.post(url.format(app_token=app_token), headers=headers, params=params, json=payload, timeout=30) + res_json = res.json() + if res.is_success: + return self.create_text_message(text=json.dumps(res_json)) + else: + return self.create_text_message( + f"Failed to create base table, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to create base table. {}".format(e)) diff --git a/api/core/tools/provider/builtin/feishu_base/tools/create_base_table.yaml b/api/core/tools/provider/builtin/feishu_base/tools/create_base_table.yaml new file mode 100644 index 00000000000000..48c46bec14f448 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/create_base_table.yaml @@ -0,0 +1,106 @@ +identity: + name: create_base_table + author: Doug Lea + label: + en_US: Create Base Table + zh_Hans: 多维表格新增一个数据表 +description: + human: + en_US: Create base table + zh_Hans: | + 多维表格新增一个数据表，详细请参考：https://open.larkoffice.com/document/server-docs/docs/bitable-v1/app-table/create + llm: A tool for add a new data table to the multidimensional table. +parameters: + - name: Authorization + type: string + required: true + label: + en_US: token + zh_Hans: 凭证 + human_description: + en_US: API access token parameter, tenant_access_token or user_access_token + zh_Hans: API 的访问凭证参数，tenant_access_token 或 user_access_token + llm_description: API access token parameter, tenant_access_token or user_access_token + form: llm + + - name: app_token + type: string + required: true + label: + en_US: app_token + zh_Hans: 多维表格 + human_description: + en_US: bitable app token + zh_Hans: 多维表格的唯一标识符 app_token + llm_description: bitable app token + form: llm + + - name: name + type: string + required: false + label: + en_US: name + zh_Hans: name + human_description: + en_US: Multidimensional table data table name + zh_Hans: 多维表格数据表名称 + llm_description: Multidimensional table data table name + form: llm + + - name: fields + type: string + required: true + label: + en_US: fields + zh_Hans: fields + human_description: + en_US: Initial fields of the data table + zh_Hans: | + 数据表的初始字段，格式为：[{"field_name":"多行文本","type":1},{"field_name":"数字","type":2},{"field_name":"单选","type":3},{"field_name":"多选","type":4},{"field_name":"日期","type":5}]。 + field_name：字段名； + type: 字段类型；可选值有 + 1：多行文本 + 2：数字 + 3：单选 + 4：多选 + 5：日期 + 7：复选框 + 11：人员 + 13：电话号码 + 15：超链接 + 17：附件 + 18：单向关联 + 20：公式 + 21：双向关联 + 22：地理位置 + 23：群组 + 1001：创建时间 + 1002：最后更新时间 + 1003：创建人 + 1004：修改人 + 1005：自动编号 + llm_description: | + 数据表的初始字段，格式为：[{"field_name":"多行文本","type":1},{"field_name":"数字","type":2},{"field_name":"单选","type":3},{"field_name":"多选","type":4},{"field_name":"日期","type":5}]。 + field_name：字段名； + type: 字段类型；可选值有 + 1：多行文本 + 2：数字 + 3：单选 + 4：多选 + 5：日期 + 7：复选框 + 11：人员 + 13：电话号码 + 15：超链接 + 17：附件 + 18：单向关联 + 20：公式 + 21：双向关联 + 22：地理位置 + 23：群组 + 1001：创建时间 + 1002：最后更新时间 + 1003：创建人 + 1004：修改人 + 1005：自动编号 + form: llm diff --git a/api/core/tools/provider/builtin/feishu_base/tools/delete_base_records.py b/api/core/tools/provider/builtin/feishu_base/tools/delete_base_records.py new file mode 100644 index 00000000000000..aa13aad6fac287 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/delete_base_records.py @@ -0,0 +1,52 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class DeleteBaseRecordsTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + url = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records/batch_delete" + + access_token = tool_parameters.get('Authorization', '') + if not access_token: + return self.create_text_message('Invalid parameter access_token') + + app_token = tool_parameters.get('app_token', '') + if not app_token: + return self.create_text_message('Invalid parameter app_token') + + table_id = tool_parameters.get('table_id', '') + if not table_id: + return self.create_text_message('Invalid parameter table_id') + + record_ids = tool_parameters.get('record_ids', '') + if not record_ids: + return self.create_text_message('Invalid parameter record_ids') + + headers = { + 'Content-Type': 'application/json', + 'Authorization': f"Bearer {access_token}", + } + + params = {} + payload = { + "records": json.loads(record_ids) + } + + try: + res = httpx.post(url.format(app_token=app_token, table_id=table_id), headers=headers, params=params, + json=payload, timeout=30) + res_json = res.json() + if res.is_success: + return self.create_text_message(text=json.dumps(res_json)) + else: + return self.create_text_message( + f"Failed to delete base records, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to delete base records. {}".format(e)) diff --git a/api/core/tools/provider/builtin/feishu_base/tools/delete_base_records.yaml b/api/core/tools/provider/builtin/feishu_base/tools/delete_base_records.yaml new file mode 100644 index 00000000000000..595b2870298af9 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/delete_base_records.yaml @@ -0,0 +1,60 @@ +identity: + name: delete_base_records + author: Doug Lea + label: + en_US: Delete Base Records + zh_Hans: 在多维表格数据表中删除多条记录 +description: + human: + en_US: Delete base records + zh_Hans: | + 该接口用于删除多维表格数据表中的多条记录，单次调用中最多删除 500 条记录。 + llm: A tool for delete multiple records in a multidimensional table data table, up to 500 records can be deleted in a single call. +parameters: + - name: Authorization + type: string + required: true + label: + en_US: token + zh_Hans: 凭证 + human_description: + en_US: API access token parameter, tenant_access_token or user_access_token + zh_Hans: API 的访问凭证参数，tenant_access_token 或 user_access_token + llm_description: API access token parameter, tenant_access_token or user_access_token + form: llm + + - name: app_token + type: string + required: true + label: + en_US: app_token + zh_Hans: 多维表格 + human_description: + en_US: bitable app token + zh_Hans: 多维表格的唯一标识符 app_token + llm_description: bitable app token + form: llm + + - name: table_id + type: string + required: true + label: + en_US: table_id + zh_Hans: 多维表格的数据表 + human_description: + en_US: bitable table id + zh_Hans: 多维表格数据表的唯一标识符 table_id + llm_description: bitable table id + form: llm + + - name: record_ids + type: string + required: true + label: + en_US: record_ids + zh_Hans: record_ids + human_description: + en_US: A list of multiple record IDs to be deleted, for example ["recwNXzPQv","recpCsf4ME"] + zh_Hans: 待删除的多条记录id列表，示例为 ["recwNXzPQv","recpCsf4ME"] + llm_description: A list of multiple record IDs to be deleted, for example ["recwNXzPQv","recpCsf4ME"] + form: llm diff --git a/api/core/tools/provider/builtin/feishu_base/tools/delete_base_tables.py b/api/core/tools/provider/builtin/feishu_base/tools/delete_base_tables.py new file mode 100644 index 00000000000000..c4280ebc21eaed --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/delete_base_tables.py @@ -0,0 +1,47 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class DeleteBaseTablesTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + url = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/batch_delete" + + access_token = tool_parameters.get('Authorization', '') + if not access_token: + return self.create_text_message('Invalid parameter access_token') + + app_token = tool_parameters.get('app_token', '') + if not app_token: + return self.create_text_message('Invalid parameter app_token') + + table_ids = tool_parameters.get('table_ids', '') + if not table_ids: + return self.create_text_message('Invalid parameter table_ids') + + headers = { + 'Content-Type': 'application/json', + 'Authorization': f"Bearer {access_token}", + } + + params = {} + payload = { + "table_ids": json.loads(table_ids) + } + + try: + res = httpx.post(url.format(app_token=app_token), headers=headers, params=params, json=payload, timeout=30) + res_json = res.json() + if res.is_success: + return self.create_text_message(text=json.dumps(res_json)) + else: + return self.create_text_message( + f"Failed to delete base tables, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to delete base tables. {}".format(e)) diff --git a/api/core/tools/provider/builtin/feishu_base/tools/delete_base_tables.yaml b/api/core/tools/provider/builtin/feishu_base/tools/delete_base_tables.yaml new file mode 100644 index 00000000000000..5d72814363d86f --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/delete_base_tables.yaml @@ -0,0 +1,48 @@ +identity: + name: delete_base_tables + author: Doug Lea + label: + en_US: Delete Base Tables + zh_Hans: 删除多维表格中的数据表 +description: + human: + en_US: Delete base tables + zh_Hans: | + 删除多维表格中的数据表 + llm: A tool for deleting a data table in a multidimensional table +parameters: + - name: Authorization + type: string + required: true + label: + en_US: token + zh_Hans: 凭证 + human_description: + en_US: API access token parameter, tenant_access_token or user_access_token + zh_Hans: API 的访问凭证参数，tenant_access_token 或 user_access_token + llm_description: API access token parameter, tenant_access_token or user_access_token + form: llm + + - name: app_token + type: string + required: true + label: + en_US: app_token + zh_Hans: 多维表格 + human_description: + en_US: bitable app token + zh_Hans: 多维表格的唯一标识符 app_token + llm_description: bitable app token + form: llm + + - name: table_ids + type: string + required: true + label: + en_US: table_ids + zh_Hans: table_ids + human_description: + en_US: The ID list of the data tables to be deleted. Currently, a maximum of 50 data tables can be deleted at a time. The example is ["tbl1TkhyTWDkSoZ3","tblsRc9GRRXKqhvW"] + zh_Hans: 待删除数据表的id列表，当前一次操作最多支持50个数据表，示例为 ["tbl1TkhyTWDkSoZ3","tblsRc9GRRXKqhvW"] + llm_description: The ID list of the data tables to be deleted. Currently, a maximum of 50 data tables can be deleted at a time. The example is ["tbl1TkhyTWDkSoZ3","tblsRc9GRRXKqhvW"] + form: llm diff --git a/api/core/tools/provider/builtin/feishu_base/tools/get_base_info.py b/api/core/tools/provider/builtin/feishu_base/tools/get_base_info.py new file mode 100644 index 00000000000000..de70f2ed9359dc --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/get_base_info.py @@ -0,0 +1,38 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class GetBaseInfoTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + url = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}" + + access_token = tool_parameters.get('Authorization', '') + if not access_token: + return self.create_text_message('Invalid parameter access_token') + + app_token = tool_parameters.get('app_token', '') + if not app_token: + return self.create_text_message('Invalid parameter app_token') + + headers = { + 'Content-Type': 'application/json', + 'Authorization': f"Bearer {access_token}", + } + + try: + res = httpx.get(url.format(app_token=app_token), headers=headers, timeout=30) + res_json = res.json() + if res.is_success: + return self.create_text_message(text=json.dumps(res_json)) + else: + return self.create_text_message( + f"Failed to get base info, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to get base info. {}".format(e)) diff --git a/api/core/tools/provider/builtin/feishu_base/tools/get_base_info.yaml b/api/core/tools/provider/builtin/feishu_base/tools/get_base_info.yaml new file mode 100644 index 00000000000000..de0868901834ee --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/get_base_info.yaml @@ -0,0 +1,54 @@ +identity: + name: get_base_info + author: Doug Lea + label: + en_US: Get Base Info + zh_Hans: 获取多维表格元数据 +description: + human: + en_US: Get base info + zh_Hans: | + 获取多维表格元数据，响应体如下: + { + "code": 0, + "msg": "success", + "data": { + "app": { + "app_token": "appbcbWCzen6D8dezhoCH2RpMAh", + "name": "mybase", + "revision": 1, + "is_advanced": false, + "time_zone": "Asia/Beijing" + } + } + } + app_token: 多维表格的 app_token; + name: 多维表格的名字; + revision: 多维表格的版本号; + is_advanced: 多维表格是否开启了高级权限。取值包括：(true-表示开启了高级权限，false-表示关闭了高级权限); + time_zone: 文档时区; + llm: A tool to get Base Metadata, imported parameter is Unique Device Identifier app_token of Base, app_token is required. +parameters: + - name: Authorization + type: string + required: true + label: + en_US: token + zh_Hans: 凭证 + human_description: + en_US: API access token parameter, tenant_access_token or user_access_token + zh_Hans: API 的访问凭证参数，tenant_access_token 或 user_access_token + llm_description: API access token parameter, tenant_access_token or user_access_token + form: llm + + - name: app_token + type: string + required: true + label: + en_US: app_token + zh_Hans: 多维表格 + human_description: + en_US: bitable app token + zh_Hans: 多维表格的唯一标识符 app_token + llm_description: bitable app token + form: llm diff --git a/api/core/tools/provider/builtin/feishu_base/tools/get_tenant_access_token.py b/api/core/tools/provider/builtin/feishu_base/tools/get_tenant_access_token.py new file mode 100644 index 00000000000000..88507bda60090f --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/get_tenant_access_token.py @@ -0,0 +1,50 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class GetTenantAccessTokenTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + url = "https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal" + + app_id = tool_parameters.get('app_id', '') + if not app_id: + return self.create_text_message('Invalid parameter app_id') + + app_secret = tool_parameters.get('app_secret', '') + if not app_secret: + return self.create_text_message('Invalid parameter app_secret') + + headers = { + 'Content-Type': 'application/json', + } + params = {} + payload = { + "app_id": app_id, + "app_secret": app_secret + } + + """ + { + "code": 0, + "msg": "ok", + "tenant_access_token": "t-caecc734c2e3328a62489fe0648c4b98779515d3", + "expire": 7200 + } + """ + try: + res = httpx.post(url, headers=headers, params=params, json=payload, timeout=30) + res_json = res.json() + if res.is_success: + return self.create_text_message(text=json.dumps(res_json)) + else: + return self.create_text_message( + f"Failed to get tenant access token, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to get tenant access token. {}".format(e)) diff --git a/api/core/tools/provider/builtin/feishu_base/tools/get_tenant_access_token.yaml b/api/core/tools/provider/builtin/feishu_base/tools/get_tenant_access_token.yaml new file mode 100644 index 00000000000000..88acc27e06eca1 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/get_tenant_access_token.yaml @@ -0,0 +1,39 @@ +identity: + name: get_tenant_access_token + author: Doug Lea + label: + en_US: Get Tenant Access Token + zh_Hans: 获取飞书自建应用的 tenant_access_token +description: + human: + en_US: Get tenant access token + zh_Hans: | + 获取飞书自建应用的 tenant_access_token，响应体示例: + {"code":0,"msg":"ok","tenant_access_token":"t-caecc734c2e3328a62489fe0648c4b98779515d3","expire":7200} + tenant_access_token: 租户访问凭证; + expire: tenant_access_token 的过期时间，单位为秒; + llm: A tool for obtaining a tenant access token. The input parameters must include app_id and app_secret. +parameters: + - name: app_id + type: string + required: true + label: + en_US: app_id + zh_Hans: 应用唯一标识 + human_description: + en_US: app_id is the unique identifier of the Lark Open Platform application + zh_Hans: app_id 是飞书开放平台应用的唯一标识 + llm_description: app_id is the unique identifier of the Lark Open Platform application + form: llm + + - name: app_secret + type: secret-input + required: true + label: + en_US: app_secret + zh_Hans: 应用秘钥 + human_description: + en_US: app_secret is the secret key of the application + zh_Hans: app_secret 是应用的秘钥 + llm_description: app_secret is the secret key of the application + form: llm diff --git a/api/core/tools/provider/builtin/feishu_base/tools/list_base_records.py b/api/core/tools/provider/builtin/feishu_base/tools/list_base_records.py new file mode 100644 index 00000000000000..2a4229f137d7fd --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/list_base_records.py @@ -0,0 +1,61 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class ListBaseRecordsTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + url = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records/search" + + access_token = tool_parameters.get('Authorization', '') + if not access_token: + return self.create_text_message('Invalid parameter access_token') + + app_token = tool_parameters.get('app_token', '') + if not app_token: + return self.create_text_message('Invalid parameter app_token') + + table_id = tool_parameters.get('table_id', '') + if not table_id: + return self.create_text_message('Invalid parameter table_id') + + page_token = tool_parameters.get('page_token', '') + page_size = tool_parameters.get('page_size', '') + sort_condition = tool_parameters.get('sort_condition', '') + filter_condition = tool_parameters.get('filter_condition', '') + + headers = { + 'Content-Type': 'application/json', + 'Authorization': f"Bearer {access_token}", + } + + params = { + "page_token": page_token, + "page_size": page_size, + } + + payload = { + "automatic_fields": True + } + if sort_condition: + payload["sort"] = json.loads(sort_condition) + if filter_condition: + payload["filter"] = json.loads(filter_condition) + + try: + res = httpx.post(url.format(app_token=app_token, table_id=table_id), headers=headers, params=params, + json=payload, timeout=30) + res_json = res.json() + if res.is_success: + return self.create_text_message(text=json.dumps(res_json)) + else: + return self.create_text_message( + f"Failed to list base records, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to list base records. {}".format(e)) diff --git a/api/core/tools/provider/builtin/feishu_base/tools/list_base_records.yaml b/api/core/tools/provider/builtin/feishu_base/tools/list_base_records.yaml new file mode 100644 index 00000000000000..8647c880a60024 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/list_base_records.yaml @@ -0,0 +1,108 @@ +identity: + name: list_base_records + author: Doug Lea + label: + en_US: List Base Records + zh_Hans: 查询多维表格数据表中的现有记录 +description: + human: + en_US: List base records + zh_Hans: | + 查询多维表格数据表中的现有记录，单次最多查询 500 行记录，支持分页获取。 + llm: Query existing records in a multidimensional table data table. A maximum of 500 rows of records can be queried at a time, and paging retrieval is supported. +parameters: + - name: Authorization + type: string + required: true + label: + en_US: token + zh_Hans: 凭证 + human_description: + en_US: API access token parameter, tenant_access_token or user_access_token + zh_Hans: API 的访问凭证参数，tenant_access_token 或 user_access_token + llm_description: API access token parameter, tenant_access_token or user_access_token + form: llm + + - name: app_token + type: string + required: true + label: + en_US: app_token + zh_Hans: 多维表格 + human_description: + en_US: bitable app token + zh_Hans: 多维表格的唯一标识符 app_token + llm_description: bitable app token + form: llm + + - name: table_id + type: string + required: true + label: + en_US: table_id + zh_Hans: 多维表格的数据表 + human_description: + en_US: bitable table id + zh_Hans: 多维表格数据表的唯一标识符 table_id + llm_description: bitable table id + form: llm + + - name: page_token + type: string + required: false + label: + en_US: page_token + zh_Hans: 分页标记 + human_description: + en_US: Pagination mark. If it is not filled in the first request, it means to traverse from the beginning. + zh_Hans: 分页标记，第一次请求不填，表示从头开始遍历。 + llm_description: 分页标记，第一次请求不填，表示从头开始遍历；分页查询结果还有更多项时会同时返回新的 page_token，下次遍历可采用该 page_token 获取查询结果。 + form: llm + + - name: page_size + type: number + required: false + default: 20 + label: + en_US: page_size + zh_Hans: 分页大小 + human_description: + en_US: paging size + zh_Hans: 分页大小，默认值为 20，最大值为 100。 + llm_description: The default value of paging size is 20 and the maximum value is 100. + form: llm + + - name: sort_condition + type: string + required: false + label: + en_US: sort_condition + zh_Hans: 排序条件 + human_description: + en_US: sort condition + zh_Hans: | + 排序条件，格式为：[{"field_name":"多行文本","desc":true}]。 + field_name: 字段名称； + desc: 是否倒序排序; + llm_description: | + Sorting conditions, the format is: [{"field_name":"multi-line text","desc":true}]. + form: llm + + - name: filter_condition + type: string + required: false + label: + en_US: filter_condition + zh_Hans: 筛选条件 + human_description: + en_US: filter condition + zh_Hans: | + 筛选条件，格式为：{"conjunction":"and","conditions":[{"field_name":"字段1","operator":"is","value":["文本内容"]}]}。 + conjunction：条件逻辑连接词； + conditions：筛选条件集合； + field_name：筛选条件的左值，值为字段的名称； + operator：条件运算符； + value：目标值； + llm_description: | + The format of the filter condition is: {"conjunction":"and","conditions":[{"field_name":"Field 1","operator":"is","value":["text content"]}]}. + form: llm diff --git a/api/core/tools/provider/builtin/feishu_base/tools/list_base_tables.py b/api/core/tools/provider/builtin/feishu_base/tools/list_base_tables.py new file mode 100644 index 00000000000000..6d82490eb3235f --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/list_base_tables.py @@ -0,0 +1,46 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class ListBaseTablesTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + url = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables" + + access_token = tool_parameters.get('Authorization', '') + if not access_token: + return self.create_text_message('Invalid parameter access_token') + + app_token = tool_parameters.get('app_token', '') + if not app_token: + return self.create_text_message('Invalid parameter app_token') + + page_token = tool_parameters.get('page_token', '') + page_size = tool_parameters.get('page_size', '') + + headers = { + 'Content-Type': 'application/json', + 'Authorization': f"Bearer {access_token}", + } + + params = { + "page_token": page_token, + "page_size": page_size, + } + + try: + res = httpx.get(url.format(app_token=app_token), headers=headers, params=params, timeout=30) + res_json = res.json() + if res.is_success: + return self.create_text_message(text=json.dumps(res_json)) + else: + return self.create_text_message( + f"Failed to list base tables, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to list base tables. {}".format(e)) diff --git a/api/core/tools/provider/builtin/feishu_base/tools/list_base_tables.yaml b/api/core/tools/provider/builtin/feishu_base/tools/list_base_tables.yaml new file mode 100644 index 00000000000000..9887124a28823a --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/list_base_tables.yaml @@ -0,0 +1,65 @@ +identity: + name: list_base_tables + author: Doug Lea + label: + en_US: List Base Tables + zh_Hans: 根据 app_token 获取多维表格下的所有数据表 +description: + human: + en_US: List base tables + zh_Hans: | + 根据 app_token 获取多维表格下的所有数据表 + llm: A tool for getting all data tables under a multidimensional table based on app_token. +parameters: + - name: Authorization + type: string + required: true + label: + en_US: token + zh_Hans: 凭证 + human_description: + en_US: API access token parameter, tenant_access_token or user_access_token + zh_Hans: API 的访问凭证参数，tenant_access_token 或 user_access_token + llm_description: API access token parameter, tenant_access_token or user_access_token + form: llm + + - name: app_token + type: string + required: true + label: + en_US: app_token + zh_Hans: 多维表格 + human_description: + en_US: bitable app token + zh_Hans: 多维表格的唯一标识符 app_token + llm_description: bitable app token + form: llm + + - name: page_token + type: string + required: false + label: + en_US: page_token + zh_Hans: 分页标记 + human_description: + en_US: Pagination mark. If it is not filled in the first request, it means to traverse from the beginning. + zh_Hans: 分页标记，第一次请求不填，表示从头开始遍历。 + llm_description: | + Pagination token. If it is not filled in the first request, it means to start traversal from the beginning. + If there are more items in the pagination query result, a new page_token will be returned at the same time. + The page_token can be used to obtain the query result in the next traversal. + 分页标记，第一次请求不填，表示从头开始遍历；分页查询结果还有更多项时会同时返回新的 page_token，下次遍历可采用该 page_token 获取查询结果。 + form: llm + + - name: page_size + type: number + required: false + default: 20 + label: + en_US: page_size + zh_Hans: 分页大小 + human_description: + en_US: paging size + zh_Hans: 分页大小，默认值为 20，最大值为 100。 + llm_description: The default value of paging size is 20 and the maximum value is 100. + form: llm diff --git a/api/core/tools/provider/builtin/feishu_base/tools/read_base_record.py b/api/core/tools/provider/builtin/feishu_base/tools/read_base_record.py new file mode 100644 index 00000000000000..bb4bd6c3a6c531 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/read_base_record.py @@ -0,0 +1,47 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class ReadBaseRecordTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + url = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records/{record_id}" + + access_token = tool_parameters.get('Authorization', '') + if not access_token: + return self.create_text_message('Invalid parameter access_token') + + app_token = tool_parameters.get('app_token', '') + if not app_token: + return self.create_text_message('Invalid parameter app_token') + + table_id = tool_parameters.get('table_id', '') + if not table_id: + return self.create_text_message('Invalid parameter table_id') + + record_id = tool_parameters.get('record_id', '') + if not record_id: + return self.create_text_message('Invalid parameter record_id') + + headers = { + 'Content-Type': 'application/json', + 'Authorization': f"Bearer {access_token}", + } + + try: + res = httpx.get(url.format(app_token=app_token, table_id=table_id, record_id=record_id), headers=headers, + timeout=30) + res_json = res.json() + if res.is_success: + return self.create_text_message(text=json.dumps(res_json)) + else: + return self.create_text_message( + f"Failed to read base record, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to read base record. {}".format(e)) diff --git a/api/core/tools/provider/builtin/feishu_base/tools/read_base_record.yaml b/api/core/tools/provider/builtin/feishu_base/tools/read_base_record.yaml new file mode 100644 index 00000000000000..400e9a1021f2db --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/read_base_record.yaml @@ -0,0 +1,60 @@ +identity: + name: read_base_record + author: Doug Lea + label: + en_US: Read Base Record + zh_Hans: 根据 record_id 的值检索多维表格数据表的记录 +description: + human: + en_US: Read base record + zh_Hans: | + 根据 record_id 的值检索多维表格数据表的记录 + llm: Retrieve records from a multidimensional table based on the value of record_id +parameters: + - name: Authorization + type: string + required: true + label: + en_US: token + zh_Hans: 凭证 + human_description: + en_US: API access token parameter, tenant_access_token or user_access_token + zh_Hans: API 的访问凭证参数，tenant_access_token 或 user_access_token + llm_description: API access token parameter, tenant_access_token or user_access_token + form: llm + + - name: app_token + type: string + required: true + label: + en_US: app_token + zh_Hans: 多维表格 + human_description: + en_US: bitable app token + zh_Hans: 多维表格的唯一标识符 app_token + llm_description: bitable app token + form: llm + + - name: table_id + type: string + required: true + label: + en_US: table_id + zh_Hans: 多维表格的数据表 + human_description: + en_US: bitable table id + zh_Hans: 多维表格数据表的唯一标识符 table_id + llm_description: bitable table id + form: llm + + - name: record_id + type: string + required: true + label: + en_US: record_id + zh_Hans: 单条记录的 id + human_description: + en_US: The id of a single record + zh_Hans: 单条记录的 id + llm_description: The id of a single record + form: llm diff --git a/api/core/tools/provider/builtin/feishu_base/tools/update_base_record.py b/api/core/tools/provider/builtin/feishu_base/tools/update_base_record.py new file mode 100644 index 00000000000000..6551053ce22535 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/update_base_record.py @@ -0,0 +1,56 @@ +import json +from typing import Any, Union + +import httpx + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + +class UpdateBaseRecordTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any] + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + url = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records/{record_id}" + + access_token = tool_parameters.get('Authorization', '') + if not access_token: + return self.create_text_message('Invalid parameter access_token') + + app_token = tool_parameters.get('app_token', '') + if not app_token: + return self.create_text_message('Invalid parameter app_token') + + table_id = tool_parameters.get('table_id', '') + if not table_id: + return self.create_text_message('Invalid parameter table_id') + + record_id = tool_parameters.get('record_id', '') + if not record_id: + return self.create_text_message('Invalid parameter record_id') + + fields = tool_parameters.get('fields', '') + if not fields: + return self.create_text_message('Invalid parameter fields') + + headers = { + 'Content-Type': 'application/json', + 'Authorization': f"Bearer {access_token}", + } + + params = {} + payload = { + "fields": json.loads(fields) + } + + try: + res = httpx.put(url.format(app_token=app_token, table_id=table_id, record_id=record_id), headers=headers, + params=params, json=payload, timeout=30) + res_json = res.json() + if res.is_success: + return self.create_text_message(text=json.dumps(res_json)) + else: + return self.create_text_message( + f"Failed to update base record, status code: {res.status_code}, response: {res.text}") + except Exception as e: + return self.create_text_message("Failed to update base record. {}".format(e)) diff --git a/api/core/tools/provider/builtin/feishu_base/tools/update_base_record.yaml b/api/core/tools/provider/builtin/feishu_base/tools/update_base_record.yaml new file mode 100644 index 00000000000000..788798c4b3b40e --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_base/tools/update_base_record.yaml @@ -0,0 +1,78 @@ +identity: + name: update_base_record + author: Doug Lea + label: + en_US: Update Base Record + zh_Hans: 更新多维表格数据表中的一条记录 +description: + human: + en_US: Update base record + zh_Hans: | + 更新多维表格数据表中的一条记录，详细请参考：https://open.larkoffice.com/document/server-docs/docs/bitable-v1/app-table-record/update + llm: Update a record in a multidimensional table data table +parameters: + - name: Authorization + type: string + required: true + label: + en_US: token + zh_Hans: 凭证 + human_description: + en_US: API access token parameter, tenant_access_token or user_access_token + zh_Hans: API 的访问凭证参数，tenant_access_token 或 user_access_token + llm_description: API access token parameter, tenant_access_token or user_access_token + form: llm + + - name: app_token + type: string + required: true + label: + en_US: app_token + zh_Hans: 多维表格 + human_description: + en_US: bitable app token + zh_Hans: 多维表格的唯一标识符 app_token + llm_description: bitable app token + form: llm + + - name: table_id + type: string + required: true + label: + en_US: table_id + zh_Hans: 多维表格的数据表 + human_description: + en_US: bitable table id + zh_Hans: 多维表格数据表的唯一标识符 table_id + llm_description: bitable table id + form: llm + + - name: record_id + type: string + required: true + label: + en_US: record_id + zh_Hans: 单条记录的 id + human_description: + en_US: The id of a single record + zh_Hans: 单条记录的 id + llm_description: The id of a single record + form: llm + + - name: fields + type: string + required: true + label: + en_US: fields + zh_Hans: 数据表的列字段内容 + human_description: + en_US: The fields of a multidimensional table data table, that is, the columns of the data table. + zh_Hans: | + 要更新一行多维表格记录，字段结构拼接如下：{"多行文本":"多行文本内容","单选":"选项1","多选":["选项1","选项2"],"复选框":true,"人员":[{"id":"ou_2910013f1e6456f16a0ce75ede950a0a"}],"群组":[{"id":"oc_cd07f55f14d6f4a4f1b51504e7e97f48"}],"电话号码":"13026162666"} + 当前接口支持的字段类型为：多行文本、单选、条码、多选、日期、人员、附件、复选框、超链接、数字、单向关联、双向关联、电话号码、地理位置。 + 不同类型字段的数据结构请参考数据结构概述：https://open.larkoffice.com/document/server-docs/docs/bitable-v1/bitable-structure + llm_description: | + 要更新一行多维表格记录，字段结构拼接如下：{"多行文本":"多行文本内容","单选":"选项1","多选":["选项1","选项2"],"复选框":true,"人员":[{"id":"ou_2910013f1e6456f16a0ce75ede950a0a"}],"群组":[{"id":"oc_cd07f55f14d6f4a4f1b51504e7e97f48"}],"电话号码":"13026162666"} + 当前接口支持的字段类型为：多行文本、单选、条码、多选、日期、人员、附件、复选框、超链接、数字、单向关联、双向关联、电话号码、地理位置。 + 不同类型字段的数据结构请参考数据结构概述：https://open.larkoffice.com/document/server-docs/docs/bitable-v1/bitable-structure + form: llm diff --git a/api/core/tools/provider/builtin/jina/jina.py b/api/core/tools/provider/builtin/jina/jina.py index b1a8d6213800a9..12e5058cdc92f0 100644 --- a/api/core/tools/provider/builtin/jina/jina.py +++ b/api/core/tools/provider/builtin/jina/jina.py @@ -1,14 +1,32 @@ +import json from typing import Any from core.tools.entities.values import ToolLabelEnum from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.jina.tools.jina_reader import JinaReaderTool from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController class GoogleProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: dict[str, Any]) -> None: try: - pass + if credentials['api_key'] is None: + credentials['api_key'] = '' + else: + result = JinaReaderTool().fork_tool_runtime( + runtime={ + "credentials": credentials, + } + ).invoke( + user_id='', + tool_parameters={ + "url": "https://example.com", + }, + )[0] + + message = json.loads(result.message) + if message['code'] != 200: + raise ToolProviderCredentialValidationError(message['message']) except Exception as e: raise ToolProviderCredentialValidationError(str(e)) diff --git a/api/core/tools/provider/builtin/jina/jina.yaml b/api/core/tools/provider/builtin/jina/jina.yaml index 67ad32a47a1ec7..06f23382d92a3a 100644 --- a/api/core/tools/provider/builtin/jina/jina.yaml +++ b/api/core/tools/provider/builtin/jina/jina.yaml @@ -14,3 +14,19 @@ identity: - search - productivity credentials_for_provider: + api_key: + type: secret-input + required: false + label: + en_US: API Key (leave empty if you don't have one) + zh_Hans: API 密钥（可留空） + pt_BR: Chave API (deixe vazio se você não tiver uma) + placeholder: + en_US: Please enter your Jina API key + zh_Hans: 请输入你的 Jina API 密钥 + pt_BR: Por favor, insira sua chave de API do Jina + help: + en_US: Get your Jina API key from Jina (optional, but you can get a higher rate) + zh_Hans: 从 Jina 获取您的 Jina API 密钥（非必须，能得到更高的速率） + pt_BR: Obtenha sua chave de API do Jina na Jina (opcional, mas você pode obter uma taxa mais alta) + url: https://jina.ai diff --git a/api/core/tools/provider/builtin/jina/tools/jina_reader.py b/api/core/tools/provider/builtin/jina/tools/jina_reader.py index beb05717ea11f7..b0bd4788466132 100644 --- a/api/core/tools/provider/builtin/jina/tools/jina_reader.py +++ b/api/core/tools/provider/builtin/jina/tools/jina_reader.py @@ -23,14 +23,24 @@ def _invoke(self, 'Accept': 'application/json' } + if 'api_key' in self.runtime.credentials and self.runtime.credentials.get('api_key'): + headers['Authorization'] = "Bearer " + self.runtime.credentials.get('api_key') + target_selector = tool_parameters.get('target_selector', None) - if target_selector is not None: + if target_selector is not None and target_selector != '': headers['X-Target-Selector'] = target_selector wait_for_selector = tool_parameters.get('wait_for_selector', None) - if wait_for_selector is not None: + if wait_for_selector is not None and wait_for_selector != '': headers['X-Wait-For-Selector'] = wait_for_selector + proxy_server = tool_parameters.get('proxy_server', None) + if proxy_server is not None and proxy_server != '': + headers['X-Proxy-Url'] = proxy_server + + if tool_parameters.get('no_cache', False): + headers['X-No-Cache'] = 'true' + response = ssrf_proxy.get( str(URL(self._jina_reader_endpoint + url)), headers=headers, diff --git a/api/core/tools/provider/builtin/jina/tools/jina_reader.yaml b/api/core/tools/provider/builtin/jina/tools/jina_reader.yaml index 73cacb7fde1a09..703fa3d389ad75 100644 --- a/api/core/tools/provider/builtin/jina/tools/jina_reader.yaml +++ b/api/core/tools/provider/builtin/jina/tools/jina_reader.yaml @@ -51,6 +51,33 @@ parameters: pt_BR: css selector for waiting for specific elements llm_description: css selector of the target element to wait for form: form + - name: proxy_server + type: string + required: false + label: + en_US: Proxy server + zh_Hans: 代理服务器 + pt_BR: Servidor de proxy + human_description: + en_US: Use proxy to access URLs + zh_Hans: 利用代理访问 URL + pt_BR: Use proxy to access URLs + llm_description: Use proxy to access URLs + form: form + - name: no_cache + type: boolean + required: false + default: false + label: + en_US: Bypass the Cache + zh_Hans: 绕过缓存 + pt_BR: Ignorar o cache + human_description: + en_US: Bypass the Cache + zh_Hans: 是否绕过缓存 + pt_BR: Ignorar o cache + llm_description: bypass the cache + form: form - name: summary type: boolean required: false diff --git a/api/core/tools/provider/builtin/jina/tools/jina_search.py b/api/core/tools/provider/builtin/jina/tools/jina_search.py index cfe36e6a3ce1c6..c13f58d0cd163c 100644 --- a/api/core/tools/provider/builtin/jina/tools/jina_search.py +++ b/api/core/tools/provider/builtin/jina/tools/jina_search.py @@ -21,6 +21,16 @@ def _invoke( 'Accept': 'application/json' } + if 'api_key' in self.runtime.credentials and self.runtime.credentials.get('api_key'): + headers['Authorization'] = "Bearer " + self.runtime.credentials.get('api_key') + + proxy_server = tool_parameters.get('proxy_server', None) + if proxy_server is not None and proxy_server != '': + headers['X-Proxy-Url'] = proxy_server + + if tool_parameters.get('no_cache', False): + headers['X-No-Cache'] = 'true' + response = ssrf_proxy.get( str(URL(self._jina_search_endpoint + query)), headers=headers, diff --git a/api/core/tools/provider/builtin/jina/tools/jina_search.yaml b/api/core/tools/provider/builtin/jina/tools/jina_search.yaml index 5ad70c03f3bbc1..f3b6c0737a9699 100644 --- a/api/core/tools/provider/builtin/jina/tools/jina_search.yaml +++ b/api/core/tools/provider/builtin/jina/tools/jina_search.yaml @@ -8,6 +8,7 @@ identity: description: human: en_US: Search on the web and get the top 5 results. Useful for grounding using information from the web. + zh_Hans: 在网络上搜索返回前 5 个结果。 llm: A tool for searching results on the web for grounding. Input should be a simple question. parameters: - name: query @@ -15,7 +16,36 @@ parameters: required: true label: en_US: Question (Query) + zh_Hans: 信息查询 human_description: en_US: used to find information on the web + zh_Hans: 在网络上搜索信息 llm_description: simple question to ask on the web form: llm + - name: proxy_server + type: string + required: false + label: + en_US: Proxy server + zh_Hans: 代理服务器 + pt_BR: Servidor de proxy + human_description: + en_US: Use proxy to access URLs + zh_Hans: 利用代理访问 URL + pt_BR: Use proxy to access URLs + llm_description: Use proxy to access URLs + form: form + - name: no_cache + type: boolean + required: false + default: false + label: + en_US: Bypass the Cache + zh_Hans: 绕过缓存 + pt_BR: Ignorar o cache + human_description: + en_US: Bypass the Cache + zh_Hans: 是否绕过缓存 + pt_BR: Ignorar o cache + llm_description: bypass the cache + form: form diff --git a/api/core/tools/provider/workflow_tool_provider.py b/api/core/tools/provider/workflow_tool_provider.py index f98ad0f26a2414..f7911fea1db18d 100644 --- a/api/core/tools/provider/workflow_tool_provider.py +++ b/api/core/tools/provider/workflow_tool_provider.py @@ -2,7 +2,7 @@ from core.app.app_config.entities import VariableEntity from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager -from core.model_runtime.entities.common_entities import I18nObject +from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_entities import ( ToolDescription, ToolIdentity, diff --git a/api/core/tools/tool/tool.py b/api/core/tools/tool/tool.py index 37432a6116d1be..b48daf0c52b381 100644 --- a/api/core/tools/tool/tool.py +++ b/api/core/tools/tool/tool.py @@ -237,10 +237,10 @@ def _transform_tool_parameters_type(self, tool_parameters: dict[str, Any]) -> di """ # Temp fix for the issue that the tool parameters will be converted to empty while validating the credentials result = deepcopy(tool_parameters) - for parameter in self.parameters: + for parameter in self.parameters or []: if parameter.name in tool_parameters: result[parameter.name] = ToolParameterConverter.cast_parameter_by_type(tool_parameters[parameter.name], parameter.type) - + return result @abstractmethod diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index 10002216f1f731..74a6c5b9de62c3 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -42,7 +42,10 @@ def is_file(self) -> bool: return any(v in content_type for v in file_content_types) def get_content_type(self) -> str: - return self.headers.get('content-type') + if 'content-type' in self.headers: + return self.headers.get('content-type') + else: + return self.headers.get('Content-Type') or "" def extract_file(self) -> tuple[str, bytes]: """ diff --git a/api/events/event_handlers/create_site_record_when_app_created.py b/api/events/event_handlers/create_site_record_when_app_created.py index 25fba591d02af8..f0eb7159b631e0 100644 --- a/api/events/event_handlers/create_site_record_when_app_created.py +++ b/api/events/event_handlers/create_site_record_when_app_created.py @@ -11,6 +11,8 @@ def handle(sender, **kwargs): site = Site( app_id=app.id, title=app.name, + icon = app.icon, + icon_background = app.icon_background, default_language=account.interface_language, customize_token_strategy='not_allow', code=Site.generate_code(16) diff --git a/api/extensions/storage/s3_storage.py b/api/extensions/storage/s3_storage.py index 8aae68a740014a..787596fa791d4a 100644 --- a/api/extensions/storage/s3_storage.py +++ b/api/extensions/storage/s3_storage.py @@ -16,14 +16,18 @@ def __init__(self, app: Flask): super().__init__(app) app_config = self.app.config self.bucket_name = app_config.get('S3_BUCKET_NAME') - self.client = boto3.client( - 's3', - aws_secret_access_key=app_config.get('S3_SECRET_KEY'), - aws_access_key_id=app_config.get('S3_ACCESS_KEY'), - endpoint_url=app_config.get('S3_ENDPOINT'), - region_name=app_config.get('S3_REGION'), - config=Config(s3={'addressing_style': app_config.get('S3_ADDRESS_STYLE')}) - ) + if app_config.get('S3_USE_AWS_MANAGED_IAM'): + session = boto3.Session() + self.client = session.client('s3') + else: + self.client = boto3.client( + 's3', + aws_secret_access_key=app_config.get('S3_SECRET_KEY'), + aws_access_key_id=app_config.get('S3_ACCESS_KEY'), + endpoint_url=app_config.get('S3_ENDPOINT'), + region_name=app_config.get('S3_REGION'), + config=Config(s3={'addressing_style': app_config.get('S3_ADDRESS_STYLE')}) + ) def save(self, filename, data): self.client.put_object(Bucket=self.bucket_name, Key=filename, Body=data) diff --git a/api/fields/member_fields.py b/api/fields/member_fields.py index 79164b3848536d..d061b59c347022 100644 --- a/api/fields/member_fields.py +++ b/api/fields/member_fields.py @@ -28,6 +28,7 @@ 'avatar': fields.String, 'email': fields.String, 'last_login_at': TimestampField, + 'last_active_at': TimestampField, 'created_at': TimestampField, 'role': fields.String, 'status': fields.String, diff --git a/api/models/account.py b/api/models/account.py index 3d5e9557324213..4911757b0759c3 100644 --- a/api/models/account.py +++ b/api/models/account.py @@ -106,6 +106,9 @@ def get_integrates(self) -> list[db.Model]: def is_admin_or_owner(self): return TenantAccountRole.is_privileged_role(self._current_tenant.current_role) + @property + def is_editor(self): + return TenantAccountRole.is_editing_role(self._current_tenant.current_role) class TenantStatus(str, enum.Enum): NORMAL = 'normal' @@ -115,11 +118,24 @@ class TenantStatus(str, enum.Enum): class TenantAccountRole(str, enum.Enum): OWNER = 'owner' ADMIN = 'admin' + EDITOR = 'editor' NORMAL = 'normal' + @staticmethod + def is_valid_role(role: str) -> bool: + return role and role in {TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR, TenantAccountRole.NORMAL} + @staticmethod def is_privileged_role(role: str) -> bool: - return role and role in {TenantAccountRole.ADMIN, TenantAccountRole.OWNER} + return role and role in {TenantAccountRole.OWNER, TenantAccountRole.ADMIN} + + @staticmethod + def is_non_owner_role(role: str) -> bool: + return role and role in {TenantAccountRole.ADMIN, TenantAccountRole.EDITOR, TenantAccountRole.NORMAL} + + @staticmethod + def is_editing_role(role: str) -> bool: + return role and role in {TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR} class Tenant(db.Model): diff --git a/api/poetry.lock b/api/poetry.lock index 4114166bd24d17..6d716a15c2beba 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -377,17 +377,17 @@ tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "p [[package]] name = "authlib" -version = "1.2.0" +version = "1.3.1" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = false -python-versions = "*" +python-versions = ">=3.8" files = [ - {file = "Authlib-1.2.0-py2.py3-none-any.whl", hash = "sha256:4ddf4fd6cfa75c9a460b361d4bd9dac71ffda0be879dbe4292a02e92349ad55a"}, - {file = "Authlib-1.2.0.tar.gz", hash = "sha256:4fa3e80883a5915ef9f5bc28630564bc4ed5b5af39812a3ff130ec76bd631e9d"}, + {file = "Authlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:d35800b973099bbadc49b42b256ecb80041ad56b7fe1216a362c7943c088f377"}, + {file = "authlib-1.3.1.tar.gz", hash = "sha256:7ae843f03c06c5c0debd63c9db91f9fda64fa62a42a77419fa15fbb7e7a58917"}, ] [package.dependencies] -cryptography = ">=3.2" +cryptography = "*" [[package]] name = "azure-core" @@ -410,20 +410,20 @@ aio = ["aiohttp (>=3.0)"] [[package]] name = "azure-identity" -version = "1.15.0" +version = "1.16.1" description = "Microsoft Azure Identity Library for Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "azure-identity-1.15.0.tar.gz", hash = "sha256:4c28fc246b7f9265610eb5261d65931183d019a23d4b0e99357facb2e6c227c8"}, - {file = "azure_identity-1.15.0-py3-none-any.whl", hash = "sha256:a14b1f01c7036f11f148f22cd8c16e05035293d714458d6b44ddf534d93eb912"}, + {file = "azure-identity-1.16.1.tar.gz", hash = "sha256:6d93f04468f240d59246d8afde3091494a5040d4f141cad0f49fc0c399d0d91e"}, + {file = "azure_identity-1.16.1-py3-none-any.whl", hash = "sha256:8fb07c25642cd4ac422559a8b50d3e77f73dcc2bbfaba419d06d6c9d7cff6726"}, ] [package.dependencies] -azure-core = ">=1.23.0,<2.0.0" +azure-core = ">=1.23.0" cryptography = ">=2.5" -msal = ">=1.24.0,<2.0.0" -msal-extensions = ">=0.3.0,<2.0.0" +msal = ">=1.24.0" +msal-extensions = ">=0.3.0" [[package]] name = "azure-storage-blob" @@ -798,23 +798,6 @@ typing = ["build[uv]", "importlib-metadata (>=5.1)", "mypy (>=1.9.0,<1.10.0)", " uv = ["uv (>=0.1.18)"] virtualenv = ["virtualenv (>=20.0.35)"] -[[package]] -name = "bump-pydantic" -version = "0.8.0" -description = "Convert Pydantic from V1 to V2 ♻" -optional = false -python-versions = ">=3.8" -files = [ - {file = "bump_pydantic-0.8.0-py3-none-any.whl", hash = "sha256:6cbb4deb5869a69baa5a477f28f3e2d8fb09b687e114c018bd54470590ae7bf7"}, - {file = "bump_pydantic-0.8.0.tar.gz", hash = "sha256:6092e61930e85619e74eeb04131b4387feda16f02d8bb2e3cf9507fa492c69e9"}, -] - -[package.dependencies] -libcst = ">=0.4.2" -rich = "*" -typer = ">=0.7.0" -typing-extensions = "*" - [[package]] name = "cachetools" version = "5.3.3" @@ -1456,6 +1439,23 @@ mypy = ["contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.8.0)", "types-Pill test = ["Pillow", "contourpy[test-no-images]", "matplotlib"] test-no-images = ["pytest", "pytest-cov", "pytest-xdist", "wurlitzer"] +[[package]] +name = "cos-python-sdk-v5" +version = "1.9.29" +description = "cos-python-sdk-v5" +optional = false +python-versions = "*" +files = [ + {file = "cos-python-sdk-v5-1.9.29.tar.gz", hash = "sha256:1bb07022368d178e7a50a3cc42e0d6cbf4b0bef2af12a3bb8436904339cdec8e"}, +] + +[package.dependencies] +crcmod = "*" +pycryptodome = "*" +requests = ">=2.8" +six = "*" +xmltodict = "*" + [[package]] name = "coverage" version = "7.2.7" @@ -3884,46 +3884,6 @@ files = [ [package.dependencies] six = "*" -[[package]] -name = "libcst" -version = "1.4.0" -description = "A concrete syntax tree with AST-like properties for Python 3.0 through 3.12 programs." -optional = false -python-versions = ">=3.9" -files = [ - {file = "libcst-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:279b54568ea1f25add50ea4ba3d76d4f5835500c82f24d54daae4c5095b986aa"}, - {file = "libcst-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3401dae41fe24565387a65baee3887e31a44e3e58066b0250bc3f3ccf85b1b5a"}, - {file = "libcst-1.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1989fa12d3cd79118ebd29ebe2a6976d23d509b1a4226bc3d66fcb7cb50bd5d"}, - {file = "libcst-1.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:addc6d585141a7677591868886f6bda0577529401a59d210aa8112114340e129"}, - {file = "libcst-1.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:17d71001cb25e94cfe8c3d997095741a8c4aa7a6d234c0f972bc42818c88dfaf"}, - {file = "libcst-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:2d47de16d105e7dd5f4e01a428d9f4dc1e71efd74f79766daf54528ce37f23c3"}, - {file = "libcst-1.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e6227562fc5c9c1efd15dfe90b0971ae254461b8b6b23c1b617139b6003de1c1"}, - {file = "libcst-1.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3399e6c95df89921511b44d8c5bf6a75bcbc2d51f1f6429763609ba005c10f6b"}, - {file = "libcst-1.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48601e3e590e2d6a7ab8c019cf3937c70511a78d778ab3333764531253acdb33"}, - {file = "libcst-1.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f42797309bb725f0f000510d5463175ccd7155395f09b5e7723971b0007a976d"}, - {file = "libcst-1.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb4e42ea107a37bff7f9fdbee9532d39f9ea77b89caa5c5112b37057b12e0838"}, - {file = "libcst-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:9d0cc3c5a2a51fa7e1d579a828c0a2e46b2170024fd8b1a0691c8a52f3abb2d9"}, - {file = "libcst-1.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7ece51d935bc9bf60b528473d2e5cc67cbb88e2f8146297e40ee2c7d80be6f13"}, - {file = "libcst-1.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:81653dea1cdfa4c6520a7c5ffb95fa4d220cbd242e446c7a06d42d8636bfcbba"}, - {file = "libcst-1.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6abce0e66bba2babfadc20530fd3688f672d565674336595b4623cd800b91ef"}, - {file = "libcst-1.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5da9d7dc83801aba3b8d911f82dc1a375db0d508318bad79d9fb245374afe068"}, - {file = "libcst-1.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c54aa66c86d8ece9c93156a2cf5ca512b0dce40142fe9e072c86af2bf892411"}, - {file = "libcst-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:62e2682ee1567b6a89c91853865372bf34f178bfd237853d84df2b87b446e654"}, - {file = "libcst-1.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b8ecdba8934632b4dadacb666cd3816627a6ead831b806336972ccc4ba7ca0e9"}, - {file = "libcst-1.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8e54c777b8d27339b70f304d16fc8bc8674ef1bd34ed05ea874bf4921eb5a313"}, - {file = "libcst-1.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:061d6855ef30efe38b8a292b7e5d57c8e820e71fc9ec9846678b60a934b53bbb"}, - {file = "libcst-1.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb0abf627ee14903d05d0ad9b2c6865f1b21eb4081e2c7bea1033f85db2b8bae"}, - {file = "libcst-1.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d024f44059a853b4b852cfc04fec33e346659d851371e46fc8e7c19de24d3da9"}, - {file = "libcst-1.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:3c6a8faab9da48c5b371557d0999b4ca51f4f2cbd37ee8c2c4df0ac01c781465"}, - {file = "libcst-1.4.0.tar.gz", hash = "sha256:449e0b16604f054fa7f27c3ffe86ea7ef6c409836fe68fe4e752a1894175db00"}, -] - -[package.dependencies] -pyyaml = ">=5.2" - -[package.extras] -dev = ["Sphinx (>=5.1.1)", "black (==23.12.1)", "build (>=0.10.0)", "coverage (>=4.5.4)", "fixit (==2.1.0)", "flake8 (==7.0.0)", "hypothesis (>=4.36.0)", "hypothesmith (>=0.0.4)", "jinja2 (==3.1.4)", "jupyter (>=1.0.0)", "maturin (>=0.8.3,<1.6)", "nbsphinx (>=0.4.2)", "prompt-toolkit (>=2.0.9)", "pyre-check (==0.9.18)", "setuptools-rust (>=1.5.2)", "setuptools-scm (>=6.0.1)", "slotscheck (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "ufmt (==2.6.0)", "usort (==1.0.8.post1)"] - [[package]] name = "llvmlite" version = "0.42.0" @@ -7468,6 +7428,21 @@ files = [ [package.extras] widechars = ["wcwidth"] +[[package]] +name = "tcvectordb" +version = "1.3.2" +description = "Tencent VectorDB Python SDK" +optional = false +python-versions = ">=3" +files = [ + {file = "tcvectordb-1.3.2-py3-none-any.whl", hash = "sha256:c4b6922d5df4cf14fcd3e61220d9374d1d53ec7270c254216ae35f8a752908f3"}, + {file = "tcvectordb-1.3.2.tar.gz", hash = "sha256:2772f5871a69744ffc7c970b321312d626078533a721de3c744059a81aab419e"}, +] + +[package.dependencies] +cos-python-sdk-v5 = ">=1.9.26" +requests = "*" + [[package]] name = "tenacity" version = "8.3.0" @@ -8698,6 +8673,17 @@ files = [ {file = "XlsxWriter-3.2.0.tar.gz", hash = "sha256:9977d0c661a72866a61f9f7a809e25ebbb0fb7036baa3b9fe74afcfca6b3cb8c"}, ] +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +optional = false +python-versions = ">=3.4" +files = [ + {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, + {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, +] + [[package]] name = "yarl" version = "1.9.4" @@ -8935,4 +8921,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "6845b0f3a5b5be84d32a9a79f23d389d1502cc70f5530becb313fa8b2268448f" +content-hash = "e967aa4b61dc7c40f2f50eb325038da1dc0ff633d8f778e7a7560bdabce744dc" diff --git a/api/pyproject.toml b/api/pyproject.toml index e43aaef62b6292..9f2786d40641ba 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -108,7 +108,7 @@ tiktoken = "~0.7.0" psycopg2-binary = "~2.9.6" pycryptodome = "3.19.1" python-dotenv = "1.0.0" -authlib = "1.2.0" +authlib = "1.3.1" boto3 = "1.28.17" cachetools = "~5.3.0" weaviate-client = "~3.21.0" @@ -167,12 +167,11 @@ yarl = "~1.9.4" twilio = "~9.0.4" qrcode = "~7.4.2" azure-storage-blob = "12.13.0" -azure-identity = "1.15.0" +azure-identity = "1.16.1" lxml = "5.1.0" xlrd = "~2.0.1" -pydantic = "~2.7.3" -pydantic_extra_types = "~2.8.0" -bump-pydantic = "~0.8.0" +pydantic = "~2.7.4" +pydantic_extra_types = "~2.8.1" pgvecto-rs = "0.1.4" firecrawl-py = "0.0.5" oss2 = "2.18.5" @@ -183,6 +182,7 @@ google-cloud-aiplatform = "1.49.0" vanna = {version = "0.5.5", extras = ["postgres", "mysql", "clickhouse", "duckdb"]} kaleido = "0.2.1" tencentcloud-sdk-python-hunyuan = "~3.0.1158" +tcvectordb = "1.3.2" chromadb = "~0.5.0" [tool.poetry.group.dev] diff --git a/api/requirements.txt b/api/requirements.txt index c0f6b2a8249887..a6a1d8c5cedc33 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -14,7 +14,7 @@ tiktoken~=0.7.0 psycopg2-binary~=2.9.6 pycryptodome==3.19.1 python-dotenv==1.0.0 -Authlib==1.2.0 +Authlib==1.3.1 boto3==1.34.123 cachetools~=5.3.0 weaviate-client~=3.21.0 @@ -73,12 +73,12 @@ yarl~=1.9.4 twilio~=9.0.4 qrcode~=7.4.2 azure-storage-blob==12.13.0 -azure-identity==1.15.0 +azure-identity==1.16.1 lxml==5.1.0 -pydantic~=2.7.3 -pydantic_extra_types~=2.8.0 -bump-pydantic~=0.8.0 +pydantic~=2.7.4 +pydantic_extra_types~=2.8.1 pgvecto-rs==0.1.4 +tcvectordb==1.3.2 firecrawl-py==0.0.5 oss2==2.18.5 pgvector==0.2.5 diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index c1ac8400064b58..b3cf15811b6d92 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -33,7 +33,7 @@ from models.model import UploadFile from models.source import DataSourceOauthBinding from services.errors.account import NoPermissionError -from services.errors.dataset import DatasetNameDuplicateError +from services.errors.dataset import DatasetInUseError, DatasetNameDuplicateError from services.errors.document import DocumentIndexingError from services.errors.file import FileNotExistsError from services.feature_service import FeatureModel, FeatureService @@ -233,7 +233,9 @@ def update_dataset(dataset_id, data, user): @staticmethod def delete_dataset(dataset_id, user): - # todo: cannot delete dataset if it is being processed + count = AppDatasetJoin.query.filter_by(dataset_id=dataset_id).count() + if count > 0: + raise DatasetInUseError() dataset = DatasetService.get_dataset(dataset_id) diff --git a/api/services/errors/dataset.py b/api/services/errors/dataset.py index 254a70ffe313dd..d36cd1111c78f8 100644 --- a/api/services/errors/dataset.py +++ b/api/services/errors/dataset.py @@ -3,3 +3,7 @@ class DatasetNameDuplicateError(BaseServiceError): pass + + +class DatasetInUseError(BaseServiceError): + pass diff --git a/api/tests/integration_tests/model_runtime/novita/__init__.py b/api/tests/integration_tests/model_runtime/novita/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/tests/integration_tests/model_runtime/novita/test_llm.py b/api/tests/integration_tests/model_runtime/novita/test_llm.py new file mode 100644 index 00000000000000..4ebc68493f26d5 --- /dev/null +++ b/api/tests/integration_tests/model_runtime/novita/test_llm.py @@ -0,0 +1,123 @@ +import os +from collections.abc import Generator + +import pytest + +from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta +from core.model_runtime.entities.message_entities import ( + AssistantPromptMessage, + PromptMessageTool, + SystemPromptMessage, + UserPromptMessage, +) +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.novita.llm.llm import NovitaLargeLanguageModel + + +def test_validate_credentials(): + model = NovitaLargeLanguageModel() + + with pytest.raises(CredentialsValidateFailedError): + model.validate_credentials( + model='meta-llama/llama-3-8b-instruct', + credentials={ + 'api_key': 'invalid_key', + 'mode': 'chat' + } + ) + + model.validate_credentials( + model='meta-llama/llama-3-8b-instruct', + credentials={ + 'api_key': os.environ.get('NOVITA_API_KEY'), + 'mode': 'chat' + } + ) + + +def test_invoke_model(): + model = NovitaLargeLanguageModel() + + response = model.invoke( + model='meta-llama/llama-3-8b-instruct', + credentials={ + 'api_key': os.environ.get('NOVITA_API_KEY'), + 'mode': 'completion' + }, + prompt_messages=[ + SystemPromptMessage( + content='You are a helpful AI assistant.', + ), + UserPromptMessage( + content='Who are you?' + ) + ], + model_parameters={ + 'temperature': 1.0, + 'top_p': 0.5, + 'max_tokens': 10, + }, + stop=['How'], + stream=False, + user="novita" + ) + + assert isinstance(response, LLMResult) + assert len(response.message.content) > 0 + + +def test_invoke_stream_model(): + model = NovitaLargeLanguageModel() + + response = model.invoke( + model='meta-llama/llama-3-8b-instruct', + credentials={ + 'api_key': os.environ.get('NOVITA_API_KEY'), + 'mode': 'chat' + }, + prompt_messages=[ + SystemPromptMessage( + content='You are a helpful AI assistant.', + ), + UserPromptMessage( + content='Who are you?' + ) + ], + model_parameters={ + 'temperature': 1.0, + 'top_k': 2, + 'top_p': 0.5, + 'max_tokens': 100 + }, + stream=True, + user="novita" + ) + + assert isinstance(response, Generator) + + for chunk in response: + assert isinstance(chunk, LLMResultChunk) + assert isinstance(chunk.delta, LLMResultChunkDelta) + assert isinstance(chunk.delta.message, AssistantPromptMessage) + + +def test_get_num_tokens(): + model = NovitaLargeLanguageModel() + + num_tokens = model.get_num_tokens( + model='meta-llama/llama-3-8b-instruct', + credentials={ + 'api_key': os.environ.get('NOVITA_API_KEY'), + }, + prompt_messages=[ + SystemPromptMessage( + content='You are a helpful AI assistant.', + ), + UserPromptMessage( + content='Hello World!' + ) + ] + ) + + assert isinstance(num_tokens, int) + assert num_tokens == 21 diff --git a/api/tests/integration_tests/model_runtime/novita/test_provider.py b/api/tests/integration_tests/model_runtime/novita/test_provider.py new file mode 100644 index 00000000000000..bb3f19dc851ea5 --- /dev/null +++ b/api/tests/integration_tests/model_runtime/novita/test_provider.py @@ -0,0 +1,21 @@ +import os + +import pytest + +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.novita.novita import NovitaProvider + + +def test_validate_provider_credentials(): + provider = NovitaProvider() + + with pytest.raises(CredentialsValidateFailedError): + provider.validate_provider_credentials( + credentials={} + ) + + provider.validate_provider_credentials( + credentials={ + 'api_key': os.environ.get('NOVITA_API_KEY'), + } + ) diff --git a/api/tests/integration_tests/vdb/__mock/__init__.py b/api/tests/integration_tests/vdb/__mock/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/tests/integration_tests/vdb/__mock/tcvectordb.py b/api/tests/integration_tests/vdb/__mock/tcvectordb.py new file mode 100644 index 00000000000000..f8165cba9468aa --- /dev/null +++ b/api/tests/integration_tests/vdb/__mock/tcvectordb.py @@ -0,0 +1,132 @@ +import os +from typing import Optional + +import pytest +from _pytest.monkeypatch import MonkeyPatch +from requests.adapters import HTTPAdapter +from tcvectordb import VectorDBClient +from tcvectordb.model.database import Collection, Database +from tcvectordb.model.document import Document, Filter +from tcvectordb.model.enum import ReadConsistency +from tcvectordb.model.index import Index +from xinference_client.types import Embedding + + +class MockTcvectordbClass: + + def VectorDBClient(self, url=None, username='', key='', + read_consistency: ReadConsistency = ReadConsistency.EVENTUAL_CONSISTENCY, + timeout=5, + adapter: HTTPAdapter = None): + self._conn = None + self._read_consistency = read_consistency + + def list_databases(self) -> list[Database]: + return [ + Database( + conn=self._conn, + read_consistency=self._read_consistency, + name='dify', + )] + + def list_collections(self, timeout: Optional[float] = None) -> list[Collection]: + return [] + + def drop_collection(self, name: str, timeout: Optional[float] = None): + return { + "code": 0, + "msg": "operation success" + } + + def create_collection( + self, + name: str, + shard: int, + replicas: int, + description: str, + index: Index, + embedding: Embedding = None, + timeout: float = None, + ) -> Collection: + return Collection(self, name, shard, replicas, description, index, embedding=embedding, + read_consistency=self._read_consistency, timeout=timeout) + + def describe_collection(self, name: str, timeout: Optional[float] = None) -> Collection: + collection = Collection( + self, + name, + shard=1, + replicas=2, + description=name, + timeout=timeout + ) + return collection + + def collection_upsert( + self, + documents: list[Document], + timeout: Optional[float] = None, + build_index: bool = True, + **kwargs + ): + return { + "code": 0, + "msg": "operation success" + } + + def collection_search( + self, + vectors: list[list[float]], + filter: Filter = None, + params=None, + retrieve_vector: bool = False, + limit: int = 10, + output_fields: Optional[list[str]] = None, + timeout: Optional[float] = None, + ) -> list[list[dict]]: + return [[{'metadata': '{"doc_id":"foo1"}', 'text': 'text', 'doc_id': 'foo1', 'score': 0.1}]] + + def collection_query( + self, + document_ids: Optional[list] = None, + retrieve_vector: bool = False, + limit: Optional[int] = None, + offset: Optional[int] = None, + filter: Optional[Filter] = None, + output_fields: Optional[list[str]] = None, + timeout: Optional[float] = None, + ) -> list[dict]: + return [{'metadata': '{"doc_id":"foo1"}', 'text': 'text', 'doc_id': 'foo1', 'score': 0.1}] + + def collection_delete( + self, + document_ids: list[str] = None, + filter: Filter = None, + timeout: float = None, + ): + return { + "code": 0, + "msg": "operation success" + } + + +MOCK = os.getenv('MOCK_SWITCH', 'false').lower() == 'true' + +@pytest.fixture +def setup_tcvectordb_mock(request, monkeypatch: MonkeyPatch): + if MOCK: + monkeypatch.setattr(VectorDBClient, '__init__', MockTcvectordbClass.VectorDBClient) + monkeypatch.setattr(VectorDBClient, 'list_databases', MockTcvectordbClass.list_databases) + monkeypatch.setattr(Database, 'collection', MockTcvectordbClass.describe_collection) + monkeypatch.setattr(Database, 'list_collections', MockTcvectordbClass.list_collections) + monkeypatch.setattr(Database, 'drop_collection', MockTcvectordbClass.drop_collection) + monkeypatch.setattr(Database, 'create_collection', MockTcvectordbClass.create_collection) + monkeypatch.setattr(Collection, 'upsert', MockTcvectordbClass.collection_upsert) + monkeypatch.setattr(Collection, 'search', MockTcvectordbClass.collection_search) + monkeypatch.setattr(Collection, 'query', MockTcvectordbClass.collection_query) + monkeypatch.setattr(Collection, 'delete', MockTcvectordbClass.collection_delete) + + yield + + if MOCK: + monkeypatch.undo() diff --git a/api/tests/integration_tests/vdb/tcvectordb/__init__.py b/api/tests/integration_tests/vdb/tcvectordb/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/tests/integration_tests/vdb/tcvectordb/test_tencent.py b/api/tests/integration_tests/vdb/tcvectordb/test_tencent.py new file mode 100644 index 00000000000000..8937fe0ea16c50 --- /dev/null +++ b/api/tests/integration_tests/vdb/tcvectordb/test_tencent.py @@ -0,0 +1,35 @@ +from unittest.mock import MagicMock + +from core.rag.datasource.vdb.tencent.tencent_vector import TencentConfig, TencentVector +from tests.integration_tests.vdb.__mock.tcvectordb import setup_tcvectordb_mock +from tests.integration_tests.vdb.test_vector_store import AbstractVectorTest, get_example_text, setup_mock_redis + +mock_client = MagicMock() +mock_client.list_databases.return_value = [{"name": "test"}] + +class TencentVectorTest(AbstractVectorTest): + def __init__(self): + super().__init__() + self.vector = TencentVector("dify", TencentConfig( + url="http://127.0.0.1", + api_key="dify", + timeout=30, + username="dify", + database="dify", + shard=1, + replicas=2, + )) + + def search_by_vector(self): + hits_by_vector = self.vector.search_by_vector(query_vector=self.example_embedding) + assert len(hits_by_vector) == 1 + + def search_by_full_text(self): + hits_by_full_text = self.vector.search_by_full_text(query=get_example_text()) + assert len(hits_by_full_text) == 0 + +def test_tencent_vector(setup_mock_redis,setup_tcvectordb_mock): + TencentVectorTest().run_all_tests() + + + diff --git a/api/tests/unit_tests/models/test_account.py b/api/tests/unit_tests/models/test_account.py index ddb4e8cb751c46..006b99fb7d0935 100644 --- a/api/tests/unit_tests/models/test_account.py +++ b/api/tests/unit_tests/models/test_account.py @@ -4,9 +4,11 @@ def test_account_is_privileged_role() -> None: assert TenantAccountRole.ADMIN == 'admin' assert TenantAccountRole.OWNER == 'owner' + assert TenantAccountRole.EDITOR == 'editor' assert TenantAccountRole.NORMAL == 'normal' assert TenantAccountRole.is_privileged_role(TenantAccountRole.ADMIN) assert TenantAccountRole.is_privileged_role(TenantAccountRole.OWNER) assert not TenantAccountRole.is_privileged_role(TenantAccountRole.NORMAL) + assert not TenantAccountRole.is_privileged_role(TenantAccountRole.EDITOR) assert not TenantAccountRole.is_privileged_role('') diff --git a/dev/sync-poetry b/dev/sync-poetry new file mode 100755 index 00000000000000..2dd4dd4fc31090 --- /dev/null +++ b/dev/sync-poetry @@ -0,0 +1,15 @@ +#!/bin/bash + +# rely on `poetry` in path +if ! command -v poetry &> /dev/null; then + echo "Installing Poetry ..." + pip install poetry +fi + +# check poetry.lock in sync with pyproject.toml +poetry check -C api --lock +if [ $? -ne 0 ]; then + # update poetry.lock + # refreshing lockfile only without updating locked versions + poetry lock -C api --no-update +fi diff --git a/dev/update-poetry b/dev/update-poetry new file mode 100755 index 00000000000000..362a5895b1c342 --- /dev/null +++ b/dev/update-poetry @@ -0,0 +1,13 @@ +#!/bin/bash + +# rely on `poetry` in path +if ! command -v poetry &> /dev/null; then + echo "Installing Poetry ..." + pip install poetry +fi + +# refreshing lockfile, updating locked versions +poetry update -C api + +# check poetry.lock in sync with pyproject.toml +poetry check -C api --lock diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 5c67406bcb5eeb..45a97c9b74c8cf 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -81,6 +81,7 @@ services: # only available when STORAGE_TYPE is `local`. STORAGE_LOCAL_PATH: storage # The S3 storage configurations, only available when STORAGE_TYPE is `s3`. + S3_USE_AWS_MANAGED_IAM: 'false' S3_ENDPOINT: 'https://xxx.r2.cloudflarestorage.com' S3_BUCKET_NAME: 'difyai' S3_ACCESS_KEY: 'ak-difyai' @@ -236,6 +237,7 @@ services: STORAGE_TYPE: local STORAGE_LOCAL_PATH: storage # The S3 storage configurations, only available when STORAGE_TYPE is `s3`. + S3_USE_AWS_MANAGED_IAM: 'false' S3_ENDPOINT: 'https://xxx.r2.cloudflarestorage.com' S3_BUCKET_NAME: 'difyai' S3_ACCESS_KEY: 'ak-difyai' @@ -296,6 +298,14 @@ services: RELYT_USER: postgres RELYT_PASSWORD: difyai123456 RELYT_DATABASE: postgres + # tencent configurations + TENCENT_VECTOR_DB_URL: http://127.0.0.1 + TENCENT_VECTOR_DB_API_KEY: dify + TENCENT_VECTOR_DB_TIMEOUT: 30 + TENCENT_VECTOR_DB_USERNAME: dify + TENCENT_VECTOR_DB_DATABASE: dify + TENCENT_VECTOR_DB_SHARD: 1 + TENCENT_VECTOR_DB_REPLICAS: 2 # pgvector configurations PGVECTOR_HOST: pgvector PGVECTOR_PORT: 5432 diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx index 014b1094776bcb..1aebec0b4f17a9 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx @@ -32,7 +32,7 @@ const AppDetailLayout: FC = (props) => { const pathname = usePathname() const media = useBreakpoints() const isMobile = media === MediaType.mobile - const { isCurrentWorkspaceManager } = useAppContext() + const { isCurrentWorkspaceManager, isCurrentWorkspaceEditor } = useAppContext() const { appDetail, setAppDetail, setAppSiderbarExpand } = useStore(useShallow(state => ({ appDetail: state.appDetail, setAppDetail: state.setAppDetail, @@ -45,9 +45,9 @@ const AppDetailLayout: FC = (props) => { selectedIcon: NavIcon }>>([]) - const getNavigations = useCallback((appId: string, isCurrentWorkspaceManager: boolean, mode: string) => { + const getNavigations = useCallback((appId: string, isCurrentWorkspaceManager: boolean, isCurrentWorkspaceEditor: boolean, mode: string) => { const navs = [ - ...(isCurrentWorkspaceManager + ...(isCurrentWorkspaceEditor ? [{ name: t('common.appMenus.promptEng'), href: `/app/${appId}/${(mode === 'workflow' || mode === 'advanced-chat') ? 'workflow' : 'configuration'}`, @@ -62,14 +62,17 @@ const AppDetailLayout: FC = (props) => { icon: TerminalSquare, selectedIcon: TerminalSquareSolid, }, - { - name: mode !== 'workflow' - ? t('common.appMenus.logAndAnn') - : t('common.appMenus.logs'), - href: `/app/${appId}/logs`, - icon: FileHeart02, - selectedIcon: FileHeart02Solid, - }, + ...(isCurrentWorkspaceManager + ? [{ + name: mode !== 'workflow' + ? t('common.appMenus.logAndAnn') + : t('common.appMenus.logs'), + href: `/app/${appId}/logs`, + icon: FileHeart02, + selectedIcon: FileHeart02Solid, + }] + : [] + ), { name: t('common.appMenus.overview'), href: `/app/${appId}/overview`, @@ -104,10 +107,13 @@ const AppDetailLayout: FC = (props) => { } else { setAppDetail(res) - setNavigation(getNavigations(appId, isCurrentWorkspaceManager, res.mode)) + setNavigation(getNavigations(appId, isCurrentWorkspaceManager, isCurrentWorkspaceEditor, res.mode)) } + }).catch((e: any) => { + if (e.status === 404) + router.replace('/apps') }) - }, [appId, isCurrentWorkspaceManager]) + }, [appId, isCurrentWorkspaceManager, isCurrentWorkspaceEditor]) useUnmount(() => { setAppDetail() diff --git a/web/app/(commonLayout)/apps/AppCard.tsx b/web/app/(commonLayout)/apps/AppCard.tsx index 6d82fbe278b79b..59040f207de3b9 100644 --- a/web/app/(commonLayout)/apps/AppCard.tsx +++ b/web/app/(commonLayout)/apps/AppCard.tsx @@ -37,7 +37,7 @@ export type AppCardProps = { const AppCard = ({ app, onRefresh }: AppCardProps) => { const { t } = useTranslation() const { notify } = useContext(ToastContext) - const { isCurrentWorkspaceManager } = useAppContext() + const { isCurrentWorkspaceEditor } = useAppContext() const { onPlanInfoChanged } = useProviderContext() const { push } = useRouter() @@ -116,7 +116,7 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => { onRefresh() mutateApps() onPlanInfoChanged() - getRedirection(isCurrentWorkspaceManager, newApp, push) + getRedirection(isCurrentWorkspaceEditor, newApp, push) } catch (e) { notify({ type: 'error', message: t('app.newApp.appCreateFailed') }) @@ -224,7 +224,7 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => {
{ e.preventDefault() - getRedirection(isCurrentWorkspaceManager, app, push) + getRedirection(isCurrentWorkspaceEditor, app, push) }} className='group flex col-span-1 bg-white border-2 border-solid border-transparent rounded-xl shadow-sm min-h-[160px] flex flex-col transition-all duration-200 ease-in-out cursor-pointer hover:shadow-lg' > @@ -298,7 +298,7 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => { />
- {isCurrentWorkspaceManager && ( + {isCurrentWorkspaceEditor && ( <>
diff --git a/web/app/(commonLayout)/apps/Apps.tsx b/web/app/(commonLayout)/apps/Apps.tsx index 744bb9c9d70981..bf91d42fc8f3c9 100644 --- a/web/app/(commonLayout)/apps/Apps.tsx +++ b/web/app/(commonLayout)/apps/Apps.tsx @@ -50,7 +50,7 @@ const getKey = ( const Apps = () => { const { t } = useTranslation() - const { isCurrentWorkspaceManager } = useAppContext() + const { isCurrentWorkspaceEditor } = useAppContext() const showTagManagementModal = useTagStore(s => s.showTagManagementModal) const [activeTab, setActiveTab] = useTabSearchParams({ defaultTab: 'all', @@ -73,10 +73,10 @@ const Apps = () => { const anchorRef = useRef(null) const options = [ - { value: 'all', text: t('app.types.all'), icon: }, - { value: 'chat', text: t('app.types.chatbot'), icon: }, - { value: 'agent-chat', text: t('app.types.agent'), icon: }, - { value: 'workflow', text: t('app.types.workflow'), icon: }, + { value: 'all', text: t('app.types.all'), icon: }, + { value: 'chat', text: t('app.types.chatbot'), icon: }, + { value: 'agent-chat', text: t('app.types.agent'), icon: }, + { value: 'workflow', text: t('app.types.workflow'), icon: }, ] useEffect(() => { @@ -130,7 +130,7 @@ const Apps = () => {