Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add tidb on qdrant type #9831

Merged
merged 25 commits into from
Oct 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
f9e9797
add tidb on qdrant vector type
JohnJyong Jul 30, 2024
c3366e8
add tidb on qdrant vector type
JohnJyong Jul 30, 2024
dcebb6b
add tidb on qdrant vector type
JohnJyong Jul 30, 2024
58412d5
fix model runtime quota
JohnJyong Aug 14, 2024
09e7f81
Merge branch 'main' into feat/add-tidb-on-qdrant-type
JohnJyong Aug 14, 2024
1a518b9
tidb on qdrant schedule
JohnJyong Aug 15, 2024
eddc834
Merge branch 'main' into feat/add-tidb-on-qdrant-type
JohnJyong Aug 15, 2024
db1c75c
tidb on qdrant schedule
JohnJyong Aug 15, 2024
b45cb68
tidb on qdrant schedule
JohnJyong Aug 15, 2024
fd226d3
feat(website-crawl): add jina reader as additional alternative for we…
mapleeit Sep 25, 2024
31df381
Merge branch 'mapleeit/main' into feat/add-tidb-on-qdrant-type
JohnJyong Sep 27, 2024
0e8b857
add tidb on qdrant whitelist and batch job
JohnJyong Sep 27, 2024
f200c0c
update tidb batch create
JohnJyong Sep 30, 2024
1d6cfcc
update batch create tidb Serverless num to 20
JohnJyong Oct 21, 2024
fc4605f
tidb serverless fix
JohnJyong Oct 22, 2024
e9742b1
tidb serverless fix
JohnJyong Oct 22, 2024
1874c34
Merge branch 'main' into feat/add-tidb-on-qdrant-type
JohnJyong Oct 22, 2024
e649586
update migration
JohnJyong Oct 22, 2024
7cb03e4
fix embedding import
JohnJyong Oct 22, 2024
9f85c97
tidb on qdrant fix
JohnJyong Oct 23, 2024
f2ff4f9
tidb set password
JohnJyong Oct 24, 2024
12a0b47
upgrade unstructured version
JohnJyong Oct 25, 2024
74d036f
Merge branch 'main' into feat/add-tidb-on-qdrant-type
JohnJyong Oct 25, 2024
67037a1
upgrade unstructured version
JohnJyong Oct 25, 2024
a1e3464
upgrade unstructured version
JohnJyong Oct 25, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions api/configs/feature/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -571,6 +571,11 @@ class DataSetConfig(BaseSettings):
default=False,
)

TIDB_SERVERLESS_NUMBER: PositiveInt = Field(
description="number of tidb serverless cluster",
default=500,
)


class WorkspaceConfig(BaseSettings):
"""
Expand Down
7 changes: 7 additions & 0 deletions api/configs/middleware/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
from configs.middleware.vdb.qdrant_config import QdrantConfig
from configs.middleware.vdb.relyt_config import RelytConfig
from configs.middleware.vdb.tencent_vector_config import TencentVectorDBConfig
from configs.middleware.vdb.tidb_on_qdrant_config import TidbOnQdrantConfig
from configs.middleware.vdb.tidb_vector_config import TiDBVectorConfig
from configs.middleware.vdb.upstash_config import UpstashConfig
from configs.middleware.vdb.vikingdb_config import VikingDBConfig
Expand Down Expand Up @@ -54,6 +55,11 @@ class VectorStoreConfig(BaseSettings):
default=None,
)

VECTOR_STORE_WHITELIST_ENABLE: Optional[bool] = Field(
description="Enable whitelist for vector store.",
default=False,
)


class KeywordStoreConfig(BaseSettings):
KEYWORD_STORE: str = Field(
Expand Down Expand Up @@ -248,5 +254,6 @@ class MiddlewareConfig(
InternalTestConfig,
VikingDBConfig,
UpstashConfig,
TidbOnQdrantConfig,
):
pass
65 changes: 65 additions & 0 deletions api/configs/middleware/vdb/tidb_on_qdrant_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
from typing import Optional

from pydantic import Field, NonNegativeInt, PositiveInt
from pydantic_settings import BaseSettings


class TidbOnQdrantConfig(BaseSettings):
"""
Tidb on Qdrant configs
"""

TIDB_ON_QDRANT_URL: Optional[str] = Field(
description="Tidb on Qdrant url",
default=None,
)

TIDB_ON_QDRANT_API_KEY: Optional[str] = Field(
description="Tidb on Qdrant api key",
default=None,
)

TIDB_ON_QDRANT_CLIENT_TIMEOUT: NonNegativeInt = Field(
description="Tidb on Qdrant client timeout in seconds",
default=20,
)

TIDB_ON_QDRANT_GRPC_ENABLED: bool = Field(
description="whether enable grpc support for Tidb on Qdrant connection",
default=False,
)

TIDB_ON_QDRANT_GRPC_PORT: PositiveInt = Field(
description="Tidb on Qdrant grpc port",
default=6334,
)

TIDB_PUBLIC_KEY: Optional[str] = Field(
description="Tidb account public key",
default=None,
)

TIDB_PRIVATE_KEY: Optional[str] = Field(
description="Tidb account private key",
default=None,
)

TIDB_API_URL: Optional[str] = Field(
description="Tidb API url",
default=None,
)

TIDB_IAM_API_URL: Optional[str] = Field(
description="Tidb IAM API url",
default=None,
)

TIDB_REGION: Optional[str] = Field(
description="Tidb serverless region",
default="regions/aws-us-east-1",
)

TIDB_PROJECT_ID: Optional[str] = Field(
description="Tidb project id",
default=None,
)
1 change: 1 addition & 0 deletions api/controllers/console/datasets/datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -639,6 +639,7 @@ def get(self):
| VectorType.ORACLE
| VectorType.ELASTICSEARCH
| VectorType.PGVECTOR
| VectorType.TIDB_ON_QDRANT
):
return {
"retrieval_method": [
Expand Down
Empty file.
17 changes: 17 additions & 0 deletions api/core/rag/datasource/vdb/tidb_on_qdrant/tidb_entities.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
from typing import Optional

from pydantic import BaseModel


class ClusterEntity(BaseModel):
"""
Model Config Entity.
"""

name: str
cluster_id: str
displayName: str
region: str
spendingLimit: Optional[int] = 1000
version: str
createdBy: str
Loading