-
Notifications
You must be signed in to change notification settings - Fork 8.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
b281a80
commit 61d846f
Showing
11 changed files
with
174 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -51,7 +51,15 @@ jobs: | |
- name: Expose Service Ports | ||
run: sh .github/workflows/expose_service_ports.sh | ||
|
||
- name: Set up Vector Stores (TiDB, Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch, Couchbase) | ||
- name: Set up Vector Store (TiDB) | ||
uses: hoverkraft-tech/[email protected] | ||
with: | ||
compose-file: docker/tidb/docker-compose.yaml | ||
services: | | ||
tidb | ||
tiflash | ||
- name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch, Couchbase) | ||
uses: hoverkraft-tech/[email protected] | ||
with: | ||
compose-file: | | ||
|
@@ -67,7 +75,9 @@ jobs: | |
pgvector | ||
chroma | ||
elasticsearch | ||
tidb | ||
- name: Check TiDB Ready | ||
run: poetry run -C api python api/tests/integration_tests/vdb/tidb_vector/check_tiflash_ready.py | ||
|
||
- name: Test Vector Stores | ||
run: poetry run -C api bash dev/pytest/pytest_vdb.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
59 changes: 59 additions & 0 deletions
59
api/tests/integration_tests/vdb/tidb_vector/check_tiflash_ready.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
import time | ||
|
||
import pymysql | ||
|
||
|
||
def check_tiflash_ready() -> bool: | ||
try: | ||
connection = pymysql.connect( | ||
host="localhost", | ||
port=4000, | ||
user="root", | ||
password="", | ||
) | ||
|
||
with connection.cursor() as cursor: | ||
# Doc reference: | ||
# https://docs.pingcap.com/zh/tidb/v8.5/information-schema-cluster-hardware | ||
select_tiflash_query = """ | ||
SELECT * FROM information_schema.cluster_hardware | ||
WHERE TYPE='tiflash' | ||
LIMIT 1; | ||
""" | ||
cursor.execute(select_tiflash_query) | ||
result = cursor.fetchall() | ||
return result is not None and len(result) > 0 | ||
except Exception as e: | ||
print(f"TiFlash is not ready. Exception: {e}") | ||
return False | ||
finally: | ||
if connection: | ||
connection.close() | ||
|
||
|
||
def main(): | ||
max_attempts = 30 | ||
retry_interval_seconds = 2 | ||
is_tiflash_ready = False | ||
for attempt in range(max_attempts): | ||
try: | ||
is_tiflash_ready = check_tiflash_ready() | ||
except Exception as e: | ||
print(f"TiFlash is not ready. Exception: {e}") | ||
is_tiflash_ready = False | ||
|
||
if is_tiflash_ready: | ||
break | ||
else: | ||
print(f"Attempt {attempt + 1} failed,retry in {retry_interval_seconds} seconds...") | ||
time.sleep(retry_interval_seconds) | ||
|
||
if is_tiflash_ready: | ||
print("TiFlash is ready in TiDB.") | ||
else: | ||
print(f"TiFlash is not ready in TiDB after {max_attempts} attempting checks.") | ||
exit(1) | ||
|
||
|
||
if __name__ == "__main__": | ||
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
# PD Configuration File reference: | ||
# https://docs.pingcap.com/tidb/stable/pd-configuration-file#pd-configuration-file | ||
[replication] | ||
max-replicas = 1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
# TiFlash tiflash-learner.toml Configuration File reference: | ||
# https://docs.pingcap.com/tidb/stable/tiflash-configuration#configure-the-tiflash-learnertoml-file | ||
|
||
log-file = "/logs/tiflash_tikv.log" | ||
|
||
[server] | ||
engine-addr = "tiflash:4030" | ||
addr = "0.0.0.0:20280" | ||
advertise-addr = "tiflash:20280" | ||
status-addr = "tiflash:20292" | ||
|
||
[storage] | ||
data-dir = "/data/flash" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
# TiFlash tiflash.toml Configuration File reference: | ||
# https://docs.pingcap.com/tidb/stable/tiflash-configuration#configure-the-tiflashtoml-file | ||
|
||
listen_host = "0.0.0.0" | ||
path = "/data" | ||
|
||
[flash] | ||
tidb_status_addr = "tidb:10080" | ||
service_addr = "tiflash:4030" | ||
|
||
[flash.proxy] | ||
config = "/tiflash-learner.toml" | ||
|
||
[logger] | ||
errorlog = "/logs/tiflash_error.log" | ||
log = "/logs/tiflash.log" | ||
|
||
[raft] | ||
pd_addr = "pd0:2379" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,62 @@ | ||
services: | ||
pd0: | ||
image: pingcap/pd:v8.5.0 | ||
ports: | ||
- "2379" | ||
volumes: | ||
- ./config/pd.toml:/pd.toml:ro | ||
- ./volumes/data:/data | ||
- ./volumes/logs:/logs | ||
command: | ||
- --name=pd0 | ||
- --client-urls=http://0.0.0.0:2379 | ||
- --peer-urls=http://0.0.0.0:2380 | ||
- --advertise-client-urls=http://pd0:2379 | ||
- --advertise-peer-urls=http://pd0:2380 | ||
- --initial-cluster=pd0=http://pd0:2380 | ||
- --data-dir=/data/pd | ||
- --config=/pd.toml | ||
- --log-file=/logs/pd.log | ||
restart: on-failure | ||
tikv: | ||
image: pingcap/tikv:v8.5.0 | ||
volumes: | ||
- ./volumes/data:/data | ||
- ./volumes/logs:/logs | ||
command: | ||
- --addr=0.0.0.0:20160 | ||
- --advertise-addr=tikv:20160 | ||
- --status-addr=tikv:20180 | ||
- --data-dir=/data/tikv | ||
- --pd=pd0:2379 | ||
- --log-file=/logs/tikv.log | ||
depends_on: | ||
- "pd0" | ||
restart: on-failure | ||
tidb: | ||
image: pingcap/tidb:v8.5.0 | ||
ports: | ||
- "4000:4000" | ||
volumes: | ||
- ./volumes/logs:/logs | ||
command: | ||
- --advertise-address=tidb | ||
- --store=tikv | ||
- --path=pd0:2379 | ||
- --log-file=/logs/tidb.log | ||
depends_on: | ||
- "tikv" | ||
restart: on-failure | ||
tiflash: | ||
image: pingcap/tiflash:v8.5.0 | ||
volumes: | ||
- ./config/tiflash.toml:/tiflash.toml:ro | ||
- ./config/tiflash-learner.toml:/tiflash-learner.toml:ro | ||
- ./volumes/data:/data | ||
- ./volumes/logs:/logs | ||
command: | ||
- --config=/tiflash.toml | ||
depends_on: | ||
- "tikv" | ||
- "tidb" | ||
restart: on-failure |