ext_redis.py support redis clusters --- Fixes #9538

api/configs/middleware/cache/redis_config.py

@@ -68,3 +68,18 @@ class RedisConfig(BaseSettings):
         description="Socket timeout in seconds for Redis Sentinel connections",
         default=0.1,
     )
+
+    REDIS_USE_CLUSTERS: Optional[bool] = Field(
+        description="Enable Redis Clusters mode for high availability",
+        default=False,
+    )
+
+    REDIS_CLUSTERS: Optional[str] = Field(
+        description="Comma-separated list of Redis Clusters nodes (host:port)",
+        default=None,
+    )
+
+    REDIS_CLUSTERS_PASSWORD: Optional[str] = Field(
+        description="Password for Redis Clusters authentication (if required)",
+        default=None,
+    )

api/extensions/ext_redis.py

@@ -1,11 +1,12 @@
 import redis
 from redis.connection import Connection, SSLConnection
 from redis.sentinel import Sentinel
+from redis.cluster import RedisCluster, ClusterNode
 
 from configs import dify_config
 
 
-class RedisClientWrapper(redis.Redis):
+class RedisClientWrapper():
     """
     A wrapper class for the Redis client that addresses the issue where the global
     `redis_client` variable cannot be updated when a new Redis instance is returned
@@ -71,6 +72,13 @@ def init_app(app):
         )
         master = sentinel.master_for(dify_config.REDIS_SENTINEL_SERVICE_NAME, **redis_params)
         redis_client.initialize(master)
+    elif dify_config.REDIS_USE_CLUSTERS:
+        startup_nodes = [
+            {"host": node.split(":")[0], "port": int(node.split(":")[1])} 
+            for node in dify_config.REDIS_CLUSTERS.split(",")
+        ]
+        nodes = [ClusterNode(host=node["host"], port=node["port"]) for node in startup_nodes]
+        redis_client.initialize(RedisCluster(startup_nodes=nodes, password=dify_config.REDIS_CLUSTERS_PASSWORD))
     else:
         redis_params.update(
             {

api/tests/unit_tests/core/test_model_manager.py

@@ -1,6 +1,8 @@
 from unittest.mock import MagicMock
 
 import pytest
+import redis
+from api.extensions.ext_redis import RedisClientWrapper
 
 from core.entities.provider_entities import ModelLoadBalancingConfiguration
 from core.model_manager import LBModelManager
@@ -55,6 +57,10 @@ def incr(key):
         start_index += 1
         return start_index
 
+    fake_redis_client = redis.Redis(host="localhost", port=6379, db=0)
+    redis_client = RedisClientWrapper()
+    redis_client.initialize(fake_redis_client)
+
     mocker.patch("redis.Redis.incr", side_effect=incr)
     mocker.patch("redis.Redis.set", return_value=None)
     mocker.patch("redis.Redis.expire", return_value=None)

docker/docker-compose.yaml

@@ -51,6 +51,9 @@ x-shared-env: &shared-api-worker-env
   REDIS_SENTINEL_PASSWORD: ${REDIS_SENTINEL_PASSWORD:-}
   ACCESS_TOKEN_EXPIRE_MINUTES: ${ACCESS_TOKEN_EXPIRE_MINUTES:-60}
   REDIS_SENTINEL_SOCKET_TIMEOUT: ${REDIS_SENTINEL_SOCKET_TIMEOUT:-0.1}
+  REDIS_CLUSTERS: ${REDIS_CLUSTERS:-}
+  REDIS_USE_CLUSTERS: ${REDIS_USE_CLUSTERS:-false}
+  REDIS_CLUSTERS_PASSWORD: ${REDIS_CLUSTERS_PASSWORD:-}
   CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://:difyai123456@redis:6379/1}
   BROKER_USE_SSL: ${BROKER_USE_SSL:-false}
   CELERY_USE_SENTINEL: ${CELERY_USE_SENTINEL:-false}