Merge branch 'dev'

.github/workflows/api-tests.yml

@@ -76,7 +76,7 @@ jobs:
       - name: Run Workflow
         run: poetry run -C api bash dev/pytest/pytest_workflow.sh
 
-      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale)
+      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch)
         uses: hoverkraft-tech/[email protected]
         with:
           compose-file: |
@@ -90,5 +90,6 @@ jobs:
             pgvecto-rs
             pgvector
             chroma
+            elasticsearch
       - name: Test Vector Stores
         run: poetry run -C api bash dev/pytest/pytest_vdb.sh

.github/workflows/expose_service_ports.sh

@@ -6,5 +6,6 @@ yq eval '.services.chroma.ports += ["8000:8000"]' -i docker/docker-compose.yaml
 yq eval '.services["milvus-standalone"].ports += ["19530:19530"]' -i docker/docker-compose.yaml
 yq eval '.services.pgvector.ports += ["5433:5432"]' -i docker/docker-compose.yaml
 yq eval '.services["pgvecto-rs"].ports += ["5431:5432"]' -i docker/docker-compose.yaml
+yq eval '.services["elasticsearch"].ports += ["9200:9200"]' -i docker/docker-compose.yaml
 
-echo "Ports exposed for sandbox, weaviate, qdrant, chroma, milvus, pgvector, pgvecto-rs."
+echo "Ports exposed for sandbox, weaviate, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch"

.github/workflows/style.yml

@@ -45,6 +45,10 @@ jobs:
         if: steps.changed-files.outputs.any_changed == 'true'
         run: poetry run -C api dotenv-linter ./api/.env.example ./web/.env.example
 
+      - name: Ruff formatter check
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: poetry run -C api ruff format --check ./api
+
       - name: Lint hints
         if: failure()
         run: echo "Please run 'dev/reformat' to fix the fixable linting errors."

api/.env.example

@@ -130,6 +130,12 @@ TENCENT_VECTOR_DB_DATABASE=dify
 TENCENT_VECTOR_DB_SHARD=1
 TENCENT_VECTOR_DB_REPLICAS=2
 
+# ElasticSearch configuration
+ELASTICSEARCH_HOST=127.0.0.1
+ELASTICSEARCH_PORT=9200
+ELASTICSEARCH_USERNAME=elastic
+ELASTICSEARCH_PASSWORD=elastic
+
 # PGVECTO_RS configuration
 PGVECTO_RS_HOST=localhost
 PGVECTO_RS_PORT=5431
@@ -261,4 +267,13 @@ APP_MAX_ACTIVE_REQUESTS=0
 
 
 # Celery beat configuration
-CELERY_BEAT_SCHEDULER_TIME=1
+CELERY_BEAT_SCHEDULER_TIME=1
+
+# Position configuration
+POSITION_TOOL_PINS=
+POSITION_TOOL_INCLUDES=
+POSITION_TOOL_EXCLUDES=
+
+POSITION_PROVIDER_PINS=
+POSITION_PROVIDER_INCLUDES=
+POSITION_PROVIDER_EXCLUDES=

api/app.py

@@ -1,6 +1,6 @@
 import os
 
-if os.environ.get("DEBUG", "false").lower() != 'true':
+if os.environ.get("DEBUG", "false").lower() != "true":
     from gevent import monkey
 
     monkey.patch_all()
@@ -57,7 +57,7 @@
 if os.name == "nt":
     os.system('tzutil /s "UTC"')
 else:
-    os.environ['TZ'] = 'UTC'
+    os.environ["TZ"] = "UTC"
     time.tzset()
 
 
@@ -70,13 +70,14 @@ class DifyApp(Flask):
 # -------------
 
 
-config_type = os.getenv('EDITION', default='SELF_HOSTED')  # ce edition first
+config_type = os.getenv("EDITION", default="SELF_HOSTED")  # ce edition first
 
 
 # ----------------------------
 # Application Factory Function
 # ----------------------------
 
+
 def create_flask_app_with_configs() -> Flask:
     """
     create a raw flask app
@@ -92,42 +93,43 @@ def create_flask_app_with_configs() -> Flask:
         elif isinstance(value, int | float | bool):
             os.environ[key] = str(value)
         elif value is None:
-            os.environ[key] = ''
+            os.environ[key] = ""
 
     return dify_app
 
 
 def create_app() -> Flask:
     app = create_flask_app_with_configs()
 
-    app.secret_key = app.config['SECRET_KEY']
+    app.secret_key = app.config["SECRET_KEY"]
 
     log_handlers = None
-    log_file = app.config.get('LOG_FILE')
+    log_file = app.config.get("LOG_FILE")
     if log_file:
         log_dir = os.path.dirname(log_file)
         os.makedirs(log_dir, exist_ok=True)
         log_handlers = [
             RotatingFileHandler(
                 filename=log_file,
                 maxBytes=1024 * 1024 * 1024,
-                backupCount=5
+                backupCount=5,
             ),
-            logging.StreamHandler(sys.stdout)
+            logging.StreamHandler(sys.stdout),
         ]
 
     logging.basicConfig(
-        level=app.config.get('LOG_LEVEL'),
-        format=app.config.get('LOG_FORMAT'),
-        datefmt=app.config.get('LOG_DATEFORMAT'),
+        level=app.config.get("LOG_LEVEL"),
+        format=app.config.get("LOG_FORMAT"),
+        datefmt=app.config.get("LOG_DATEFORMAT"),
         handlers=log_handlers,
-        force=True
+        force=True,
     )
-    log_tz = app.config.get('LOG_TZ')
+    log_tz = app.config.get("LOG_TZ")
     if log_tz:
         from datetime import datetime
 
         import pytz
+
         timezone = pytz.timezone(log_tz)
 
         def time_converter(seconds):
@@ -162,24 +164,24 @@ def initialize_extensions(app):
 @login_manager.request_loader
 def load_user_from_request(request_from_flask_login):
     """Load user based on the request."""
-    if request.blueprint not in ['console', 'inner_api']:
+    if request.blueprint not in ["console", "inner_api"]:
         return None
     # Check if the user_id contains a dot, indicating the old format
-    auth_header = request.headers.get('Authorization', '')
+    auth_header = request.headers.get("Authorization", "")
     if not auth_header:
-        auth_token = request.args.get('_token')
+        auth_token = request.args.get("_token")
         if not auth_token:
-            raise Unauthorized('Invalid Authorization token.')
+            raise Unauthorized("Invalid Authorization token.")
     else:
-        if ' ' not in auth_header:
-            raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
+        if " " not in auth_header:
+            raise Unauthorized("Invalid Authorization header format. Expected 'Bearer <api-key>' format.")
         auth_scheme, auth_token = auth_header.split(None, 1)
         auth_scheme = auth_scheme.lower()
-        if auth_scheme != 'bearer':
-            raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
+        if auth_scheme != "bearer":
+            raise Unauthorized("Invalid Authorization header format. Expected 'Bearer <api-key>' format.")
 
     decoded = PassportService().verify(auth_token)
-    user_id = decoded.get('user_id')
+    user_id = decoded.get("user_id")
 
     account = AccountService.load_logged_in_account(account_id=user_id, token=auth_token)
     if account:
@@ -190,10 +192,11 @@ def load_user_from_request(request_from_flask_login):
 @login_manager.unauthorized_handler
 def unauthorized_handler():
     """Handle unauthorized requests."""
-    return Response(json.dumps({
-        'code': 'unauthorized',
-        'message': "Unauthorized."
-    }), status=401, content_type="application/json")
+    return Response(
+        json.dumps({"code": "unauthorized", "message": "Unauthorized."}),
+        status=401,
+        content_type="application/json",
+    )
 
 
 # register blueprint routers
@@ -204,38 +207,36 @@ def register_blueprints(app):
     from controllers.service_api import bp as service_api_bp
     from controllers.web import bp as web_bp
 
-    CORS(service_api_bp,
-         allow_headers=['Content-Type', 'Authorization', 'X-App-Code'],
-         methods=['GET', 'PUT', 'POST', 'DELETE', 'OPTIONS', 'PATCH']
-         )
+    CORS(
+        service_api_bp,
+        allow_headers=["Content-Type", "Authorization", "X-App-Code"],
+        methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"],
+    )
     app.register_blueprint(service_api_bp)
 
-    CORS(web_bp,
-         resources={
-             r"/*": {"origins": app.config['WEB_API_CORS_ALLOW_ORIGINS']}},
-         supports_credentials=True,
-         allow_headers=['Content-Type', 'Authorization', 'X-App-Code'],
-         methods=['GET', 'PUT', 'POST', 'DELETE', 'OPTIONS', 'PATCH'],
-         expose_headers=['X-Version', 'X-Env']
-         )
+    CORS(
+        web_bp,
+        resources={r"/*": {"origins": app.config["WEB_API_CORS_ALLOW_ORIGINS"]}},
+        supports_credentials=True,
+        allow_headers=["Content-Type", "Authorization", "X-App-Code"],
+        methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"],
+        expose_headers=["X-Version", "X-Env"],
+    )
 
     app.register_blueprint(web_bp)
 
-    CORS(console_app_bp,
-         resources={
-             r"/*": {"origins": app.config['CONSOLE_CORS_ALLOW_ORIGINS']}},
-         supports_credentials=True,
-         allow_headers=['Content-Type', 'Authorization'],
-         methods=['GET', 'PUT', 'POST', 'DELETE', 'OPTIONS', 'PATCH'],
-         expose_headers=['X-Version', 'X-Env']
-         )
+    CORS(
+        console_app_bp,
+        resources={r"/*": {"origins": app.config["CONSOLE_CORS_ALLOW_ORIGINS"]}},
+        supports_credentials=True,
+        allow_headers=["Content-Type", "Authorization"],
+        methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"],
+        expose_headers=["X-Version", "X-Env"],
+    )
 
     app.register_blueprint(console_app_bp)
 
-    CORS(files_bp,
-         allow_headers=['Content-Type'],
-         methods=['GET', 'PUT', 'POST', 'DELETE', 'OPTIONS', 'PATCH']
-         )
+    CORS(files_bp, allow_headers=["Content-Type"], methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"])
     app.register_blueprint(files_bp)
 
     app.register_blueprint(inner_api_bp)
@@ -245,29 +246,29 @@ def register_blueprints(app):
 app = create_app()
 celery = app.extensions["celery"]
 
-if app.config.get('TESTING'):
+if app.config.get("TESTING"):
     print("App is running in TESTING mode")
 
 
 @app.after_request
 def after_request(response):
     """Add Version headers to the response."""
-    response.set_cookie('remember_token', '', expires=0)
-    response.headers.add('X-Version', app.config['CURRENT_VERSION'])
-    response.headers.add('X-Env', app.config['DEPLOY_ENV'])
+    response.set_cookie("remember_token", "", expires=0)
+    response.headers.add("X-Version", app.config["CURRENT_VERSION"])
+    response.headers.add("X-Env", app.config["DEPLOY_ENV"])
     return response
 
 
-@app.route('/health')
+@app.route("/health")
 def health():
-    return Response(json.dumps({
-        'pid': os.getpid(),
-        'status': 'ok',
-        'version': app.config['CURRENT_VERSION']
-    }), status=200, content_type="application/json")
+    return Response(
+        json.dumps({"pid": os.getpid(), "status": "ok", "version": app.config["CURRENT_VERSION"]}),
+        status=200,
+        content_type="application/json",
+    )
 
 
-@app.route('/threads')
+@app.route("/threads")
 def threads():
     num_threads = threading.active_count()
     threads = threading.enumerate()
@@ -278,32 +279,34 @@ def threads():
         thread_id = thread.ident
         is_alive = thread.is_alive()
 
-        thread_list.append({
-            'name': thread_name,
-            'id': thread_id,
-            'is_alive': is_alive
-        })
+        thread_list.append(
+            {
+                "name": thread_name,
+                "id": thread_id,
+                "is_alive": is_alive,
+            }
+        )
 
     return {
-        'pid': os.getpid(),
-        'thread_num': num_threads,
-        'threads': thread_list
+        "pid": os.getpid(),
+        "thread_num": num_threads,
+        "threads": thread_list,
     }
 
 
-@app.route('/db-pool-stat')
+@app.route("/db-pool-stat")
 def pool_stat():
     engine = db.engine
     return {
-        'pid': os.getpid(),
-        'pool_size': engine.pool.size(),
-        'checked_in_connections': engine.pool.checkedin(),
-        'checked_out_connections': engine.pool.checkedout(),
-        'overflow_connections': engine.pool.overflow(),
-        'connection_timeout': engine.pool.timeout(),
-        'recycle_time': db.engine.pool._recycle
+        "pid": os.getpid(),
+        "pool_size": engine.pool.size(),
+        "checked_in_connections": engine.pool.checkedin(),
+        "checked_out_connections": engine.pool.checkedout(),
+        "overflow_connections": engine.pool.overflow(),
+        "connection_timeout": engine.pool.timeout(),
+        "recycle_time": db.engine.pool._recycle,
     }
 
 
-if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=5001)
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=5001)