fix openai ft models

api/core/indexing_runner.py

@@ -297,13 +297,14 @@ def file_indexing_estimate(self, tenant_id: str, file_details: List[UploadFile],
                     "qa_preview": document_qa_list,
                     "preview": preview_texts
                 }
-        embedding_model_type_instance = cast(TextEmbeddingModel, embedding_model_instance.model_type_instance)
-        embedding_price_info = embedding_model_type_instance.get_price(
-            model=embedding_model_instance.model,
-            credentials=embedding_model_instance.credentials,
-            price_type=PriceType.INPUT,
-            tokens=tokens
-        )
+        if embedding_model_instance:
+            embedding_model_type_instance = cast(TextEmbeddingModel, embedding_model_instance.model_type_instance)
+            embedding_price_info = embedding_model_type_instance.get_price(
+                model=embedding_model_instance.model,
+                credentials=embedding_model_instance.credentials,
+                price_type=PriceType.INPUT,
+                tokens=tokens
+            )
         return {
             "total_segments": total_segments,
             "tokens": tokens,

api/core/model_runtime/model_providers/openai/llm/llm.py

@@ -125,7 +125,7 @@ def validate_credentials(self, model: str, credentials: dict) -> None:
                 base_model = model.split(':')[1]
 
                 # check if model exists
-                remote_models = self.remote_models()
+                remote_models = self.remote_models(credentials)
                 remote_model_map = {model.model: model for model in remote_models}
                 if model not in remote_model_map:
                     raise CredentialsValidateFailedError(f'Fine-tuned model {model} not found')
@@ -687,6 +687,9 @@ def _num_tokens_from_messages(self, model: str, messages: List[PromptMessage],
 
         Official documentation: https://github.com/openai/openai-cookbook/blob/
         main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb"""
+        if model.startswith('ft:'):
+            model = model.split(':')[1]
+
         try:
             encoding = tiktoken.encoding_for_model(model)
         except KeyError:

api/migrations/versions/187385f442fc_modify_provider_model_name_length.py

@@ -0,0 +1,38 @@
+"""modify provider model name length
+
+Revision ID: 187385f442fc
+Revises: 88072f0caa04
+Create Date: 2024-01-02 07:18:43.887428
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '187385f442fc'
+down_revision = '88072f0caa04'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('provider_models', schema=None) as batch_op:
+        batch_op.alter_column('model_name',
+                              existing_type=sa.VARCHAR(length=40),
+                              type_=sa.String(length=255),
+                              existing_nullable=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('provider_models', schema=None) as batch_op:
+        batch_op.alter_column('model_name',
+                              existing_type=sa.String(length=255),
+                              type_=sa.VARCHAR(length=40),
+                              existing_nullable=False)
+
+    # ### end Alembic commands ###

api/models/provider.py

@@ -96,7 +96,7 @@ class ProviderModel(db.Model):
     id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
     tenant_id = db.Column(UUID, nullable=False)
     provider_name = db.Column(db.String(40), nullable=False)
-    model_name = db.Column(db.String(40), nullable=False)
+    model_name = db.Column(db.String(255), nullable=False)
     model_type = db.Column(db.String(40), nullable=False)
     encrypted_config = db.Column(db.Text, nullable=True)
     is_valid = db.Column(db.Boolean, nullable=False, server_default=db.text('false'))