Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

LUMI-127 adding inference API #328

Merged
merged 21 commits into from
Nov 14, 2024
Merged

LUMI-127 adding inference API #328

Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
7348558
First pass at inference API
aittalam Oct 29, 2024
edc774e
Added tests
aittalam Oct 30, 2024
227b68e
Refactored to have a single create_job method
aittalam Nov 2, 2024
bab45c1
Provide a class to handle the jobs endpoint (#302)
javiermtorres Oct 29, 2024
7449345
Backend: Tests use Alembic (#320)
peteski22 Oct 31, 2024
a0c8ec1
chore: tear down containers launched after make test (#327)
ividal Oct 31, 2024
8a1cf1e
Remove docker-compose volume mount to avoid .venv conflicts (#323)
agpituk Oct 31, 2024
bd518d1
Update README.md (#334)
njbrake Nov 1, 2024
4ca1b41
Fixed sdk and tests
aittalam Nov 2, 2024
792db2c
Merge branch 'main' into davide/LUMI-127-inference-api
aittalam Nov 2, 2024
528fc33
Merged with main
aittalam Nov 14, 2024
354ba37
Fixed merge issues
aittalam Nov 14, 2024
5031572
Fixed notebook
aittalam Nov 14, 2024
e03e03f
Merge branch 'main' into davide/LUMI-127-inference-api
aittalam Nov 14, 2024
5822f5f
Linting imports
aittalam Nov 14, 2024
3df241f
Refactored server to inference_server in job config
aittalam Nov 14, 2024
f6fa32e
Merge branch 'main' into davide/LUMI-127-inference-api
aittalam Nov 14, 2024
a922b71
Fixed erros found in review
aittalam Nov 14, 2024
42fd160
Removed print logging
aittalam Nov 14, 2024
38b7af5
Update lumigator/python/mzai/jobs/inference/tests/test_configs.py
aittalam Nov 14, 2024
696a1b2
Removed TODOs
aittalam Nov 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .vscode/settings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,8 @@
"source.organizeImports.ruff": "explicit"
}
},
"python.testing.pytestEnabled": true
"python.testing.pytestEnabled": true,
"python.analysis.extraPaths": [
aittalam marked this conversation as resolved.
Show resolved Hide resolved
"./lumigator/python/mzai/schemas"
aittalam marked this conversation as resolved.
Show resolved Hide resolved
]
}
11 changes: 6 additions & 5 deletions lumigator/python/mzai/backend/backend/api/routes/jobs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
from fastapi import APIRouter, status
from lumigator_schemas.extras import ListingResponse
from lumigator_schemas.jobs import (
JobCreate,
JobEvalCreate,
JobInferenceCreate,
aittalam marked this conversation as resolved.
Show resolved Hide resolved
JobResponse,
JobResultDownloadResponse,
JobResultResponse,
Expand All @@ -20,11 +21,11 @@
@router.post("/inference/", status_code=status.HTTP_201_CREATED)
def create_inference_job(
service: JobServiceDep,
job_create_request: JobCreate,
job_create_request: JobInferenceCreate,
request: Request,
response: Response
) -> JobResponse:
job_response = service.create_inference_job(job_create_request)
job_response = service.create_job(job_create_request)

url = request.url_for(get_job.__name__, job_id=job_response.id)
response.headers[HttpHeaders.LOCATION] = f"{url}"
Expand All @@ -35,11 +36,11 @@ def create_inference_job(
@router.post("/evaluate/", status_code=status.HTTP_201_CREATED)
def create_evaluation_job(
service: JobServiceDep,
job_create_request: JobCreate,
job_create_request: JobEvalCreate,
request: Request,
response: Response
) -> JobResponse:
job_response = service.create_evaluation_job(job_create_request)
job_response = service.create_job(job_create_request)

url = request.url_for(get_job.__name__, job_id=job_response.id)
response.headers[HttpHeaders.LOCATION] = f"{url}"
Expand Down
79 changes: 47 additions & 32 deletions lumigator/python/mzai/backend/backend/config_templates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
# Evaluation templates
from lumigator_schemas.jobs import JobType

seq2seq_eval_template = """{{
"name": "{job_name}/{job_id}",
"model": {{ "path": "{model_path}" }},
Expand Down Expand Up @@ -83,42 +85,55 @@
"dataset": {{ "path": "{dataset_path}" }},
}}"""


oai_infer_template = """{{
"name": "{job_name}/{job_id}",
"model": {{
"inference": {{
"base_url": "{model_url}",
"engine": "{model_path}",
"system_prompt": "{system_prompt}",
"max_retries": 3
}}
}},
"dataset": {{ "path": "{dataset_path}" }},
"job": {{
"max_samples": {max_samples},
"storage_path": "{storage_path}",
"output_field": "{output_field}"
}},
"inference_server": {{
"base_url": "{model_url}",
"engine": "{model_path}",
"system_prompt": "{system_prompt}",
"max_retries": 3
}},
"params": {{
"max_tokens": {max_tokens},
"frequency_penalty": {frequency_penalty},
"temperature": {temperature},
"top_p": {top_p}
}}
}}"""

config_infer_template = {
"hf://facebook/bart-large-cnn": bart_eval_template,
"hf://mikeadimech/longformer-qmsum-meeting-summarization": seq2seq_eval_template,
"hf://mrm8488/t5-base-finetuned-summarize-news": seq2seq_eval_template,
"hf://Falconsai/text_summarization": seq2seq_eval_template,
"hf://mistralai/Mistral-7B-Instruct-v0.3": causal_eval_template,
"oai://gpt-4o-mini": oai_eval_template,
"oai://gpt-4-turbo": oai_eval_template,
"oai://gpt-3.5-turbo-0125": oai_eval_template,
"mistral://open-mistral-7b": oai_eval_template,
"llamafile://mistralai/Mistral-7B-Instruct-v0.2": oai_eval_template,
}


config_eval_template = {
"hf://facebook/bart-large-cnn": bart_eval_template,
"hf://mikeadimech/longformer-qmsum-meeting-summarization": seq2seq_eval_template,
"hf://mrm8488/t5-base-finetuned-summarize-news": seq2seq_eval_template,
"hf://Falconsai/text_summarization": seq2seq_eval_template,
"hf://mistralai/Mistral-7B-Instruct-v0.3": causal_eval_template,
"oai://gpt-4o-mini": oai_eval_template,
"oai://gpt-4-turbo": oai_eval_template,
"oai://gpt-3.5-turbo-0125": oai_eval_template,
"mistral://open-mistral-7b": oai_eval_template,
"llamafile://mistralai/Mistral-7B-Instruct-v0.2": oai_eval_template,
templates = {
JobType.INFERENCE: {
"default": causal_infer_template,
"hf://facebook/bart-large-cnn": bart_eval_template,
"hf://mikeadimech/longformer-qmsum-meeting-summarization": seq2seq_eval_template,
aittalam marked this conversation as resolved.
Show resolved Hide resolved
"hf://mrm8488/t5-base-finetuned-summarize-news": seq2seq_eval_template,
"hf://Falconsai/text_summarization": seq2seq_eval_template,
"hf://mistralai/Mistral-7B-Instruct-v0.3": causal_eval_template,
"oai://gpt-4o-mini": oai_infer_template,
"oai://gpt-4-turbo": oai_infer_template,
"oai://gpt-3.5-turbo-0125": oai_infer_template,
"mistral://open-mistral-7b": oai_infer_template,
"llamafile://mistralai/Mistral-7B-Instruct-v0.2": oai_infer_template,
},
JobType.EVALUATION: {
"default": causal_eval_template,
"hf://facebook/bart-large-cnn": bart_eval_template,
"hf://mikeadimech/longformer-qmsum-meeting-summarization": seq2seq_eval_template,
"hf://mrm8488/t5-base-finetuned-summarize-news": seq2seq_eval_template,
"hf://Falconsai/text_summarization": seq2seq_eval_template,
"hf://mistralai/Mistral-7B-Instruct-v0.3": causal_eval_template,
"oai://gpt-4o-mini": oai_eval_template,
"oai://gpt-4-turbo": oai_eval_template,
"oai://gpt-3.5-turbo-0125": oai_eval_template,
"mistral://open-mistral-7b": oai_eval_template,
"llamafile://mistralai/Mistral-7B-Instruct-v0.2": oai_eval_template,
},
}
3 changes: 3 additions & 0 deletions lumigator/python/mzai/backend/backend/services/datasets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,9 @@ def delete_dataset(self, dataset_id: UUID) -> None:
self.dataset_repo.delete(record.id)


# Delete DB record
aittalam marked this conversation as resolved.
Show resolved Hide resolved
self.dataset_repo.delete(record.id)

def get_dataset_download(self, dataset_id: UUID) -> DatasetDownloadResponse:
"""Generate presigned download URLs for dataset files."""
record = self._get_dataset_record(dataset_id)
Expand Down
Loading