Skip to content

Commit

Permalink
parse tokenizer_backend=None properly (#2509)
Browse files Browse the repository at this point in the history
  • Loading branch information
baberabb authored Nov 22, 2024
1 parent 867413f commit 9d36354
Showing 1 changed file with 4 additions and 2 deletions.
6 changes: 4 additions & 2 deletions lm_eval/models/api_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ def __init__(
# however the requests can be sent as a string if the API doesn't support token inputs.
# use tokenized_requests=False
tokenizer_backend: Optional[
Literal["tiktoken", "huggingface", None]
Literal["tiktoken", "huggingface", "None", "none"]
] = "huggingface",
truncate: bool = False,
# number of concurrent requests. More useful if not batching
Expand Down Expand Up @@ -116,7 +116,9 @@ def __init__(
"Concurrent requests are disabled. To enable concurrent requests, set `num_concurrent` > 1."
)
self._concurrent = int(num_concurrent)
self.tokenizer_backend = tokenizer_backend
self.tokenizer_backend = (
None if tokenizer_backend in ("None", "none") else tokenizer_backend
)
self.add_bos_token = add_bos_token
self.custom_prefix_token_id = custom_prefix_token_id
self.tokenized_requests = tokenized_requests
Expand Down

0 comments on commit 9d36354

Please sign in to comment.