Update README.md
Browse files
README.md
CHANGED
@@ -43,7 +43,7 @@ from transformers import AutoTokenizer
|
|
43 |
from vllm import LLM, SamplingParams
|
44 |
|
45 |
max_model_len, tp_size = 4096, 1
|
46 |
-
model_name = "nm-testing/Mistral-Small-24B-Instruct-2501-FP8-Dynamic
|
47 |
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
48 |
llm = LLM(model=model_name, tensor_parallel_size=tp_size, max_model_len=max_model_len, trust_remote_code=True)
|
49 |
sampling_params = SamplingParams(temperature=0.3, max_tokens=256, stop_token_ids=[tokenizer.eos_token_id])
|
|
|
43 |
from vllm import LLM, SamplingParams
|
44 |
|
45 |
max_model_len, tp_size = 4096, 1
|
46 |
+
model_name = "nm-testing/Mistral-Small-24B-Instruct-2501-FP8-Dynamic"
|
47 |
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
48 |
llm = LLM(model=model_name, tensor_parallel_size=tp_size, max_model_len=max_model_len, trust_remote_code=True)
|
49 |
sampling_params = SamplingParams(temperature=0.3, max_tokens=256, stop_token_ids=[tokenizer.eos_token_id])
|