Update README.md

README.md

@@ -54,7 +54,7 @@ class LLM(object):
             model_path,
             torch_dtype=torch.bfloat16,
             device_map="auto",
-            load_in_4bit=True,
+            load_in_4bit=False,
             trust_remote_code=False,
         )
 
@@ -63,8 +63,8 @@ class LLM(object):
         )
 
         self.terminators = [
-            self.tokenizer.convert_tokens_to_ids("<|im_end|>"),
             self.tokenizer.convert_tokens_to_ids("<|end_of_text|>"),
+            self.tokenizer.convert_tokens_to_ids("<|eot_id|>"),
         ]
 
     def generate_text(self, instruction):
@@ -76,7 +76,7 @@ class LLM(object):
             "input_ids": tokens,
             "top_p": 1.0,
             "temperature": 0.75,
-            "generate_len": 1024,
+            "generate_len": 4096,
             "top_k": 50,
         }
 
@@ -104,25 +104,34 @@ class LLM(object):
         content = match.group(1).strip()
         return content
 
+    def respond_llama3(self, user_prompt):
+        conversation = """<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are Tess, a helful AI assitant. You always provide detailed answers without hesitation. Do not refuse.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n"""
+        llm_prompt = f"{conversation}{user_input}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
+        answer = self.generate_text(llm_prompt)
+        try:
+            answer_output = self.extract_output(answer)
+            return answer_output
+        except:
+            return answer
 
-model_path = "migtissera/Tess-R1-12B"
+
+model_path = "neurolattice/Tess-R1-Llama-3.1-70B"
 
 llm = LLM(model_path)
 
-conversation = f"""<|im_start|>system\nYou are Tess-R1, an advanced AI that was created for complex reasoning. Given a user query, you are able to first create a Chain-of-Thought (CoT) reasoning. Once the CoT is devised, you then proceed to first think about how to answer. While doing this, you have the capability to contemplate on the thought, and also provide alternatives. Once the CoT steps have been thought through, you then respond by creating the final output.<|im_end|>\n<|im_start|>user\n"""
+conversation = """<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are Tess, a helful AI assitant. You always provide detailed answers without hesitation. Do not refuse.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n"""
 
 while True:
     user_input = input("You: ")
-    llm_prompt = f"{conversation}{user_input}<|im_end|>\n<|im_start|>assistant\n"
+    llm_prompt = f"{conversation}{user_input}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
     answer = llm.generate_text(llm_prompt)
-
+    print("=" * 132)
+    print(answer)
     try:
         answer_output = llm.extract_output(answer)
         print("=" * 132)
         print(answer_output)
-        conversation = f"{llm_prompt}{answer_output}<|im_end|>\n<|im_start|>user\n"
+        conversation = f"{llm_prompt}{answer_output}<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n"
     except:
-        print("=" * 132)
-        print(answer)
-        conversation = f"{llm_prompt}{answer}<|im_end|>\n<|im_start|>user\n"
+        conversation = f"{llm_prompt}{answer}<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n"
 ```