Update handler.py

handler.py

@@ -13,15 +13,10 @@ device = "cuda" if torch.cuda.is_available() else "cpu"
 class EndpointHandler():
     def __init__(self, path=""):
         config = PeftConfig.from_pretrained(path)
-    
-        bnb_config = BitsAndBytesConfig(
-            bnb_4bit_quant_type="nf4",
-            bnb_4bit_compute_dtype=torch.float16,
-        )
 
         model = AutoModelForCausalLM.from_pretrained(
             model_name,
-            quantization_config=bnb_config,
+            load_in_4bit=True,
             trust_remote_code=True,
             device_map="auto"
         )