性能検査

ml_api/api.py

@@ -38,7 +38,7 @@ def chat(request, data: ChatInput):
     with torch.no_grad():
         generated_ids = model.generate(
             **inputs,
-            max_new_tokens=256,
+            max_new_tokens=1024,
             do_sample=True,
             temperature=0.7,
             top_p=0.9,

ml_api/model_loader.py

@@ -28,7 +28,7 @@ def get_model():
 
         _model = AutoModelForCausalLM.from_pretrained(
             model_name,
-            torch_dtype=dtype,
+            dtype=dtype,
             trust_remote_code=True,
         )
         print(f"Model Loaded! (dtype: {dtype})")