Try with cublas

app.py

@@ -1,4 +1,6 @@
 import os
+import nvidia.cublas.lib
+import nvidia.cudnn.lib
 import tempfile
 from pathlib import Path
 import subprocess
@@ -31,6 +33,13 @@ inference_model = None
 
 LLAMA_CKPT_PATH = "meta-llama/Meta-Llama-3.1-8B-Instruct"
 
+
+cublas_path = os.path.dirname(nvidia.cublas.lib.__file__)
+cudnn_path = os.path.dirname(nvidia.cudnn.lib.__file__)
+
+ld_library_path = f"{cublas_path}:{cudnn_path}"
+os.environ["LD_LIBRARY_PATH"] = ld_library_path
+
 def install_cudnn():
   """Installs specific versions of libcudnn and configures torch for TF32."""
 

requirements.txt

@@ -1,5 +1,7 @@
 torch==2.5.1+cu121
 torchaudio==2.5.1+cu121
+nvidia-cublas-cu11
+nvidia-cudnn-cu11
 faster-whisper
 pyannote.audio
 torchmetrics