milvas schema addition

app.py

@@ -2,6 +2,7 @@ import gradio as gr
 import os
 
 from loaddataset import ExtractRagBenchData
+from createmilvusschema import CreateMilvusDbSchema
 from model import generate_response
 from huggingface_hub import login
 from huggingface_hub import whoami
@@ -13,6 +14,12 @@ login(hf_token)
 
 rag_extracted_data = ExtractRagBenchData()
 
+#invoke create milvus db function
+try:
+    db_collection = CreateMilvusDbSchema()
+except Exception as e:
+    print(f"Error creating Milvus DB schema: {e}")
+
 print(rag_extracted_data.head(5))
 
 def chatbot(prompt):

createmilvusschema.py

@@ -0,0 +1,47 @@
+import os
+from pymilvus import connections, FieldSchema, CollectionSchema, DataType, Collection
+
+milvus_token = os.getenv("MILVUS_TOKEN")
+
+COLLECTION_NAME = "final_ragbench_document_embeddings"
+MILVUS_CLOUD_URI = "https://in03-7b4da1b7b588a88.serverless.gcp-us-west1.cloud.zilliz.com"
+connections.connect("default", uri=MILVUS_CLOUD_URI, token=milvus_token)
+
+#Function to create milvus db schema to insert the data
+def CreateMilvusDbSchema():
+
+    connections.connect("default", uri=MILVUS_CLOUD_URI, token=milvus_token)
+    print(connections.get_connection_addr("default"))
+
+    # Define the fields for the collection
+    fields = [
+        FieldSchema(name="chunk_doc_id", dtype=DataType.VARCHAR, max_length=350, is_primary=True, auto_id=False),  # Primary Key
+        FieldSchema(name="doc_id", dtype=DataType.VARCHAR, max_length=300),  # Document ID
+        FieldSchema(name="chunk_embedding", dtype=DataType.FLOAT_VECTOR, dim=384),  # Vector Field (embedding)
+        FieldSchema(name="context_relevance", dtype=DataType.FLOAT),  # Context Relevance Score
+        FieldSchema(name="context_utilization", dtype=DataType.FLOAT),  # Context Utilization Score
+        FieldSchema(name="adherence", dtype=DataType.FLOAT),  # Adherence Score
+        FieldSchema(name="dataset_name", dtype=DataType.VARCHAR, max_length=300),  # Dataset Name
+        FieldSchema(name="relevance_score", dtype=DataType.FLOAT),  # Relevance Score
+        FieldSchema(name="utilization_score", dtype=DataType.FLOAT),  # Utilization Score
+        FieldSchema(name="completeness_score", dtype=DataType.FLOAT)  # Completeness Score
+    ]
+
+    # Define the collection schema
+    schema = CollectionSchema(fields, description="RAG Bench document vector collection")
+
+    # Create the collection in Milvus
+    collection = Collection(name=COLLECTION_NAME, schema=schema)
+
+    # Create an optimized index for fast vector search
+    collection.create_index(
+        "chunk_embedding",
+        {
+            "index_type": "HNSW",  # Hierarchical Navigable Small World (HNSW) index
+            "metric_type": "COSINE",  # Cosine similarity for vector search
+            "params": {"M": 16, "efConstruction": 200}  # HNSW parameters
+        }
+    )
+
+    print(f"Collection '{COLLECTION_NAME}' created successfully.")
+    return collection

crudmilvus.py

@@ -9,4 +9,5 @@ MILVUS_CLOUD_URI = "https://in03-7b4da1b7b588a88.serverless.gcp-us-west1.cloud.z
 connections.connect("default", uri=MILVUS_CLOUD_URI, token=milvus_token)
 
 # Verify connection
-print(connections.get_connection_addr("default"))
+print(connections.get_connection_addr("default"))
+