embedding.py

import logging
import warnings
from langchain.document_loaders import PyPDFLoader, DirectoryLoader
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import FAISS

# Logging Setup
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Suppress Warnings
warnings.filterwarnings("ignore", category=FutureWarning)
warnings.filterwarnings("ignore", category=DeprecationWarning)

# Load documents from a directory containing PDFs
loader = DirectoryLoader('ipc-data', glob="./*.pdf", loader_cls=PyPDFLoader)
documents = loader.load()
logger.info(f"Loaded {len(documents)} documents from 'ipc-data'.")

# Split documents into chunks for embedding
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1024, chunk_overlap=200)
texts = text_splitter.split_documents(documents)
logger.info(f"Split documents into {len(texts)} text chunks.")

# Initialize embeddings using HuggingFace
embeddings = HuggingFaceEmbeddings(
    model_name="nomic-ai/nomic-embed-text-v1",
    model_kwargs={"trust_remote_code": True, "revision": "289f532e14dbbbd5a04753fa58739e9ba766f3c7"},
)
logger.info("Initialized HuggingFace embeddings.")

# Create FAISS vector store and save it
faiss_db = FAISS.from_documents(texts, embeddings)
faiss_db.save_local("ipc_vector_db")
logger.info("FAISS vector database created and saved as 'ipc_vector_db'.")