Merge branch 'import_sorbobot' into main

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 SU_CSV.db filter=lfs diff=lfs merge=lfs -text
+*.pgdata filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,161 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+.ruff_cache
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

Dockerfile

@@ -0,0 +1,56 @@
+FROM postgres:14.9-bookworm
+
+WORKDIR /app
+
+RUN apt update && \
+    apt install -y --no-install-recommends \
+    build-essential \
+    python3 \
+    python3-pip \
+    python3-dev \
+    postgresql-server-dev-14 \
+    libpq-dev \
+    libblas-dev \
+    htop \
+    git
+
+COPY ./ /app/
+
+RUN pip3 install -r ./requirements.txt --break-system-packages
+
+EXPOSE 5432
+EXPOSE 7860
+
+ENV POSTGRES_USER=postgres
+ENV POSTGRES_PASSWORD=pwd
+ENV POSTGRES_DB=sorbobot
+
+# User
+RUN useradd -m -u 1000 user
+ENV HOME /home/user
+ENV PATH $HOME/.local/bin:$PATH
+
+# Install PGVector
+WORKDIR /tmp
+RUN git clone --branch v0.5.1 https://github.com/pgvector/pgvector.git
+WORKDIR /tmp/pgvector
+RUN make
+RUN make install # may need sudo
+WORKDIR $HOME
+COPY ./ $HOME
+
+COPY "execution.sh" "/usr/local/bin/"
+
+COPY ./docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/
+
+RUN chown -R user:user /var/lib/postgresql/data
+
+USER user
+
+ENTRYPOINT ["execution.sh"]
+
+STOPSIGNAL SIGINT
+
+HEALTHCHECK CMD curl --fail http://localhost:7860/_stcore/health
+
+CMD ["postgres"]

README.md

@@ -1,42 +1,13 @@
 ---
-title: TEST2
-emoji: 🦀
-colorFrom: yellow
-colorTo: blue
-sdk: streamlit
-sdk_version: 1.21.0
+title: Test Streamlit
+emoji: 📈
+colorFrom: gray
+colorTo: indigo
+sdk: docker
+app_port: 7860
+sdk_version: 1.27.2
 app_file: app.py
 pinned: false
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
-
-Malheureusement, faute de temps, je n'ai pas pu terminer le fine-tuning du modèle GPT-2 et de BERT à temps, car ce dernier prenait plus de 130 heures de calcul.
-
-Vous pouvez relancer le fine-tuning du GPT-2 en vous rendant sur le dépôt du projet et en exécutant la commande `python3 GPT2_Lorafinetune.py`.
-
-Une fois le processus de fine-tuning terminé, vous pouvez créer un modèle et le déployer sur HuggingFace pour une utilisation ultérieure. Pour ce faire :
-
-- Cliquez sur l'icône de votre profil sur HuggingFace -> New model (choisissez un nom / public ou privé) -> create model. Une fois le modèle créé, allez dans "Files" et ajoutez les fichiers nécessaires.
-
-Vous pouvez vous référer à ce tutoriel : https://huggingface.co/transformers/v4.0.1/model_sharing.html
-
-Assurez-vous qu'il n'y a pas de fichiers inutiles dans le répertoire que vous allez charger. Il doit contenir uniquement :
-
-- Un fichier config.json, qui enregistre la configuration de votre modèle.
-
-- Un fichier pytorch_model.bin, qui est la checkpoint PyTorch (sauf si vous ne pouvez pas l'avoir pour une raison quelconque).
-
-- Un fichier tf_model.h5, qui est la checkpoint TensorFlow (sauf si vous ne pouvez pas l'avoir pour une raison quelconque).
-
-- Un fichier special_tokens_map.json, qui fait partie de l'enregistrement de votre tokenizer.
-
-- Un fichier tokenizer_config.json, qui fait partie de l'enregistrement de votre tokenizer.
-
-- Des fichiers nommés vocab.json, vocab.txt, merges.txt ou similaires, qui contiennent le vocabulaire de votre tokenizer, faisant partie de l'enregistrement de votre tokenizer.
-
-- Éventuellement un fichier added_tokens.json, qui fait partie de l'enregistrement de votre tokenizer.
-
-Une fois que vous avez créé et déployé votre modèle sur HuggingFace, vous devez remplacer le modèle dans app.py par le chemin vers votre modèle.
-
-Référence : https://huggingface.co/learn/nlp-course/chapter4/2?fw=pt

app.py

@@ -1,65 +0,0 @@
-import streamlit as st
-from transformers import pipeline
-import sqlite3
-from sentence_transformers import SentenceTransformer
-from sklearn.feature_extraction.text import CountVectorizer
-from sklearn.metrics.pairwise import cosine_similarity
-import numpy as np
-pipe=pipeline('sentiment-analysis')
-
-text = """
-Welcome to SorboBot, a Hugging Face Space designed to revolutionize the way you find published articles.
-
-Powered by a full export from ScanR and Hal at Sorbonne University, SorboBot utilizes advanced language model technology to provide you with a list of published articles based on your prompt
-
-Work in progress
-
-Write your request:
-"""
-text=st.text_area(text)
-
-
-if text:
-    n_gram_range = (2, 2)
-    stop_words = "english"
-    # Extract candidate words/phrases
-    count = CountVectorizer(ngram_range=n_gram_range, stop_words=stop_words).fit([text])
-    candidates = count.get_feature_names_out()
-    model = SentenceTransformer('distilbert-base-nli-mean-tokens')
-    doc_embedding = model.encode([text])
-    candidate_embeddings = model.encode(candidates)
-    top_n = 5
-    distances = cosine_similarity(doc_embedding, candidate_embeddings)
-    keywords = [candidates[index] for index in distances.argsort()[0][-top_n:]]
-    conn = sqlite3.connect('SU_CSV.db')
-    cursor = conn.cursor()
-    
-    mots_cles_recherches = keywords
-    
-    # Création de la requête SQL
-    query = f"SELECT title_s FROM BDD_Provisoire_SU WHERE {' OR '.join(['keyword_s LIKE ?'] * len(mots_cles_recherches))}"
-    params = ['%' + mot + '%' for mot in mots_cles_recherches]
-    
-    cursor.execute(query, params)
-    resultats = cursor.fetchall()
-    
-    # Affichage des titres d'articles trouvés
-    if resultats:
-        st.write("Titles of articles corresponding to your search:")
-        for row in resultats[:3]:
-            st.json(row[0])
-    else:
-        st.write("No article found in the database\n\n")
-        st.json({})
-    
-    
-    conn.close()
-    generator = pipeline("text-generation", model="gpt2") # to modify for another model
-    txt = generator(
-        text,
-        max_length=150,
-        num_return_sequences=1,
-    )[0]["generated_text"]
-
-    st.write("Model output")
-    st.write(txt)

docker-entrypoint-initdb.d/01-restore.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+file="/docker-entrypoint-initdb.d/dump.pgdata"
+dbname=sorbobot
+
+echo "Restoring DB using $file"
+pg_restore -U postgres --dbname=$dbname --verbose --single-transaction < "$file" || exit 1

SU_CSV.db → docker-entrypoint-initdb.d/dump.pgdata

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:305722e8432c3680d70c91554f495055687387b91fb55334113c08ef63136b26
-size 248946688
+oid sha256:48d2146bac1789085d813baa28930706bcf618b516d2233d7a6146b36b4ed6e9
+size 917294274

docs/docker.md

@@ -0,0 +1,31 @@
+# Docker and Deployment Documentation
+
+## Introduction
+
+This document outlines the Docker configuration and deployment strategy for Sorbobot, a chatbot designed for Sorbonne Université to assist in locating academic experts. The application is containerized using Docker and hosted on Hugging Face Spaces.
+
+## Docker Configuration
+
+### Base Image and Dependencies
+
+The Docker environment for Sorbobot is based on a Postgres image, supplemented with necessary dependencies to run a Streamlit server. This approach integrates the database and front-end interface within the same Docker container, in order to deploy it on Hugging Face which supports only one docker image.
+
+### Database Initialization
+
+The database for Sorbobot is initialized using a dump that has been previously created and stored in a Git repository. This repository, located at [https://git.isir.upmc.fr/sorbobot/sorbobot](https://git.isir.upmc.fr/sorbobot/sorbobot), contains essential data and schema information to set up the Postgres database correctly. During the Docker container's initialization phase, this dump is used to populate the database with the required structure and data, ensuring that the chatbot has immediate access to all necessary information for expert retrieval.
+
+## Deployment Process
+
+### Hosting on Hugging Face Spaces
+
+Sorbobot is hosted on Hugging Face Spaces, a platform specifically designed for machine learning models and applications. This hosting choice offers seamless integration and effective showcasing of the chatbot's capabilities.
+
+### Continuous Deployment via Git
+
+The deployment of Sorbobot is managed through an automated process linked with Git. Every 'git push' to the repository initiates an automatic update and deployment sequence
+
+### Deployment Workflow
+
+1. **Code Updates**: Developers push the latest code changes to the Git repository.
+2. **Automatic Docker Build**: The new changes trigger the Docker build process, incorporating recent updates into the Docker container.
+3. **Deployment on Hugging Face Spaces**: Following the build, the updated version of Sorbobot is automatically deployed to Hugging Face Spaces, making it accessible to users with the latest features and improvements.

docs/sorbobot.md

@@ -0,0 +1,55 @@
+# Sorbobot: Expert Finder Chatbot Documentation
+
+## Overview
+
+Sorbobot is a chatbot designed for Sorbonne Université to assist their administration in locating academic experts within the university. This document outlines the structure, functionality, and implementation details of Sorbobot.
+
+### Context
+
+Sorbobot centers around identifying experts with precision, avoiding confusion with individuals sharing similar names. It leverages HAL unique identifiers to distinguish between experts.
+
+## System Architecture
+
+Sorbobot operates on a Retrieval Augmented Generation (RAG) system, composed of two primary steps:
+
+1. **Retrieval**: Identifies publications most similar to the user queries.
+2. **Generation**: Produces responses based on the context extracted from relevant publications.
+
+## Implementation Details
+
+### Programming Language and Libraries
+
+- **Language**: Python
+- **Frontend**: Streamlit
+- **Database**: PostgreSQL with pgvector for similarity search
+- **NLP Processing**: langchain and GPT4all libraries
+
+### Database
+
+- **Postgres with pgvector**: Used for storing data and performing similarity searches based on cosine similarity metrics.
+
+### Natural Language Processing
+
+- **Abstracts as Data Source**: The chatbot utilizes publication abstracts to identify experts.
+- **GPT4all for Word Embedding**: Converts text from author publications into word embeddings, enhancing the accuracy of expert identification.
+
+### Retrieval Process
+
+1. **Query Processing**: User queries are processed to extract key terms.
+2. **Similarity Search**: The system searches the database using pgvector to find publications with low cosine distance to the query.
+3. **Expert Identification**: The system identifies authors of these publications, ensuring unique identification of experts.
+
+### Generation Process
+
+1. **Context Extraction**: Relevant information is extracted from the identified publications.
+2. **Response Generation**: Utilizes a LLM to generate informative responses based on the extracted context.
+
+## User Interaction Flow
+
+1. **Query Submission**: Users submit queries related to their expert search.
+2. **Chatbot Processing**: Sorbobot processes the query, retrieves relevant publications, and identifies experts.
+3. **Response Presentation**: The system presents a list of experts, including unique identifiers and relevant publication abstracts.
+
+## Conclusion
+
+Sorbobot is a powerful tool for Sorbonne Université, streamlining the process of finding academic experts. Its advanced NLP capabilities, combined with a robust database and intelligent retrieval-generation framework, ensure accurate and efficient expert identification.

execution.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+bash /usr/local/bin/docker-entrypoint.sh "$@" &
+postgres &
+sleep 2
+
+streamlit run sorbobotapp/app.py --server.port=7860 --server.address=0.0.0.0

pyproject.toml

@@ -0,0 +1,91 @@
+# Packages configs
+[project]
+name = "sorbobotapp"
+version = "0.0.1"
+requires-python = ">=3.10"
+readme = "README.md"
+
+[build-system]
+requires = ["setuptools"]
+
+## coverage
+
+[tool.coverage.run]
+branch = true
+
+[tool.coverage.report]
+skip_empty = true
+fail_under = 70.00
+precision = 2
+
+## black
+
+[tool.black]
+target-version = ['py310']
+
+## ruff
+# Recommended ruff config for now, to be updated as we go along.
+[tool.ruff]
+target-version = 'py310'
+
+# See all rules at https://beta.ruff.rs/docs/rules/
+select = [
+    "E",   # pycodestyle
+    "W",   # pycodestyle
+    "F",   # Pyflakes
+    "B",   # flake8-bugbear
+    "C4",  # flake8-comprehensions
+    "D",   # flake8-docstrings
+    "I",   # isort
+    "SIM", # flake8-simplify
+    "TCH", # flake8-type-checking
+    "TID", # flake8-tidy-imports
+    "Q",   # flake8-quotes
+    "UP",  # pyupgrade
+    "PT",  # flake8-pytest-style
+    "RUF", # Ruff-specific rules
+]
+
+ignore = [
+    "E501", # "Line too long"
+    # -> line length already regulated by black
+    "PT011", # "pytest.raises() should specify expected exception"
+    # -> would imply to update tests every time you update exception message
+    "SIM102", # "Use a single `if` statement instead of nested `if` statements"
+    # -> too restrictive
+    "D100",
+]
+
+[tool.ruff.pydocstyle]
+# Automatically disable rules that are incompatible with Google docstring convention
+convention = "google"
+
+[tool.ruff.pycodestyle]
+max-doc-length = 88
+
+[tool.ruff.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.ruff.flake8-type-checking]
+strict = true
+runtime-evaluated-base-classes = ["pydantic.BaseModel"]
+# Pydantic needs to be able to evaluate types at runtime
+# see https://pypi.org/project/flake8-type-checking/ for flake8-type-checking documentation
+# see https://beta.ruff.rs/docs/settings/#flake8-type-checking-runtime-evaluated-base-classes for ruff documentation
+
+[tool.ruff.per-file-ignores]
+# Allow missing docstrings for tests
+"tests/**/*.py" = ["D100", "D103"]
+
+## mypy
+
+[tool.mypy]
+python_version = "3.10"
+# Enable all optional error checking flags, providing stricter type checking; see https://mypy.readthedocs.io/en/stable/getting_started.html#strict-mode-and-configuration
+strict = true
+
+# Type-check the interiors of functions without type annotations; if missing, mypy won't check function bodies without type hints, for instance those coming from third-party libraries
+check_untyped_defs = true
+
+# Make __init__.py file optional for package definitions; if missing, mypy requires __init__.py at packages roots, see https://mypy.readthedocs.io/en/stable/running_mypy.html#mapping-file-paths-to-modules
+explicit_package_bases = true

requirements.txt

@@ -1,11 +1,14 @@
-sentence_transformers
-scikit-learn
-torch
-streamlit==0.84.1
-Pillow
-jax[cpu]
-flax
-transformers
-huggingface_hub
-googletrans==4.0.0-rc1
-protobuf==3.20
+black==23.11.0
+gpt4all==1.0.12
+langchain==0.0.313
+openai==0.28.1
+pandas==2.1.1
+pgvector==0.2.3
+pre-commit==3.5.0
+psycopg2-binary==2.9.9
+psycopg2==2.9.9
+streamlit==1.27.2
+streamlit-chat==0.1.1
+SQLAlchemy==2.0.22
+sqlite-vss==0.1.2
+tiktoken==0.5.1

setup.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+from distutils.core import setup
+
+setup(
+    name="sorbobotapp",
+    version="0.0.1",
+    authors=["Leo Bourrel <[email protected]>"],
+    package_dir={"": "sorbobotapp"},
+)

sorbobotapp/app.py

@@ -0,0 +1,190 @@
+import json
+import os
+
+import streamlit as st
+import streamlit.components.v1 as components
+from chain import get_chain
+from chat_history import insert_chat_history, insert_chat_history_articles
+from connection import connect
+from css import load_css
+from langchain.callbacks import get_openai_callback
+from message import Message
+
+st.set_page_config(layout="wide")
+
+st.title("Sorbobot - Le futur de la recherche scientifique interactive")
+
+chat_column, doc_column = st.columns([2, 1])
+
+conn = connect()
+
+
+def initialize_session_state():
+    if "history" not in st.session_state:
+        st.session_state.history = []
+    if "token_count" not in st.session_state:
+        st.session_state.token_count = 0
+    if "conversation" not in st.session_state:
+        st.session_state.conversation = get_chain(conn)
+
+
+def send_message_callback():
+    with st.spinner("Wait for it..."):
+        with get_openai_callback() as cb:
+            human_prompt = st.session_state.human_prompt.strip()
+            if len(human_prompt) == 0:
+                return
+            llm_response = st.session_state.conversation(human_prompt)
+            st.session_state.history.append(Message("human", human_prompt))
+            st.session_state.history.append(
+                Message(
+                    "ai",
+                    llm_response["answer"],
+                    documents=llm_response["source_documents"],
+                )
+            )
+            st.session_state.token_count += cb.total_tokens
+            if os.environ.get("ENVIRONMENT") == "dev":
+                history_id = insert_chat_history(
+                    conn, human_prompt, llm_response["answer"]
+                )
+                insert_chat_history_articles(
+                    conn, history_id, llm_response["source_documents"]
+                )
+
+
+def exemple_message_callback_button(args):
+    st.session_state.human_prompt = args
+    send_message_callback()
+    st.session_state.human_prompt = ""
+
+
+def clear_history():
+    st.session_state.history.clear()
+    st.session_state.token_count = 0
+    st.session_state.conversation.memory.clear()
+
+
+load_css()
+initialize_session_state()
+
+exemples = [
+    "Who has published influential research on quantum computing?",
+    "List any prominent authors in the field of artificial intelligence ethics?",
+    "Who are the leading experts on climate change mitigation strategies?",
+]
+
+with chat_column:
+    chat_placeholder = st.container()
+    prompt_placeholder = st.form("chat-form", clear_on_submit=True)
+    information_placeholder = st.container()
+
+    with chat_placeholder:
+        div = f"""
+            <div class="chat-row">
+                <img class="chat-icon" src="https://cdn-icons-png.flaticon.com/512/1129/1129398.png" width=32 height=32>
+                <div class="chat-bubble ai-bubble">
+                    Welcome to SorboBot, a Hugging Face Space designed to revolutionize the way you find published articles. <br/>
+                    Powered by a full export from ScanR and Hal at Sorbonne University, SorboBot utilizes advanced language model technology
+                    to provide you with a list of published articles based on your prompt.
+                </div>
+            </div>
+        """
+        st.markdown(div, unsafe_allow_html=True)
+
+        for chat in st.session_state.history:
+            div = f"""
+                <div class="chat-row
+                    {'' if chat.origin == 'ai' else 'row-reverse'}">
+                    <img class="chat-icon" src="https://cdn-icons-png.flaticon.com/512/{
+                        '1129/1129398.png' if chat.origin == 'ai'
+                                    else '1077/1077012.png'}"
+                        width=32 height=32>
+                    <div class="chat-bubble
+                    {'ai-bubble' if chat.origin == 'ai' else 'human-bubble'}">
+                        &#8203;{chat.message}
+                    </div>
+                </div>
+            """
+            st.markdown(div, unsafe_allow_html=True)
+
+        for _ in range(3):
+            st.markdown("")
+
+    with prompt_placeholder:
+        st.markdown("**Chat**")
+        cols = st.columns((6, 1))
+        cols[0].text_input(
+            "Chat",
+            label_visibility="collapsed",
+            key="human_prompt",
+        )
+        cols[1].form_submit_button(
+            "Submit",
+            type="primary",
+            on_click=send_message_callback,
+        )
+
+    if st.session_state.token_count == 0:
+        information_placeholder.markdown("### Test me !")
+        for idx_exemple, exemple in enumerate(exemples):
+            information_placeholder.button(
+                exemple,
+                key=f"{idx_exemple}_button",
+                on_click=exemple_message_callback_button,
+                args=(exemple,),
+            )
+
+    st.button(
+        ":new: Start a new conversation", on_click=clear_history, type="secondary"
+    )
+
+    if os.environ.get("ENVIRONMENT") == "dev":
+        information_placeholder.caption(
+            f"""
+        Used {st.session_state.token_count} tokens \n
+        Debug Langchain conversation:
+        {st.session_state.history}
+        """
+        )
+
+    components.html(
+        """
+    <script>
+    const streamlitDoc = window.parent.document;
+
+    const buttons = Array.from(
+        streamlitDoc.querySelectorAll('.stButton > button')
+    );
+    const submitButton = buttons.find(
+        el => el.innerText === 'Submit'
+    );
+
+    streamlitDoc.addEventListener('keydown', function(e) {
+        switch (e.key) {
+            case 'Enter':
+                submitButton.click();
+                break;
+        }
+    });
+    </script>
+    """,
+        height=0,
+        width=0,
+    )
+
+with doc_column:
+    st.markdown("**Source documents**")
+    if len(st.session_state.history) > 0:
+        for doc in st.session_state.history[-1].documents:
+            doc_content = json.loads(doc.page_content)
+            doc_metadata = doc.metadata
+
+            expander = st.expander(doc_content["title"])
+            expander.markdown(
+                f"**HalID** : https://hal.science/{doc_metadata['hal_id']}"
+            )
+            expander.markdown(doc_metadata["abstract"])
+            expander.markdown(f"**Authors** : {doc_content['authors']}")
+            expander.markdown(f"**Keywords** : {doc_content['keywords']}")
+            expander.markdown(f"**Distance** : {doc_metadata['distance']}")

sorbobotapp/chain.py

@@ -0,0 +1,39 @@
+import os
+
+import sqlalchemy
+from conversation_retrieval_chain import CustomConversationalRetrievalChain
+from langchain.chains.conversation.memory import ConversationBufferMemory
+from langchain.embeddings import GPT4AllEmbeddings
+from langchain.llms import OpenAI
+from vector_store import CustomVectorStore
+
+
+def get_chain(conn: sqlalchemy.engine.Connection):
+    embeddings = GPT4AllEmbeddings()
+
+    db = CustomVectorStore(
+        embedding_function=embeddings,
+        table_name="article",
+        column_name="abstract_embedding",
+        connection=conn,
+    )
+
+    retriever = db.as_retriever()
+
+    llm = OpenAI(
+        temperature=0,
+        openai_api_key=os.environ["OPENAI_API_KEY"],
+        model="text-davinci-003",
+    )
+
+    memory = ConversationBufferMemory(
+        output_key="answer", memory_key="chat_history", return_messages=True
+    )
+    return CustomConversationalRetrievalChain.from_llm(
+        llm=llm,
+        retriever=retriever,
+        verbose=True,
+        memory=memory,
+        return_source_documents=True,
+        max_tokens_limit=3700,
+    )

sorbobotapp/chat_history.py

@@ -0,0 +1,49 @@
+import json
+from typing import List
+
+import sqlalchemy
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+
+def insert_chat_history(conn: sqlalchemy.engine.Connection, query: str, answer: str):
+    with Session(conn) as conn:
+        conn.execute(
+            text("INSERT INTO chat_history (query, answer) VALUES (:query, :answer);"),
+            [
+                {
+                    "query": query,
+                    "answer": answer,
+                }
+            ],
+        )
+        conn.commit()
+
+        result = conn.execute(
+            text("SELECT id FROM chat_history ORDER BY id DESC LIMIT 1;")
+        )
+        last_row_id = result.fetchone()[0]
+        conn.commit()
+    return last_row_id
+
+
+def insert_chat_history_articles(
+    conn: sqlalchemy.engine.Connection, chat_history_id: int, articles: List[str]
+):
+    with Session(conn) as conn:
+        conn.execute(
+            text(
+                """
+                    INSERT INTO chat_history_articles (chat_history_id, article_id)
+                    VALUES (:chat_history_id, :article_id) ON CONFLICT DO NOTHING;
+                """
+            ),
+            [
+                {
+                    "chat_history_id": chat_history_id,
+                    "article_id": article.metadata["id"],
+                }
+                for article in articles
+            ],
+        )
+        conn.commit()

sorbobotapp/connection.py

@@ -0,0 +1,10 @@
+import sqlalchemy
+
+CONNECTION_STRING = "postgresql+psycopg2://postgres@/sorbobot?host=0.0.0.0"
+
+
+def connect() -> sqlalchemy.engine.Connection:
+    engine = sqlalchemy.create_engine(CONNECTION_STRING, pool_pre_ping=True)
+
+    conn = engine.connect()
+    return conn

sorbobotapp/conversation_retrieval_chain.py

@@ -0,0 +1,100 @@
+import inspect
+import json
+from typing import Any, Dict, Optional
+
+from keyword_extraction import KeywordExtractor
+from langchain.callbacks.manager import CallbackManagerForChainRun
+from langchain.chains.conversational_retrieval.base import (
+    ConversationalRetrievalChain,
+    _get_chat_history,
+)
+from langchain.schema import Document
+
+
+class CustomConversationalRetrievalChain(ConversationalRetrievalChain):
+    keyword_extractor: KeywordExtractor = KeywordExtractor()
+
+    def _handle_docs(self, docs):
+        if len(docs) == 0:
+            return False, "No documents found. Can you rephrase ?"
+        elif len(docs) == 1:
+            return False, "Only one document found. Can you rephrase ?"
+        elif len(docs) > 10:
+            return False, "Too many documents found. Can you specify your request ?"
+        return True, ""
+
+    def rerank_documents(self, question: str, docs: list[Document]) -> list[Document]:
+        """Rerank documents based on the number of similar keywords
+
+        Args:
+            question (str): Orinal question
+            docs (list[Document]): List of documents
+
+        Returns:
+            list[Document]: List of documents sorted by the number of similar keywords
+        """
+        keywords = self.keyword_extractor(question)
+
+        for doc in docs:
+            doc.metadata["similar_keyword"] = 0
+            doc_keywords = json.loads(doc.page_content)["keywords"]
+            if doc_keywords is None:
+                continue
+            doc_keywords = doc_keywords.lower().split(",")
+
+            for kw in keywords:
+                if kw.lower() in doc_keywords:
+                    doc.metadata["similar_keyword"] += 1
+                    print("similar keyword : ", kw)
+
+        docs = sorted(docs, key=lambda x: x.metadata["similar_keyword"])
+        return docs
+
+    def _call(
+        self,
+        inputs: Dict[str, Any],
+        run_manager: Optional[CallbackManagerForChainRun] = None,
+    ) -> Dict[str, Any]:
+        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
+        question = inputs["question"]
+        get_chat_history = self.get_chat_history or _get_chat_history
+        chat_history_str = get_chat_history(inputs["chat_history"])
+
+        if chat_history_str:
+            callbacks = _run_manager.get_child()
+            new_question = self.question_generator.run(
+                question=question, chat_history=chat_history_str, callbacks=callbacks
+            )
+        else:
+            new_question = question
+        accepts_run_manager = (
+            "run_manager" in inspect.signature(self._get_docs).parameters
+        )
+        if accepts_run_manager:
+            docs = self._get_docs(new_question, inputs, run_manager=_run_manager)
+        else:
+            docs = self._get_docs(new_question, inputs)  # type: ignore[call-arg]
+
+        valid_docs, message = self._handle_docs(docs)
+        if not valid_docs:
+            return {
+                self.output_key: message,
+                "source_documents": docs,
+            }
+
+        # Add reranking
+        docs = self.rerank_documents(new_question, docs)
+
+        new_inputs = inputs.copy()
+        if self.rephrase_question:
+            new_inputs["question"] = new_question
+        new_inputs["chat_history"] = chat_history_str
+        answer = self.combine_docs_chain.run(
+            input_documents=docs, callbacks=_run_manager.get_child(), **new_inputs
+        )
+        output: Dict[str, Any] = {self.output_key: answer}
+        if self.return_source_documents:
+            output["source_documents"] = docs
+        if self.return_generated_question:
+            output["generated_question"] = new_question
+        return output

sorbobotapp/css.py

@@ -0,0 +1,7 @@
+import streamlit as st
+
+
+def load_css():
+    with open("sorbobotapp/static/styles.css", "r") as f:
+        css = f"<style>{f.read()}</style>"
+        st.markdown(css, unsafe_allow_html=True)

sorbobotapp/keyword_extraction.py

@@ -0,0 +1,58 @@
+from typing import Any
+
+from langchain.chat_models import ChatOpenAI
+from langchain.output_parsers import NumberedListOutputParser
+from langchain.prompts import ChatPromptTemplate
+from utils import str_to_list
+
+query_template = """
+You are a bi-lingual (french and english) linguistic teacher working at a top-tier university.
+We are conducting a research project that requires the extraction of keywords from chatbot queries.
+Below, you will find a query. Please identify and rank the three most important keywords or phrases (n-grams) based on their relevance to the main topic of the query.
+For each keyword or phrase, assign it to one of the following categories: ["University / Company", "Research domain", "Country", "Name", "Other"].
+An 'n-gram' refers to a contiguous sequence of words, where 'n' can be 1 for a single word, 2 for a pair of words, and so on, up to two words in length.
+Please ensure not to list more than three n-grams in total.
+Your expertise in linguistic analysis is crucial for the success of this project. Thank you for your contribution.
+
+Please attach your ranked list in the following format:
+1. Keyword/Phrase - Category
+2. Keyword/Phrase - Category
+3. Keyword/Phrase - Category
+
+You must be concise and don't need to justify your choices.
+```
+{query}
+```
+"""
+
+output_parser = NumberedListOutputParser()
+format_instructions = output_parser.get_format_instructions()
+
+
+class KeywordExtractor:
+    def __init__(self):
+        super().__init__()
+        self.model = ChatOpenAI()
+        self.prompt = ChatPromptTemplate.from_template(
+            template=query_template,
+        )
+
+        self.chain = self.prompt | self.model  # | output_parser
+
+    def __call__(
+        self, inputs: str, filter_categories: list[str] = ["Research domain"]
+    ) -> Any:
+        output = self.chain.invoke({"query": inputs})
+
+        keywords = output_parser.parse(output.content)
+
+        filtered_keywords = []
+        for keyword in keywords:
+            if " - " not in keyword:
+                continue
+
+            keyword, category = keyword.split(" - ", maxsplit=2)
+            if category in filter_categories:
+                filtered_keywords.append(keyword)
+
+        return filtered_keywords

sorbobotapp/message.py

@@ -0,0 +1,16 @@
+from dataclasses import dataclass
+from typing import List, Literal, Optional
+
+from langchain.schema import Document
+
+
+@dataclass
+class Message:
+    """Class for keeping track of a chat message."""
+
+    origin: Literal["human", "ai"]
+    message: str
+    documents: Optional[List[Document]] = None
+
+    def __repr__(self) -> str:
+        return f"Message(origin={self.origin}, message={self.message})"

sorbobotapp/model.py

@@ -0,0 +1,17 @@
+import sqlalchemy
+from pgvector.sqlalchemy import Vector
+from sqlalchemy.orm import declarative_base
+
+Base = declarative_base()  # type: Any
+
+
+class Article(Base):
+    """Embedding store."""
+
+    __tablename__ = "article"
+
+    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, nullable=False)
+    title = sqlalchemy.Column(sqlalchemy.String, nullable=True)
+    abstract = sqlalchemy.Column(sqlalchemy.String, nullable=True)
+    embedding: Vector = sqlalchemy.Column("abstract_embedding", Vector(None))
+    doi = sqlalchemy.Column(sqlalchemy.String, nullable=True)

sorbobotapp/models/article.py

@@ -0,0 +1,17 @@
+import sqlalchemy
+from pgvector.sqlalchemy import Vector
+from sqlalchemy.orm import declarative_base
+
+Base = declarative_base()  # type: Any
+
+
+class Article(Base):
+    """Embedding store."""
+
+    __tablename__ = "article"
+
+    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, nullable=False)
+    title = sqlalchemy.Column(sqlalchemy.String, nullable=True)
+    abstract = sqlalchemy.Column(sqlalchemy.String, nullable=True)
+    embedding: Vector = sqlalchemy.Column("abstract_embedding", Vector(None))
+    doi = sqlalchemy.Column(sqlalchemy.String, nullable=True)

sorbobotapp/models/distance.py

@@ -0,0 +1,15 @@
+import enum
+
+distance_strategy_limit = {
+    "l2": 1.05,
+    "cosine": 0.55,
+    "inner": 1.0,
+}
+
+
+class DistanceStrategy(str, enum.Enum):
+    """Enumerator of the Distance strategies."""
+
+    EUCLIDEAN = "l2"
+    COSINE = "cosine"
+    MAX_INNER_PRODUCT = "inner"

sorbobotapp/static/styles.css

@@ -0,0 +1,36 @@
+.chat-row {
+    display: flex;
+    margin: 5px;
+    width: 100%;
+}
+
+.row-reverse {
+    flex-direction: row-reverse;
+}
+
+.appview-container .main .block-container {
+    padding-top: 2rem;
+}
+
+.chat-bubble {
+    font-family: "Source Sans Pro", sans-serif, "Segoe UI", "Roboto", sans-serif;
+    border: 1px solid transparent;
+    padding: 5px 10px;
+    margin: 0px 7px;
+    max-width: 70%;
+}
+
+.ai-bubble {
+    background: rgb(240, 242, 246);
+    border-radius: 10px;
+}
+
+.human-bubble {
+    background: linear-gradient(135deg, rgb(0, 178, 255) 0%, rgb(0, 106, 255) 100%);
+    color: white;
+    border-radius: 20px;
+}
+
+.chat-icon {
+    border-radius: 5px;
+}

sorbobotapp/utils.py

@@ -0,0 +1,20 @@
+import re
+
+
+def str_to_list(str_input: str) -> list[str]:
+    if isinstance(str_input, list):
+        return str_input
+
+    splits = re.split(r"', '|\", \"|', \"|\", '", str_input)
+    splits = [
+        split.removeprefix("[")
+        .removesuffix("]")
+        .removeprefix("(")
+        .removesuffix(")")
+        .removeprefix("'")
+        .removesuffix("'")
+        .removeprefix('"')
+        .removesuffix('"')
+        for split in splits
+    ]
+    return splits

sorbobotapp/vector_store.py

@@ -0,0 +1,364 @@
+from __future__ import annotations
+
+import contextlib
+import json
+import logging
+from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, Type
+
+import pandas as pd
+import sqlalchemy
+from langchain.docstore.document import Document
+from langchain.schema.embeddings import Embeddings
+from langchain.vectorstores.base import VectorStore
+from models.article import Article
+from models.distance import DistanceStrategy, distance_strategy_limit
+from sqlalchemy import delete, text
+from sqlalchemy.orm import Session
+from utils import str_to_list
+
+DEFAULT_DISTANCE_STRATEGY = DistanceStrategy.COSINE
+
+_LANGCHAIN_DEFAULT_COLLECTION_NAME = "langchain"
+
+
+def _results_to_docs(docs_and_scores: Any) -> List[Document]:
+    """Return docs from docs and scores."""
+    return [doc for doc, _ in docs_and_scores]
+
+
+class CustomVectorStore(VectorStore):
+    """`Postgres`/`PGVector` vector store.
+
+    To use, you should have the ``pgvector`` python package installed.
+
+    Args:
+        connection: Postgres connection string.
+        embedding_function: Any embedding function implementing
+            `langchain.embeddings.base.Embeddings` interface.
+        table_name: The name of the collection to use. (default: langchain)
+            NOTE: This is not the name of the table, but the name of the collection.
+            The tables will be created when initializing the store (if not exists)
+            So, make sure the user has the right permissions to create tables.
+        distance_strategy: The distance strategy to use. (default: COSINE)
+        pre_delete_collection: If True, will delete the collection if it exists.
+            (default: False). Useful for testing.
+
+    Example:
+        .. code-block:: python
+
+            from langchain.vectorstores import PGVector
+            from langchain.embeddings.openai import OpenAIEmbeddings
+
+            COLLECTION_NAME = "state_of_the_union_test"
+            embeddings = OpenAIEmbeddings()
+            vectorestore = PGVector.from_documents(
+                embedding=embeddings,
+                documents=docs,
+                table_name=COLLECTION_NAME,
+                connection=connection,
+            )
+
+
+    """
+
+    def __init__(
+        self,
+        connection: sqlalchemy.engine.Connection,
+        embedding_function: Embeddings,
+        table_name: str,
+        column_name: str,
+        collection_metadata: Optional[dict] = None,
+        distance_strategy: DistanceStrategy = DEFAULT_DISTANCE_STRATEGY,
+        pre_delete_collection: bool = False,
+        logger: Optional[logging.Logger] = None,
+    ) -> None:
+        self._conn = connection
+        self.embedding_function = embedding_function
+        self.table_name = table_name
+        self.column_name = column_name
+        self.collection_metadata = collection_metadata
+        self._distance_strategy = distance_strategy
+        self.pre_delete_collection = pre_delete_collection
+        self.logger = logger or logging.getLogger(__name__)
+        self.__post_init__()
+
+    def __post_init__(
+        self,
+    ) -> None:
+        """
+        Initialize the store.
+        """
+        # self._conn = self.connect()
+
+        self.EmbeddingStore = Article
+
+    @property
+    def embeddings(self) -> Embeddings:
+        return self.embedding_function
+
+    @contextlib.contextmanager
+    def _make_session(self) -> Generator[Session, None, None]:
+        """Create a context manager for the session, bind to _conn string."""
+        yield Session(self._conn)
+
+    def add_embeddings(
+        self,
+        texts: Iterable[str],
+        embeddings: List[List[float]],
+        metadatas: Optional[List[dict]] = None,
+        ids: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> List[str]:
+        """Add embeddings to the vectorstore.
+
+        Args:
+            texts: Iterable of strings to add to the vectorstore.
+            embeddings: List of list of embedding vectors.
+            metadatas: List of metadatas associated with the texts.
+            kwargs: vectorstore specific parameters
+        """
+        if not metadatas:
+            metadatas = [{} for _ in texts]
+
+        with Session(self._conn) as session:
+            for txt, metadata, embedding, id in zip(texts, metadatas, embeddings, ids):
+                embedding_store = self.EmbeddingStore(
+                    embedding=embedding,
+                    document=txt,
+                    cmetadata=metadata,
+                    custom_id=id,
+                )
+                session.add(embedding_store)
+            session.commit()
+
+        return ids
+
+    def add_texts(
+        self,
+        texts: Iterable[str],
+        metadatas: Optional[List[dict]] = None,
+        ids: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> List[str]:
+        """Run more texts through the embeddings and add to the vectorstore.
+
+        Args:
+            texts: Iterable of strings to add to the vectorstore.
+            metadatas: Optional list of metadatas associated with the texts.
+            kwargs: vectorstore specific parameters
+
+        Returns:
+            List of ids from adding the texts into the vectorstore.
+        """
+        embeddings = self.embedding_function.embed_documents(list(texts))
+        return self.add_embeddings(
+            texts=texts, embeddings=embeddings, metadatas=metadatas, ids=ids, **kwargs
+        )
+
+    def similarity_search(
+        self,
+        query: str,
+        k: int = 4,
+        filter: Optional[dict] = None,
+        **kwargs: Any,
+    ) -> List[Document]:
+        """Run similarity search with PGVector with distance.
+
+        Args:
+            query (str): Query text to search for.
+            k (int): Number of results to return. Defaults to 4.
+            filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None.
+
+        Returns:
+            List of Documents most similar to the query.
+        """
+        embedding = self.embedding_function.embed_query(text=query)
+        return self.similarity_search_by_vector(
+            embedding=embedding,
+            k=k,
+        )
+
+    def similarity_search_with_score(
+        self,
+        query: str,
+        k: int = 4,
+        filter: Optional[dict] = None,
+    ) -> List[Tuple[Document, float]]:
+        """Return docs most similar to query.
+
+        Args:
+            query: Text to look up documents similar to.
+            k: Number of Documents to return. Defaults to 4.
+            filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None.
+
+        Returns:
+            List of Documents most similar to the query and score for each.
+        """
+        embedding = self.embedding_function.embed_query(query)
+        docs = self.similarity_search_with_score_by_vector(embedding=embedding, k=k)
+        return docs
+
+    @property
+    def distance_strategy(self) -> str | None:
+        if self._distance_strategy == DistanceStrategy.EUCLIDEAN:
+            return "<->"
+        elif self._distance_strategy == DistanceStrategy.COSINE:
+            return "<=>"
+        elif self._distance_strategy == DistanceStrategy.MAX_INNER_PRODUCT:
+            return "<#>"
+        else:
+            raise ValueError(
+                f"Got unexpected value for distance: {self._distance_strategy}. "
+                f"Should be one of {', '.join([ds.value for ds in DistanceStrategy])}."
+            )
+
+    def similarity_search_with_score_by_vector(
+        self,
+        embedding: List[float],
+        k: int = 4,
+    ) -> List[Tuple[Document, float]]:
+        results = self.__query_collection(embedding=embedding, k=k)
+
+        return self._results_to_docs_and_scores(results)
+
+    @staticmethod
+    def _fetch_title(title: str, abstract: str):
+        if len(title) > 0:
+            return title
+        return abstract.split(".")[0]
+
+    def _results_to_docs_and_scores(self, results: Any) -> List[Tuple[Document, float]]:
+        """Return docs and scores from results."""
+        docs = [
+            (
+                Document(
+                    page_content=json.dumps(
+                        {
+                            "title": self._fetch_title(
+                                result["title"][0], result["abstract"][0]
+                            ),
+                            "authors": result["authors"],
+                            "keywords": result["keywords"],
+                        }
+                    ),
+                    metadata={
+                        "id": result["id"],
+                        "doi": result["doi"],
+                        "hal_id": result["hal_id"],
+                        "distance": result["distance"],
+                        "abstract": result["abstract"][0],
+                    },
+                ),
+                result["distance"] if self.embedding_function is not None else None,
+            )
+            for result in results
+        ]
+        return docs
+
+    def __query_collection(
+        self,
+        embedding: List[float],
+        k: int = 4,
+    ) -> List[Any]:
+        """Query the collection."""
+
+        limit = distance_strategy_limit[self._distance_strategy]
+        with Session(self._conn) as session:
+            results = session.execute(
+                text(
+                    f"""
+                    select
+                        a.id,
+                        a.title_en,
+                        a.doi,
+                        a.hal_id,
+                        a.abstract_en,
+                        string_agg(distinct keyword."name", ', ') as keywords,
+                        string_agg(distinct author."name", ', ') as authors,
+                        abstract_embedding_en {self.distance_strategy} '{str(embedding)}' as distance
+                    from article a
+                    left join article_keyword ON article_keyword.article_id = a.id
+                    left join keyword on article_keyword.keyword_id = keyword.id
+                    left join article_author ON article_author.article_id = a.id
+                    left join author on author.id = article_author.author_id
+                    where
+                        abstract_en != '' and
+                        abstract_en != 'None' and
+                        abstract_embedding_en {self.distance_strategy} '{str(embedding)}' < {limit}
+                    GROUP BY a.id
+                    ORDER BY distance
+                    LIMIT 100;
+                """
+                )
+            )
+            results = results.fetchall()
+            results = pd.DataFrame(
+                results,
+                columns=[
+                    "id",
+                    "title",
+                    "doi",
+                    "hal_id",
+                    "abstract",
+                    "keywords",
+                    "authors",
+                    "distance",
+                ],
+            )
+            results["abstract"] = results["abstract"].apply(str_to_list)
+            results["title"] = results["title"].apply(str_to_list)
+        results = results.to_dict(orient="records")
+        return results
+
+    def similarity_search_by_vector(
+        self,
+        embedding: List[float],
+        k: int = 4,
+        **kwargs: Any,
+    ) -> List[Document]:
+        """Return docs most similar to embedding vector.
+
+        Args:
+            embedding: Embedding to look up documents similar to.
+            k: Number of Documents to return. Defaults to 4.
+            filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None.
+
+        Returns:
+            List of Documents most similar to the query vector.
+        """
+        docs_and_scores = self.similarity_search_with_score_by_vector(
+            embedding=embedding, k=k
+        )
+        return _results_to_docs(docs_and_scores)
+
+    @classmethod
+    def from_texts(
+        cls: Type[CustomVectorStore],
+        texts: List[str],
+        embedding: Embeddings,
+        metadatas: Optional[List[dict]] = None,
+        table_name: str = _LANGCHAIN_DEFAULT_COLLECTION_NAME,
+        distance_strategy: DistanceStrategy = DEFAULT_DISTANCE_STRATEGY,
+        ids: Optional[List[str]] = None,
+        pre_delete_collection: bool = False,
+        **kwargs: Any,
+    ) -> CustomVectorStore:
+        """
+        Return VectorStore initialized from texts and embeddings.
+        Postgres connection string is required
+        "Either pass it as a parameter
+        or set the PGVECTOR_CONNECTION_STRING environment variable.
+        """
+        embeddings = embedding.embed_documents(list(texts))
+
+        return cls.__from(
+            texts,
+            embeddings,
+            embedding,
+            metadatas=metadatas,
+            ids=ids,
+            table_name=table_name,
+            distance_strategy=distance_strategy,
+            pre_delete_collection=pre_delete_collection,
+            **kwargs,
+        )