Update app.py

app.py

@@ -359,42 +359,96 @@ def is_content_unique(new_content, existing_contents, similarity_threshold=0.8):
     return True
 
 def assess_relevance_and_summarize(llm_client, query, document, temperature=0.2):
-    system_prompt = """You are a world class AI assistant. Your task is to assess whether the given text is relevant to the user's query and provide a brief summary if it is relevant."""
+    """
+    Enhanced function to assess document relevance with entity detection and URL analysis.
+    
+    Args:
+        llm_client: The LLM client instance
+        query: User's search query
+        document: Dictionary containing document info (url, content, etc.)
+        temperature: Temperature parameter for LLM
+    
+    Returns:
+        String containing relevance assessment and summary
+    """
+    # First, detect entities in the query using LLM
+    entity_detection_prompt = """Analyze the following query and identify any specific named entities (companies, people, organizations, products, etc.). Return ONLY the entities, separated by commas. If no entities are found, return 'None'.
 
-    user_prompt = f"""
 Query: {query}
 
+Entities:"""
+
+    entity_messages = [
+        {"role": "system", "content": "You are an expert at identifying named entities in text."},
+        {"role": "user", "content": entity_detection_prompt.format(query=query)}
+    ]
+
+    try:
+        entity_response = llm_client.chat_completion(
+            messages=entity_messages,
+            max_tokens=100,
+            temperature=0.1  # Lower temperature for more consistent entity detection
+        )
+        entities = entity_response.choices[0].message.content.strip()
+        
+        # Calculate URL relevance score based on entities
+        url_relevance_score = 0
+        if entities.lower() != 'none':
+            url = document['url'].lower()
+            for entity in entities.split(','):
+                entity = entity.strip().lower()
+                if entity in url:
+                    url_relevance_score += 1
+
+        # Prepare the main assessment prompt with entity and URL information
+        system_prompt = """You are a world class AI assistant specializing in document relevance assessment and summarization. Your task is to:
+1. Consider any detected entities and URL relevance
+2. Assess if the document content is relevant to the user's query
+3. Provide a relevance score and summary if relevant
+
+Use the following scoring criteria:
+- URL contains query entities: +1 point per entity
+- Content directly addresses the query topic: +2 points
+- Content contains relevant but indirect information: +1 point
+- Content is recent and up-to-date (if time-sensitive): +1 point
+- Content provides unique insights: +1 point"""
+
+        user_prompt = f"""
+Query: {query}
+Detected Entities: {entities}
+URL Relevance Score: {url_relevance_score}
+
 Document Content:
 {document['content']}
 
-Instructions:
-1. Assess if the document is relevant to the QUERY  made by the user.
-2. If relevant, summarize the main points in 1-2 sentences.
-3. If not relevant, simply state "Not relevant".
-
-Your response should be in the following format:
+Please provide your assessment in the following format:
 Relevant: [Yes/No]
+Relevance Score: [Score out of 5]
+URL Priority: [High if URL contains entities, Low if not]
 Summary: [Your 1-2 sentence summary if relevant, or "Not relevant" if not]
+Entities Mentioned: [List entities from the query that appear in the content]"""
 
-Remember to focus on financial aspects and implications in your assessment and summary.
-"""
-
-    messages = [
-        {"role": "system", "content": system_prompt},
-        {"role": "user", "content": user_prompt}
-    ]
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt}
+        ]
 
-    try:
+        # Get the final assessment
         response = llm_client.chat_completion(
             messages=messages,
-            max_tokens=150,
-            temperature=temperature,
-            top_p=0.9
+            max_tokens=250,
+            temperature=temperature
         )
+        
         return response.choices[0].message.content.strip()
+
     except Exception as e:
-        logger.error(f"Error assessing relevance and summarizing with LLM: {e}")
-        return "Error: Unable to assess relevance and summarize"
+        logger.error(f"Error in enhanced relevance assessment: {e}")
+        return f"""Relevant: No
+Relevance Score: 0
+URL Priority: Low
+Summary: Error during assessment - {str(e)}
+Entities Mentioned: None"""
 
 def scrape_full_content(url, scraper="bs4", max_chars=3000, timeout=5):
     try:
@@ -601,28 +655,63 @@ def search_and_scrape(query, chat_history, num_results=5, scraper="bs4", max_cha
         # Step 3: Assess relevance, summarize, and check for uniqueness
         relevant_documents = []
         unique_summaries = []
+        
+        # Sort scraped_content based on initial URL analysis (if entities are in URL)
         for doc in scraped_content:
             assessment = assess_relevance_and_summarize(client, rephrased_query, doc, temperature=llm_temperature)
-            relevance, summary = assessment.split('\n', 1)
             
-            if relevance.strip().lower() == "relevant: yes":
-                summary_text = summary.replace("Summary: ", "").strip()
-                
+            # Parse the structured assessment response
+            assessment_parts = {}
+            for line in assessment.split('\n'):
+                if ':' in line:
+                    key, value = line.split(':', 1)
+                    assessment_parts[key.strip()] = value.strip()
+            
+            # Extract relevant information
+            is_relevant = assessment_parts.get('Relevant', 'No').lower() == 'yes'
+            relevance_score = float(assessment_parts.get('Relevance Score', '0').split('/')[0])
+            url_priority = assessment_parts.get('URL Priority', 'Low')
+            summary_text = assessment_parts.get('Summary', 'Not relevant')
+            entities_mentioned = assessment_parts.get('Entities Mentioned', 'None')
+            
+            # Define relevance threshold
+            RELEVANCE_THRESHOLD = 2.5  # Documents must score above 2.5 out of 5 to be considered
+            
+            if is_relevant and relevance_score >= RELEVANCE_THRESHOLD:
+                # Check for content uniqueness
                 if is_content_unique(summary_text, unique_summaries):
-                    relevant_documents.append({
+                    # Create enhanced document record
+                    doc_record = {
                         "title": doc['title'],
                         "url": doc['url'],
                         "summary": summary_text,
-                        "scraper": doc['scraper']
-                    })
+                        "scraper": doc['scraper'],
+                        "relevance_score": relevance_score,
+                        "url_priority": url_priority,
+                        "entities_mentioned": entities_mentioned,
+                        "original_content": doc.get('content', '')  # Keep original content if needed
+                    }
+                    
+                    relevant_documents.append(doc_record)
                     unique_summaries.append(summary_text)
+                    logger.info(f"Added relevant document: {doc['title']} (Score: {relevance_score}, Priority: {url_priority})")
                 else:
                     logger.info(f"Skipping similar content: {doc['title']}")
-
+            else:
+                logger.info(f"Skipping irrelevant or low-scoring document: {doc['title']} (Score: {relevance_score})")
+        
+        # Sort relevant documents by relevance score and URL priority
+        relevant_documents.sort(key=lambda x: (
+            x['url_priority'] == 'High',  # True sorts before False
+            x['relevance_score']
+        ), reverse=True)
+        
         if not relevant_documents:
             logger.warning("No relevant and unique documents found.")
-            return "No relevant and unique financial news found for the given query."
-            logger.debug(f"Assessment result: {assessment}")
+            return "No relevant and unique documents found for the given query."
+        
+        logger.info(f"Found {len(relevant_documents)} relevant and unique documents")
+        logger.debug(f"Top document scores: {[(doc['title'], doc['relevance_score']) for doc in relevant_documents[:3]]}")
 
         # Step 4: Rerank documents based on similarity to query
         reranked_docs = rerank_documents(rephrased_query, relevant_documents)