Update app.py

app.py

@@ -8,6 +8,7 @@ import nltk
 import nltkmodule
 from newspaper import Article
 from nltk.tokenize import sent_tokenize
+import xml.etree.ElementTree as ET
 from sentence_transformers import SentenceTransformer, util
 import spacy
 import en_core_sci_lg
@@ -52,23 +53,46 @@ def get_keybert_query(text, top_n=10):
     return query
 
 # --- PubMed retrieval ---
-def retrieve_pubmed_abstracts_simple(query, n=100):
+
+
+def retrieve_pubmed_abstracts_simple(text, n=100, fallback_headline=None):
+    query = get_keybert_query(text, top_n=7)
+    if not query or query.strip() == '""':
+        query = fallback_headline
+    print("Trying PubMed query:", query)
     ncbi_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
-    search_url = f"{ncbi_url}esearch.fcgi?db=pubmed&term={query}&retmax={n}&sort=relevance&retmode=xml"
-    r = requests.get(search_url)
-    root = ET.fromstring(r.text)
-    pmids = [el.text for el in root.findall('.//Id')]
-    if not pmids:
-        return [], []
-    ids = ','.join(pmids)
-    fetch_url = f"{ncbi_url}efetch.fcgi?db=pubmed&id={ids}&rettype=abstract&retmode=xml&retmax={n}&sort=relevance"
-    resp = requests.get(fetch_url)
-    root2 = ET.fromstring(resp.text)
-    titles = [a.text for a in root2.findall('.//ArticleTitle')]
-    abstracts = [b.text for b in root2.findall('.//AbstractText')]
-    return titles, abstracts
+    tried_queries = [q for q in [query, fallback_headline, text] if q]
+
+    for q in tried_queries:
+        # Always request XML, never parse as JSON or HTML
+        search_url = f"{ncbi_url}esearch.fcgi?db=pubmed&term={q}&retmax={n}&sort=relevance&retmode=xml"
+        r = requests.get(search_url)
+        try:
+            root = ET.fromstring(r.text)
+            pmids = [el.text for el in root.findall('.//Id')]
+        except Exception as e:
+            print(f"Failed to parse PMIDs for query '{q}': {e}")
+            pmids = []
+        print(f"Query: {q} => {len(pmids)} PMIDs")
+        if pmids:
+            ids = ','.join(pmids)
+            fetch_url = f"{ncbi_url}efetch.fcgi?db=pubmed&id={ids}&rettype=abstract&retmode=xml&retmax={n}&sort=relevance"
+            resp = requests.get(fetch_url)
+            try:
+                root2 = ET.fromstring(resp.text)
+                titles = [a.text for a in root2.findall('.//ArticleTitle')]
+                abstracts = [b.text for b in root2.findall('.//AbstractText')]
+            except Exception as e:
+                print(f"Failed to parse titles/abstracts for query '{q}': {e}")
+                titles, abstracts = [], []
+            # Sanitize output
+            if not abstracts:
+                abstracts = [""] * len(titles)
+            titles = [re.sub(r"\s+", " ", t).strip() if t else "" for t in titles]
+            abstracts = [re.sub(r"\s+", " ", a).strip() if a else "" for a in abstracts]
+            return titles, abstracts
+    return [], []
 
-    #return [], []
 
 # --- Claim extraction ---
 indicator_phrases = [