Update app.py

app.py

@@ -5,15 +5,17 @@ import requests
 import gradio as gr
 import numpy as np
 import nltk 
-import nltkmodule
 from newspaper import Article
 from nltk.tokenize import sent_tokenize
-import xml.etree.ElementTree as ET
 from sentence_transformers import SentenceTransformer, util
 import spacy
 import en_core_sci_lg
 from transformers import AutoTokenizer, AutoModelForSequenceClassification, pipeline
 import torch
+import nltkmodule
+
+# Download NLTK punkt if not present
+#nltk.download('punkt')
 
 # --- Models (load once, globally) ---
 scispacy = en_core_sci_lg.load()
@@ -27,13 +29,35 @@ NLI_LABELS = ['CONTRADICTION', 'NEUTRAL', 'ENTAILMENT']
 PUBMED_N = 100
 TOP_ABSTRACTS = 10
 
-# --- Summarizer model options ---
-model_options = {
-    "Llama-3.2-1B-Instruct (Meta, gated)": "meta-llama/Llama-3.2-1B-Instruct",
-    "Gemma-3-1B-it (Google, gated)": "google/gemma-3-1b-it",
-    "TinyLlama-1.1B-Chat (Open)": "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
-}
-pipe_cache = {}
+# --- Sentence section classifier model (BioBert-PubMed200kRCT) ---
+EVIDENCE_MODEL = "pritamdeka/BioBert-PubMed200kRCT"
+evidence_tokenizer = AutoTokenizer.from_pretrained(EVIDENCE_MODEL)
+evidence_model = AutoModelForSequenceClassification.from_pretrained(EVIDENCE_MODEL)
+label_map = {0: "BACKGROUND", 1: "OBJECTIVE", 2: "METHODS", 3: "RESULTS", 4: "CONCLUSIONS"}
+
+def extract_evidence_sentences_from_abstract(abstract, keep_labels=("RESULTS", "CONCLUSIONS")):
+    sents = sent_tokenize(abstract)
+    evidence_sents = []
+    for s in sents:
+        inputs = evidence_tokenizer(s, return_tensors="pt", truncation=True, padding=True)
+        with torch.no_grad():
+            logits = evidence_model(**inputs).logits
+            pred = torch.argmax(logits, dim=1).item()
+            label = label_map[pred]
+            if label in keep_labels:
+                evidence_sents.append((label, s))
+    return evidence_sents
+
+# --- Europe PMC retrieval ---
+def retrieve_europepmc_abstracts_simple(text, n=TOP_ABSTRACTS):
+    query = get_keybert_query(text, top_n=7)
+    print("Trying Europe PMC query:", query)
+    url = f'https://www.ebi.ac.uk/europepmc/webservices/rest/search?query={query}&resulttype=core&format=json&pageSize={n}'
+    r = requests.get(url)
+    results = r.json().get('resultList', {}).get('result', [])
+    titles = [res.get('title', '') for res in results]
+    abstracts = [res.get('abstractText', '') for res in results]
+    return titles, abstracts
 
 # --- Utility: get robust keybert-style query ---
 def get_keybert_query(text, top_n=10):
@@ -52,48 +76,6 @@ def get_keybert_query(text, top_n=10):
     query = " OR ".join(f'"{kw}"' for kw in keywords)
     return query
 
-# --- PubMed retrieval ---
-
-
-def retrieve_pubmed_abstracts_simple(text, n=100, fallback_headline=None):
-    query = get_keybert_query(text, top_n=7)
-    if not query or query.strip() == '""':
-        query = fallback_headline
-    print("Trying PubMed query:", query)
-    ncbi_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
-    tried_queries = [q for q in [query, fallback_headline, text] if q]
-
-    for q in tried_queries:
-        # Always request XML, never parse as JSON or HTML
-        search_url = f"{ncbi_url}esearch.fcgi?db=pubmed&term={q}&retmax={n}&sort=relevance&retmode=xml"
-        r = requests.get(search_url)
-        try:
-            root = ET.fromstring(r.text)
-            pmids = [el.text for el in root.findall('.//Id')]
-        except Exception as e:
-            print(f"Failed to parse PMIDs for query '{q}': {e}")
-            pmids = []
-        print(f"Query: {q} => {len(pmids)} PMIDs")
-        if pmids:
-            ids = ','.join(pmids)
-            fetch_url = f"{ncbi_url}efetch.fcgi?db=pubmed&id={ids}&rettype=abstract&retmode=xml&retmax={n}&sort=relevance"
-            resp = requests.get(fetch_url)
-            try:
-                root2 = ET.fromstring(resp.text)
-                titles = [a.text for a in root2.findall('.//ArticleTitle')]
-                abstracts = [b.text for b in root2.findall('.//AbstractText')]
-            except Exception as e:
-                print(f"Failed to parse titles/abstracts for query '{q}': {e}")
-                titles, abstracts = [], []
-            # Sanitize output
-            if not abstracts:
-                abstracts = [""] * len(titles)
-            titles = [re.sub(r"\s+", " ", t).strip() if t else "" for t in titles]
-            abstracts = [re.sub(r"\s+", " ", a).strip() if a else "" for a in abstracts]
-            return titles, abstracts
-    return [], []
-
-
 # --- Claim extraction ---
 indicator_phrases = [
     "found that", "findings suggest", "shows that", "showed that", "demonstrated", "demonstrates",
@@ -147,11 +129,10 @@ def semantic_rerank_claim_abstracts(claim, titles, abstracts, top_k=TOP_ABSTRACT
     idxs = np.argsort(-sims.cpu().numpy())[:top_k]
     return [titles[i] for i in idxs], [abstracts[i] for i in idxs]
 
-# --- NLI evidence extraction ---
-def extract_evidence_nli(claim, title, abstract):
-    sentences = sent_tokenize(abstract)
+# --- NLI evidence extraction (run only on results/conclusion sentences) ---
+def extract_evidence_nli(claim, evidence_sentences):
     evidence = []
-    for sent in sentences:
+    for sent in evidence_sentences:
         encoding = nli_tokenizer(
             sent, claim,
             return_tensors='pt',
@@ -172,7 +153,14 @@ def extract_evidence_nli(claim, title, abstract):
         })
     return evidence
 
-# --- Summarizer model loading ---
+# --- Summarizer model options ---
+model_options = {
+    "Llama-3.2-1B-Instruct (Meta, gated)": "meta-llama/Llama-3.2-1B-Instruct",
+    "Gemma-3-1B-it (Google, gated)": "google/gemma-3-1b-it",
+    "TinyLlama-1.1B-Chat (Open)": "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
+}
+pipe_cache = {}
+
 def get_summarizer(model_choice):
     model_id = model_options[model_choice]
     if model_id in pipe_cache:
@@ -249,20 +237,23 @@ def factcheck_app(article_url, summarizer_choice):
     results_html = ""
     all_results = []
     for claim in matched_claims:
-        titles, abstracts = retrieve_pubmed_abstracts_simple(claim, fallback_headline=headline)
+        titles, abstracts = retrieve_europepmc_abstracts_simple(claim)
         if not titles:
-            results_html += f"<hr><b>Claim:</b> {claim}<br><i>No PubMed results found.</i><br>"
-            all_results.append({"claim": claim, "summary": "No PubMed results found.", "evidence": []})
+            results_html += f"<hr><b>Claim:</b> {claim}<br><i>No relevant abstracts found in Europe PMC.</i><br>"
+            all_results.append({"claim": claim, "summary": "No abstracts found.", "evidence": []})
             continue
         top_titles, top_abstracts = semantic_rerank_claim_abstracts(claim, titles, abstracts)
-        idx_non_top = random.choice([i for i in range(len(titles)) if i not in [titles.index(t) for t in top_titles]]) if len(titles) > len(top_titles) else None
         evidence_results = []
         for title, abstract in zip(top_titles, top_abstracts):
-            evidence = extract_evidence_nli(claim, title, abstract)
+            # Extract evidence (results/conclusions) sentences from abstract
+            ev_sents = extract_evidence_sentences_from_abstract(abstract)
+            # If none found, fallback to all sentences
+            if ev_sents:
+                sent_list = [s for lbl, s in ev_sents]
+            else:
+                sent_list = sent_tokenize(abstract)
+            evidence = extract_evidence_nli(claim, sent_list)
             evidence_results.append({"title": title, "evidence": evidence})
-        if idx_non_top is not None:
-            control_ev = extract_evidence_nli(claim, titles[idx_non_top], abstracts[idx_non_top])
-            evidence_results.append({"title": f"(Control) {titles[idx_non_top]}", "evidence": control_ev})
         all_evidence_sentences = [ev for abs_res in evidence_results for ev in abs_res["evidence"]]
         summary = summarize_evidence_llm(claim, all_evidence_sentences, summarizer_choice)
         results_html += f"<hr><b>Claim:</b> {claim}<br><b>Layman summary:</b> {summary}<br>"
@@ -273,7 +264,7 @@ def factcheck_app(article_url, summarizer_choice):
 
 description = """
 <b>What does this app do?</b><br>
-This app extracts key scientific claims from a news article, finds the most relevant PubMed biomedical research papers using robust keyphrase extraction and semantic reranking, checks which sentences in those papers support or contradict each claim, and gives you a plain-English summary verdict.<br><br>
+This app extracts key scientific claims from a news article, finds the most relevant biomedical research papers using robust keyphrase extraction and semantic reranking, checks which sentences in those papers support or contradict each claim, and gives you a plain-English summary verdict.<br><br>
 <b>How to use it:</b><br>
 1. Paste the link to a biomedical news article.<br>
 2. Choose an AI summarizer model below. If you have no special access, use 'TinyLlama' (works for everyone).<br>
@@ -281,7 +272,7 @@ This app extracts key scientific claims from a news article, finds the most rele
 4. For each claim, you will see:<br>
 - A plain summary of what research says.<br>
 - Color-coded evidence sentences (green=support, red=contradict, gray=neutral).<br>
-- The titles of the most relevant PubMed articles.<br><br>
+- The titles of the most relevant research articles.<br><br>
 <b>Everything is 100% open source and runs on this website—no personal info or cloud API needed.</b>
 """