import os
import gradio as gr
import pandas as pd
import numpy as np
import datetime
import urllib.parse
import urllib.request
import urllib.error
import json
import time
import re
import html as html_lib
from supabase import create_client, Client
# ── Helper: build query from terms and operators ──────────────
def build_boolean_query(term1, op1, term2, op2, term3):
"""
Build a boolean query from up to three terms with operators.
"""
terms = []
if term1 and term1.strip():
terms.append(term1.strip())
if term2 and term2.strip():
terms.append(term2.strip())
if term3 and term3.strip():
terms.append(term3.strip())
if len(terms) == 0:
return ""
if len(terms) == 1:
return terms[0]
# Build query with operators
query_parts = []
for i, t in enumerate(terms):
# Put quotes around phrases with spaces
if ' ' in t and not (t.startswith('"') and t.endswith('"')):
t = f'"{t}"'
query_parts.append(t)
# Insert operators between terms
if len(query_parts) == 2:
return f"{query_parts[0]} {op1} {query_parts[1]}"
else:
return f"{query_parts[0]} {op1} {query_parts[1]} {op2} {query_parts[2]}"
def normalize_boolean_query(query: str) -> str:
"""
Replace lowercase 'and' and 'or' with uppercase operators.
"""
if not query:
return query
pattern_and = re.compile(r'(? bool:
"""Return True if query contains AND or OR operators."""
q = normalize_boolean_query(query)
return ' AND ' in q or ' OR ' in q
# ── Admin authentication using session tracking ──────────────────
_admin_sessions = {} # session_id -> timestamp
_admin_roles = {} # session_id -> "senior" or "junior"
_admin_users = {} # session_id -> current app user name
def set_admin_active(session_id, role="senior"):
if session_id:
_admin_sessions[session_id] = time.time()
_admin_roles[session_id] = role
return True
def is_admin_active(session_id):
return session_id in _admin_sessions
def get_admin_role(session_id):
return _admin_roles.get(session_id)
def is_senior_admin(session_id):
return get_admin_role(session_id) == "senior"
def is_junior_admin(session_id):
return get_admin_role(session_id) == "junior"
def set_current_app_user(session_id, user_name):
if session_id:
_admin_users[session_id] = _normalize_app_user(user_name)
def get_current_app_user(session_id):
return _normalize_app_user(_admin_users.get(session_id, ""))
def clear_current_app_user(session_id):
if session_id:
_admin_users.pop(session_id, None)
# ── Supabase credentials ──────────────────────────────────────────
url: str = os.environ.get("SUPABASE_URL")
key: str = os.environ.get("SUPABASE_KEY")
AUTH_USER = os.environ.get("UPLOAD_USER")
AUTH_PASS = os.environ.get("UPLOAD_PASS")
if not url or not key:
print("Warning: Missing Supabase credentials in Space Secrets!")
supabase = None
else:
supabase: Client = create_client(url, key)
HOMEPAGE_FILE = "homepage_content.html"
HOME_CONTACT_HTML = """
"""
def search_by_doi(doi, user_name=""):
"""Look up a DOI across CrossRef, Europe PMC and PubMed."""
doi = _extract_doi(doi)
if not doi or not re.match(r'^10\.\d{4,9}/\S+$', doi):
return "
Please enter a valid DOI, for example 10.1093/nar/gkac123.
"
# CrossRef — primary DOI metadata source
cr_url = "https://api.crossref.org/works/" + urllib.parse.quote(doi, safe='')
cr = _safe_request(cr_url)
if cr and cr.get('message'):
item = cr['message']
titles = item.get('title') or ['Untitled']
title = titles[0] if titles else 'Untitled'
auth = item.get('author') or []
names = []
for a in auth[:5]:
name = ' '.join(x for x in [a.get('given',''), a.get('family','')] if x).strip() or a.get('name','')
if name:
names.append(name)
if len(auth) > 5:
names.append('et al.')
journals = item.get('container-title') or []
journal = journals[0] if journals else ''
date_parts = (item.get('published-print') or item.get('published-online') or item.get('published') or {}).get('date-parts', [[]])
parts = date_parts[0] if date_parts else []
date = '-'.join(str(x) for x in parts) if parts else ''
url = item.get('URL') or f"https://doi.org/{doi}"
return _doi_html_result(doi, title, ', '.join(names), journal, date, url, 'CrossRef', user_name)
# Europe PMC fallback
epmc = _safe_request(
"https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=" +
urllib.parse.quote(f'DOI:"{doi}"') + "&resultType=core&pageSize=5&format=json"
)
if epmc:
items = epmc.get('resultList', {}).get('result', [])
if items:
item = items[0]
authors_list = item.get('authorList', {}).get('author', [])
names = [a.get('fullName','') for a in authors_list[:5] if a.get('fullName')]
if len(authors_list) > 5:
names.append('et al.')
pmid = item.get('pmid','')
url = f"https://pubmed.ncbi.nlm.nih.gov/{pmid}/" if pmid else f"https://doi.org/{doi}"
return _doi_html_result(
doi,
item.get('title','Untitled'),
', '.join(names),
item.get('journalTitle',''),
item.get('firstPublicationDate', str(item.get('pubYear',''))),
url,
'Europe PMC',
user_name
)
# PubMed final fallback
pm_url = (
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&retmode=json&retmax=5&term=" +
urllib.parse.quote(f'"{doi}"[AID] OR "{doi}"[DOI]')
)
pm = _safe_request(pm_url)
ids = (pm or {}).get('esearchresult', {}).get('idlist', [])
if ids:
sm = _safe_request(
"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&retmode=json&id=" + ','.join(ids)
)
if sm:
result = sm.get('result', {})
uid = result.get('uids', [ids[0]])[0]
item = result.get(uid, {})
raw = item.get('authors', [])
names = [a.get('name','') for a in raw[:5] if a.get('name')]
if len(raw) > 5:
names.append('et al.')
return _doi_html_result(
doi,
item.get('title','Untitled'),
', '.join(names),
item.get('fulljournalname', item.get('source','')),
item.get('pubdate',''),
f"https://pubmed.ncbi.nlm.nih.gov/{uid}/",
'PubMed',
user_name
)
return "
No publication metadata found for " + doi + ".
"
def get_reviewed_table_df(user_name=''):
"""Return reviewed papers belonging only to the supplied app user."""
columns = ['DOI', 'Reviewed At', 'User', 'Link']
user_name = _normalize_app_user(user_name)
if not user_name:
return pd.DataFrame(columns=columns)
rows = []
if _check_reviewed_table():
try:
res = (
supabase.table('reviewed_articles')
.select('doi,reviewed_at,user,link')
.eq('user', user_name)
.order('reviewed_at', desc=True)
.execute()
)
for r in (res.data or []):
doi = (r.get('doi') or '').strip()
link = (r.get('link') or '').strip()
if doi or link:
rows.append({
'DOI': doi,
'Reviewed At': r.get('reviewed_at') or '',
'User': r.get('user') or user_name,
'Link': link or (f'https://doi.org/{doi}' if doi else ''),
})
except Exception:
# Backward compatibility when the optional link column has not been added.
try:
res = (
supabase.table('reviewed_articles')
.select('doi,reviewed_at,user')
.eq('user', user_name)
.order('reviewed_at', desc=True)
.execute()
)
for r in (res.data or []):
doi = (r.get('doi') or '').strip()
if doi:
rows.append({
'DOI': doi,
'Reviewed At': r.get('reviewed_at') or '',
'User': r.get('user') or user_name,
'Link': f'https://doi.org/{doi}',
})
except Exception:
pass
db_dois = {str(r['DOI']).lower() for r in rows if str(r['DOI']).strip()}
db_links = {str(r['Link']).lower() for r in rows if str(r['Link']).strip()}
for doi in sorted(_reviewed_cache_by_user.get(user_name, set())):
if doi not in db_dois:
rows.append({
'DOI': doi,
'Reviewed At': 'Session only',
'User': user_name,
'Link': f'https://doi.org/{doi}',
})
for link in sorted(_reviewed_link_cache_by_user.get(user_name, set())):
if link.lower() not in db_links:
rows.append({
'DOI': _extract_doi(link),
'Reviewed At': 'Session only',
'User': user_name,
'Link': link,
})
return pd.DataFrame(rows, columns=columns)
def reviewed_row_selection(evt: gr.SelectData, current_df):
try:
row_idx = evt.index[0]
doi = str(current_df.iloc[row_idx].get('DOI', '')).strip()
link = str(current_df.iloc[row_idx].get('Link', '')).strip()
identifier = doi or link
return identifier, f'Selected reviewed article: {identifier}' if identifier else 'No article selected.'
except Exception as e:
return '', f'Selection error: {e}'
def remove_selected_reviewed_doi(selected_doi, user_name, request: gr.Request):
session_hash = getattr(request, 'session_hash', None)
if not session_hash or session_hash not in _admin_sessions:
return 'Admin login required. Please log in first.', get_reviewed_table_df(user_name), ''
if not is_senior_admin(session_hash):
return 'Senior admin access required to remove reviewed articles.', get_reviewed_table_df(user_name), ''
user_name = get_current_app_user(session_hash)
if not user_name:
return 'Please select a user first.', get_reviewed_table_df(), ''
if not selected_doi or not selected_doi.strip():
return 'Select a marked article first.', get_reviewed_table_df(user_name), ''
msg = remove_reviewed_article(selected_doi, user_name)
return msg, get_reviewed_table_df(user_name), ''
def _fetch_biorxiv(query, since_date_str, max_results=100):
"""Fetch from bioRxiv/medRxiv – basic keyword filtering (simple AND on split terms)."""
results = []
try:
today_str = datetime.date.today().strftime("%Y-%m-%d")
since_bxr = since_date_str.replace('/', '-')
# Extract keywords (remove quotes and split)
keywords = [w.strip('"').lower() for w in query.split() if len(w) > 2 and w not in ('AND', 'OR', 'NOT')]
# If query has quotes, keep them as phrase?
# We'll keep it simple: use the raw query as a phrase for title/abstract checking.
for server in ['biorxiv', 'medrxiv']:
url = f"https://api.biorxiv.org/details/{server}/{since_bxr}/{today_str}/0/json"
data = _safe_request(url)
if not data:
continue
collection = data.get('collection', [])
for item in collection:
title = item.get('title', 'Untitled')
abstract = item.get('abstract', '').lower()
# Check if all keywords appear (simple AND)
match = True
for kw in keywords:
if kw not in title.lower() and kw not in abstract:
match = False
break
if not match:
continue
doi = item.get('doi', '')
authors_str = item.get('authors', 'Unknown Authors')
author_list = [a.strip() for a in authors_str.split(';')]
names = author_list[:3]
if len(author_list) > 3:
names.append('et al.')
results.append({
'title': title,
'authors': '; '.join(names),
'journal': f"{server.capitalize()} (Preprint)",
'date': item.get('date', 'N/A'),
'url': f"https://doi.org/{doi}" if doi else f"https://www.{server}.org/",
'doi': doi,
'source': server.capitalize(),
'source_color': '#0891b2',
'source_bg': '#ecfeff',
})
if len(results) >= max_results:
break
if len(results) >= max_results:
break
return results, f"bioRxiv/medRxiv: {len(results)} results"
except Exception as e:
return results, f"bioRxiv/medRxiv: error – {str(e)}"
def _deduplicate(all_results):
"""Remove duplicates by DOI (if present) or by normalised title."""
seen_dois = set()
seen_titles = set()
unique = []
for r in all_results:
doi = r.get('doi', '').strip().lower()
title_key = r.get('title', '').strip().lower()[:80]
if doi and doi in seen_dois:
continue
if title_key and title_key in seen_titles:
continue
if doi:
seen_dois.add(doi)
if title_key:
seen_titles.add(title_key)
unique.append(r)
return unique
def _sort_results(results):
"""Sort results by date descending, with unparseable dates last."""
def date_key(r):
d = r.get('date', '') or ''
for fmt in ('%Y-%m-%d', '%Y/%m/%d', '%Y %b %d', '%Y %b', '%Y'):
try:
return datetime.datetime.strptime(d.strip()[:len(fmt)+2], fmt)
except Exception:
pass
parts = d.strip().split()
if parts:
try:
return datetime.datetime(int(parts[0]), 1, 1)
except Exception:
pass
return datetime.datetime.min
return sorted(results, key=date_key, reverse=True)
def _render_source_badge(source, color, bg):
return (
f'{source}'
)
def _run_multi_db_search(query, selected_sources, since_date, since_date_dash, window_label, user_name=""):
"""
Shared engine for running a multi-database search given an explicit
'since' date window, and rendering the resulting HTML.
"""
if not query or not query.strip():
return "
Please enter a valid search query.
"
# If query contains boolean operators, force only PubMed
if _is_boolean_query(query):
selected_sources = ['PubMed'] # override
all_results = []
source_logs = []
source_map = {
'PubMed': lambda: _fetch_pubmed(query, since_date),
'Europe PMC': lambda: _fetch_europe_pmc(query, since_date_dash),
'CrossRef': lambda: _fetch_crossref(query, since_date_dash),
'Semantic Scholar': lambda: _fetch_semantic_scholar(query, since_date_dash),
'bioRxiv / medRxiv': lambda: _fetch_biorxiv(query, since_date_dash),
}
for src_name, fetch_fn in source_map.items():
if src_name not in selected_sources:
continue
try:
res, log = fetch_fn()
all_results.extend(res)
source_logs.append(log)
except Exception as e:
source_logs.append(f"{src_name}: unexpected error – {str(e)}")
if not all_results:
log_html = ' | '.join(source_logs)
return f"""
No papers found for "{query}"
{window_label}. {log_html}
"""
unique_results = _deduplicate(all_results)
sorted_results = _sort_results(unique_results)
source_counts = {}
for r in sorted_results:
src = r.get('source', 'Unknown')
source_counts[src] = source_counts.get(src, 0) + 1
badge_row = ' '.join(
f''
f'{src}: {cnt}'
for src, cnt in source_counts.items()
)
log_html = ' | '.join(source_logs)
reviewed_dois, reviewed_links = load_reviewed_keys(user_name)
html = f"""
"""
# No reviewed DOIs or mark buttons - removed entirely
for r in sorted_results:
src_badge = _render_source_badge(r['source'], r['source_color'], r['source_bg'])
title = r.get('title', 'Untitled')
url = r.get('url', '')
authors = r.get('authors', 'Unknown Authors')
journal = r.get('journal', '')
date = r.get('date', 'N/A')
doi = _extract_doi(r.get('doi', ''))
reviewed_badge = (
' Read'
if (doi and doi in reviewed_dois) or (_normalize_reviewed_link(url) and _normalize_reviewed_link(url) in reviewed_links) else ''
)
doi_link = (
f'DOI: {doi}'
if doi else ''
)
title_html = (
f'{title}'
if url else
f'{title}'
)
html += f"""
"
return html
def fetch_multi_db_papers(query, selected_sources, user_name=""):
"""
Entry-point for the recent (past 6 months) multi-database literature search.
"""
six_months_ago = datetime.date.today() - datetime.timedelta(days=180)
since_date = six_months_ago.strftime("%Y/%m/%d")
since_date_dash = six_months_ago.strftime("%Y-%m-%d")
return _run_multi_db_search(query, selected_sources, since_date, since_date_dash, "past 6 months", user_name)
# ── Paged All-Time Literature Search ─────────────────────────────────────────
LS_FETCH_SIZE = 50
LS_PAGE_SIZE = 20
LS_YEAR_MIN = 1900
LS_YEAR_MAX = datetime.date.today().year
def _fetch_pubmed_paged(query, start_year, end_year, offset):
results, exhausted = [], False
try:
query = normalize_boolean_query(query)
query_enc = urllib.parse.quote(query)
search_url = (
f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
f"?db=pubmed&retmax={LS_FETCH_SIZE}&retstart={offset}"
f"&retmode=json&sort=date&term={query_enc}"
f"&mindate={start_year}/01/01&maxdate={end_year}/12/31"
)
search_data = _safe_request(search_url)
if not search_data:
return results, True
esresult = search_data.get('esearchresult', {})
total = int(esresult.get('count', 0))
ids = esresult.get('idlist', [])
if not ids:
return results, True
exhausted = (offset + len(ids)) >= total
summary_url = (
f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi"
f"?db=pubmed&retmode=json&id={','.join(ids)}"
)
summary_data = _safe_request(summary_url)
if not summary_data:
return results, exhausted
result_dict = summary_data.get('result', {})
for uid in result_dict.get('uids', []):
a = result_dict.get(uid, {})
raw_auth = a.get('authors', [])
names = [x.get('name', '') for x in raw_auth[:3]]
if len(raw_auth) > 3:
names.append('et al.')
results.append({
'title': a.get('title', 'Untitled'),
'authors': ', '.join(names) if names else 'Unknown Authors',
'journal': a.get('fulljournalname', a.get('source', '')),
'date': a.get('pubdate', 'N/A'),
'url': f"https://pubmed.ncbi.nlm.nih.gov/{uid}/",
'doi': a.get('elocationid', '').replace('doi: ', '').strip(),
'source': 'PubMed',
'source_color': '#dc2626', 'source_bg': '#fef2f2',
})
return results, exhausted
except Exception:
return results, True
def _fetch_europe_pmc_paged(query, start_year, end_year, offset):
results, exhausted = [], False
try:
query = normalize_boolean_query(query)
page_num = offset // LS_FETCH_SIZE
query_enc = urllib.parse.quote(query)
url = (
f"https://www.ebi.ac.uk/europepmc/webservices/rest/search"
f"?query={query_enc}&resultType=core&pageSize={LS_FETCH_SIZE}"
f"&page={page_num}&format=json&sort=P_PDATE_D%20desc"
f"&dateFrom={start_year}-01-01&dateTo={end_year}-12-31"
)
data = _safe_request(url)
if not data:
return results, True
hit_count = int(data.get('hitCount', 0))
articles = data.get('resultList', {}).get('result', [])
if not articles:
return results, True
exhausted = (offset + len(articles)) >= hit_count
for a in articles:
auth_list = a.get('authorList', {}).get('author', [])
names = [au.get('fullName', '') for au in auth_list[:3]]
if len(auth_list) > 3:
names.append('et al.')
doi = a.get('doi', '')
pmid = a.get('pmid', '')
article_url = (
f"https://pubmed.ncbi.nlm.nih.gov/{pmid}/" if pmid
else (f"https://doi.org/{doi}" if doi
else f"https://europepmc.org/article/{a.get('source','')}/{a.get('id','')}")
)
results.append({
'title': a.get('title', 'Untitled').rstrip('.'),
'authors': ', '.join(names) if names else 'Unknown Authors',
'journal': a.get('journalTitle', ''),
'date': a.get('firstPublicationDate', a.get('pubYear', 'N/A')),
'url': article_url, 'doi': doi,
'source': 'Europe PMC',
'source_color': '#7c3aed', 'source_bg': '#f5f3ff',
})
return results, exhausted
except Exception:
return results, True
def _fetch_crossref_paged(query, start_year, end_year, offset):
results, exhausted = [], False
try:
query_enc = urllib.parse.quote(query)
url = (
f"https://api.crossref.org/works"
f"?query={query_enc}"
f"&filter=from-pub-date:{start_year}-01-01,until-pub-date:{end_year}-12-31"
f"&rows={LS_FETCH_SIZE}&offset={offset}&sort=published&order=desc"
f"&select=DOI,title,author,container-title,published,type"
f"&mailto=biohub@example.com"
)
data = _safe_request(url)
if not data:
return results, True
msg = data.get('message', {})
total = msg.get('total-results', 0)
items = msg.get('items', [])
if not items:
return results, True
exhausted = (offset + len(items)) >= total
for item in items:
if item.get('type', '') not in ('journal-article', 'proceedings-article', 'posted-content'):
continue
titles = item.get('title', ['Untitled'])
title = titles[0] if titles else 'Untitled'
authors_raw = item.get('author', [])
names = []
for au in authors_raw[:3]:
fn = au.get('given', ''); ln = au.get('family', '')
names.append(f"{fn} {ln}".strip() if fn or ln else au.get('name', ''))
if len(authors_raw) > 3:
names.append('et al.')
journals = item.get('container-title', [])
journal = journals[0] if journals else ''
doi = item.get('DOI', '')
parts = (item.get('published', {}).get('date-parts', [[]])[0] or [])
pub_date = '-'.join(str(p) for p in parts) if parts else 'N/A'
results.append({
'title': title,
'authors': ', '.join(n for n in names if n) or 'Unknown Authors',
'journal': journal, 'date': pub_date,
'url': f"https://doi.org/{doi}" if doi else '',
'doi': doi, 'source': 'CrossRef',
'source_color': '#059669', 'source_bg': '#ecfdf5',
})
return results, exhausted
except Exception:
return results, True
def _fetch_semantic_scholar_paged(query, start_year, end_year, offset):
results, exhausted = [], False
try:
query_enc = urllib.parse.quote(query)
url = (
f"https://api.semanticscholar.org/graph/v1/paper/search"
f"?query={query_enc}"
f"&publicationDateOrYear={start_year}:{end_year}"
f"&fields=title,authors,venue,year,publicationDate,externalIds,openAccessPdf"
f"&limit={min(LS_FETCH_SIZE, 100)}&offset={offset}"
)
data = _safe_request(url, headers={'x-api-key': ''})
if not data:
return results, True
total = data.get('total', 0)
papers = data.get('data', [])
if not papers:
return results, True
exhausted = (offset + len(papers)) >= total
for p in papers:
authors_raw = p.get('authors', [])
names = [a.get('name', '') for a in authors_raw[:3]]
if len(authors_raw) > 3:
names.append('et al.')
ext_ids = p.get('externalIds', {})
doi = ext_ids.get('DOI', '')
paper_id = p.get('paperId', '')
pdf_info = p.get('openAccessPdf') or {}
article_url = (
pdf_info.get('url') or
(f"https://doi.org/{doi}" if doi else '') or
(f"https://www.semanticscholar.org/paper/{paper_id}" if paper_id else '')
)
pub_date = p.get('publicationDate') or str(p.get('year', 'N/A'))
results.append({
'title': p.get('title', 'Untitled'),
'authors': ', '.join(n for n in names if n) or 'Unknown Authors',
'journal': p.get('venue', ''), 'date': pub_date,
'url': article_url, 'doi': doi,
'source': 'Semantic Scholar',
'source_color': '#d97706', 'source_bg': '#fffbeb',
})
return results, exhausted
except Exception:
return results, True
def _fetch_biorxiv_paged(query, start_year, end_year, cursor):
results, exhausted = [], False
try:
start_dt = f"{start_year}-01-01"
end_dt = f"{end_year}-12-31"
# Use normalized query for keyword extraction
norm_query = normalize_boolean_query(query)
keywords = [w.strip('"').lower() for w in norm_query.split() if len(w) > 2 and w not in ('AND', 'OR', 'NOT')]
for server in ['biorxiv', 'medrxiv']:
url = f"https://api.biorxiv.org/details/{server}/{start_dt}/{end_dt}/{cursor}/json"
data = _safe_request(url)
if not data:
continue
collection = data.get('collection', [])
if not collection:
exhausted = True
continue
if len(collection) < 100:
exhausted = True
for item in collection:
title = item.get('title', 'Untitled')
abstract = item.get('abstract', '').lower()
match = True
for kw in keywords:
if kw not in title.lower() and kw not in abstract:
match = False
break
if not match:
continue
doi = item.get('doi', '')
authors_str = item.get('authors', 'Unknown Authors')
author_list = [a.strip() for a in authors_str.split(';')]
names = author_list[:3]
if len(author_list) > 3:
names.append('et al.')
results.append({
'title': title,
'authors': '; '.join(names),
'journal': f"{server.capitalize()} (Preprint)",
'date': item.get('date', 'N/A'),
'url': f"https://doi.org/{doi}" if doi else f"https://www.{server}.org/",
'doi': doi,
'source': server.capitalize(),
'source_color': '#0891b2', 'source_bg': '#ecfeff',
})
return results, exhausted
except Exception:
return results, True
def _ls_fetch_batch(state):
"""Fetch one batch from all non-exhausted sources and merge into pool."""
topic = state['topic']
start_year = state['start_year']
end_year = state['end_year']
sources = state['sources']
exhausted = set(state['exhausted'])
seen_dois = set(state['seen_dois'])
seen_titles = set(state['seen_titles'])
fetcher_map = {
'PubMed': lambda off: _fetch_pubmed_paged(topic, start_year, end_year, off),
'Europe PMC': lambda off: _fetch_europe_pmc_paged(topic, start_year, end_year, off),
'CrossRef': lambda off: _fetch_crossref_paged(topic, start_year, end_year, off),
'Semantic Scholar': lambda off: _fetch_semantic_scholar_paged(topic, start_year, end_year, off),
'bioRxiv / medRxiv': lambda off: _fetch_biorxiv_paged(topic, start_year, end_year, off),
}
new_results = []
for src_name, fetcher in fetcher_map.items():
if src_name not in sources or src_name in exhausted:
continue
offset = state['offsets'].get(src_name, 0)
try:
res, is_exhausted = fetcher(offset)
if is_exhausted:
exhausted.add(src_name)
state['offsets'][src_name] = offset + LS_FETCH_SIZE
new_results.extend(res)
except Exception:
exhausted.add(src_name)
for r in new_results:
doi = r.get('doi', '').strip().lower()
title_key = r.get('title', '').strip().lower()[:80]
if doi and doi in seen_dois:
continue
if title_key and title_key in seen_titles:
continue
if doi:
seen_dois.add(doi)
if title_key:
seen_titles.add(title_key)
state['pool'].append(r)
state['pool'] = _sort_results(state['pool'])
state['exhausted'] = list(exhausted)
state['seen_dois'] = list(seen_dois)
state['seen_titles'] = list(seen_titles)
def _ls_render_page(state, user_name=""):
"""Render the current Literature Search page using the active user's reviewed state."""
state = state or _ls_empty_state()
page = state.get('page', 1)
pool = state.get('pool', [])
start = (page - 1) * LS_PAGE_SIZE
end = start + LS_PAGE_SIZE
items = pool[start:end]
# Always load the reviewed state here. This was previously omitted, which
# caused Literature Search to fail with an undefined-variable error and
# also broke the Mark as Read refresh path.
user_name = _normalize_app_user(user_name)
reviewed_dois, reviewed_links = load_reviewed_keys(user_name)
if not items:
return ("
No results on this page.
")
html = ""
for r in items:
src_badge = _render_source_badge(r['source'], r['source_color'], r['source_bg'])
title = r.get('title', 'Untitled')
url = r.get('url', '')
authors = r.get('authors', 'Unknown Authors')
journal = r.get('journal', '')
date = r.get('date', 'N/A')
doi = _extract_doi(r.get('doi', ''))
reviewed_badge = (
' Read'
if (doi and doi in reviewed_dois) or (_normalize_reviewed_link(url) and _normalize_reviewed_link(url) in reviewed_links) else ''
)
doi_link = (
f'DOI: {doi}'
if doi else ''
)
title_html = (
f'{title}'
if url else
f'{title}'
)
html += f"""
"""
return html
def _ls_render_nav(state):
page = state['page']
pool = state['pool']
exhausted = state['exhausted']
sources = state['sources']
total_pool = len(pool)
max_known_page = max(1, (total_pool + LS_PAGE_SIZE - 1) // LS_PAGE_SIZE)
all_exhausted = all(s in exhausted for s in sources) if sources else True
has_more = not all_exhausted
pages_to_show = list(range(1, min(max_known_page, 10) + 1))
show_ellipsis = (max_known_page > 10) or has_more
btns = ""
for p in pages_to_show:
if p == page:
btns += (f'{p}')
else:
btns += (f'{p}')
if show_ellipsis:
btns += ('…')
src_counts = {}
for r in pool:
s = r.get('source', '')
src_counts[s] = src_counts.get(s, 0) + 1
badges = ' '.join(
f'{s}: {c}'
for s, c in src_counts.items()
)
status_line = (
f'
'
f'Page {page} · {total_pool} results loaded'
+ (' · fetching more…' if has_more else ' · all sources loaded')
+ f'
'
)
return f"""
{status_line}
{btns}
{badges}
"""
def _ls_empty_state():
return {
'topic': '', 'sources': [], 'start_year': LS_YEAR_MIN, 'end_year': LS_YEAR_MAX,
'page': 1, 'pool': [], 'offsets': {}, 'exhausted': [],
'seen_dois': [], 'seen_titles': [],
}
def ls_init_search(query, sources, start_year, end_year, user_name=""):
start_year = int(start_year) if start_year else LS_YEAR_MIN
end_year = int(end_year) if end_year else LS_YEAR_MAX
if not query or not query.strip():
err = ("
"
" Please enter a valid search query.
")
return err, "", _ls_empty_state()
# If boolean query, restrict to PubMed
if _is_boolean_query(query):
sources = ['PubMed']
state = {
'topic': query.strip(), 'sources': list(sources),
'start_year': start_year, 'end_year': end_year,
'page': 1, 'pool': [], 'offsets': {}, 'exhausted': [],
'seen_dois': [], 'seen_titles': [],
}
_ls_fetch_batch(state)
if len(state['pool']) < LS_PAGE_SIZE:
_ls_fetch_batch(state)
if not state['pool']:
empty_html = (
"
"
" No papers found for “"
+ query +
"” in the selected year range across chosen databases.
"
)
return empty_html, "", state
return _ls_render_page(state, user_name), _ls_render_nav(state), state
def ls_prev_page(state, user_name=""):
if not state or state['page'] <= 1:
return _ls_render_page(state, user_name), _ls_render_nav(state), state
state = {**state, 'page': state['page'] - 1}
return _ls_render_page(state, user_name), _ls_render_nav(state), state
def ls_next_page(state, user_name=""):
if not state:
return "", "", state
state = dict(state)
next_page = state['page'] + 1
needed = next_page * LS_PAGE_SIZE
max_batches = 6
batches_done = 0
all_exhausted = lambda: all(s in state['exhausted'] for s in state['sources'])
while len(state['pool']) < needed and not all_exhausted() and batches_done < max_batches:
_ls_fetch_batch(state)
batches_done += 1
if len(state['pool']) >= (next_page - 1) * LS_PAGE_SIZE + 1:
state['page'] = next_page
return _ls_render_page(state, user_name), _ls_render_nav(state), state
# ── Database functions ────────────────────────────────────────────────────────
def search_supabase(species, tissue, tool, accession=""):
if not supabase:
return pd.DataFrame({"Error": ["Supabase keys are missing. Connection failed."]})
try:
table_name = "sra_database" if tool == "SRA" else "geo_database"
response = supabase.table(table_name).select("*").execute()
raw_data = response.data
if not raw_data:
return pd.DataFrame({"Status": [f"No records found in {table_name}."]})
df = pd.DataFrame(raw_data)
if 'created_at' in df.columns:
df = df.drop(columns=['created_at'])
if accession and accession.strip():
if "Accession" in df.columns:
df = df[df["Accession"].astype(str).str.lower().str.strip() == accession.lower().strip()]
if df.empty:
return pd.DataFrame({"Status": [f"No matching record for Accession '{accession}'."]})
else:
if "Organism" in df.columns:
df = df[df["Organism"].apply(normalize_organism) == normalize_organism(species)]
if "tissue" in df.columns:
df = df[df["tissue"].astype(str).str.lower().str.strip() == tissue.lower().strip()]
if df.empty:
return pd.DataFrame({"Status": [f"No records for '{species}' in {tool}."]})
display_cols = [c for c in ALL_UNIQUE_COLUMNS if c in df.columns]
return df[display_cols].fillna("N/A")
except Exception as e:
return pd.DataFrame({"Database Error": [f"Failed to fetch data: {str(e)}"]})
def load_entire_table():
if not supabase:
return pd.DataFrame({"Error": ["Supabase connection is inactive."]})
try:
sra_res = supabase.table("sra_database").select("*").execute()
df_sra = pd.DataFrame(sra_res.data) if sra_res.data else pd.DataFrame()
if 'created_at' in df_sra.columns:
df_sra = df_sra.drop(columns=['created_at'])
geo_res = supabase.table("geo_database").select("*").execute()
df_geo = pd.DataFrame(geo_res.data) if geo_res.data else pd.DataFrame()
if 'created_at' in df_geo.columns:
df_geo = df_geo.drop(columns=['created_at'])
if df_sra.empty and df_geo.empty:
return pd.DataFrame(columns=ALL_UNIQUE_COLUMNS)
combined_df = pd.concat([df_sra, df_geo], ignore_index=True)
ordered_cols = [c for c in ALL_UNIQUE_COLUMNS if c in combined_df.columns]
return combined_df[ordered_cols].fillna("N/A")
except Exception as e:
return pd.DataFrame({"Error": [str(e)]})
def generate_tools_launchpad_html(dataset_type="GEO"):
tools = SRA_TOOLS_DATA if dataset_type == "SRA" else GEO_TOOLS_DATA
icon = "" if dataset_type == "SRA" else ""
label = "SRA Download & QC" if dataset_type == "SRA" else "GEO Analytical"
accent = "#0891b2" if dataset_type == "SRA" else "#2563eb"
html = f"""
{icon} Biological Data Located. Launch a {label} Suite below:
")
def admin_search_supabase(species, tissue, tool, request: gr.Request):
session_hash = getattr(request, "session_hash", None)
if not is_senior_admin(session_hash):
return pd.DataFrame({"Error": [" Senior admin access required."]})
return search_supabase(species, tissue, tool)
def logout_admin(request: gr.Request):
"""Log out the current admin session and clear the selected app user."""
global _admin_active
session_hash = getattr(request, "session_hash", None)
if session_hash:
_admin_sessions.pop(session_hash, None)
_admin_roles.pop(session_hash, None)
_admin_users.pop(session_hash, None)
_admin_active = False
return (
gr.update(visible=True), # login_panel
gr.update(visible=True), # user_gate_panel
gr.update(visible=False), # junior_access_panel
gr.update(visible=False), # upload_panel
"Logged out.",
BLANK_TEMPLATE_DF,
gr.update(visible=True),
gr.update(visible=False),
gr.update(value=generate_view_html()),
False,
gr.update(visible=False),
gr.update(visible=False),
get_reviewed_table_df(),
get_reviewed_table_df(),
gr.update(visible=False),
gr.update(visible=False),
gr.update(value="Set your user in the Administrative Portal."),
gr.update(value="Current user: not selected"),
gr.update(value="Current user: not selected"),
"",
)
def check_upload_credentials_generator(username, password, request: gr.Request):
"""Authenticate the senior administrator; ordinary users use the public user gate."""
global _admin_active
session_hash = getattr(request, "session_hash", None)
username = "" if username is None else str(username).strip()
password = "" if password is None else str(password)
role = None
if AUTH_USER and AUTH_PASS and username == str(AUTH_USER).strip() and password == str(AUTH_PASS):
role = "senior"
if role and session_hash:
_admin_active = True
set_admin_active(session_hash, role)
clear_current_app_user(session_hash)
else:
_admin_active = False
if session_hash:
_admin_sessions.pop(session_hash, None)
_admin_roles.pop(session_hash, None)
_admin_users.pop(session_hash, None)
common = (
gr.Column(visible=True), # login_panel
gr.Column(visible=True), # user_gate_panel
gr.Column(visible=False), # junior_access_panel
gr.Column(visible=False), # upload_panel
"",
BLANK_TEMPLATE_DF,
gr.Column(visible=True),
gr.Column(visible=False),
gr.HTML(value=generate_view_html()),
False,
gr.Column(visible=False),
gr.Column(visible=False),
get_reviewed_table_df(),
get_reviewed_table_df(),
gr.update(visible=False),
gr.update(visible=False),
gr.update(value="Set your user in the Administrative Portal."),
gr.update(value="Current user: not selected"),
gr.update(value="Current user: not selected"),
"",
)
if not role:
if not AUTH_USER or not AUTH_PASS:
message = "Senior admin credentials are not configured. Public users can still create or log in as an app user below."
else:
message = "Invalid credentials. Access denied."
common = list(common)
common[4] = message
return tuple(common)
return (
gr.Column(visible=False),
gr.Column(visible=True),
gr.Column(visible=False),
gr.Column(visible=False),
f'{role.title()} admin authenticated. Create or log in as a user to continue.',
BLANK_TEMPLATE_DF,
gr.Column(visible=False),
gr.Column(visible=False),
gr.HTML(value=generate_view_html()),
True,
gr.Column(visible=False),
gr.Column(visible=False),
get_reviewed_table_df(),
get_reviewed_table_df(),
gr.update(visible=True), # PubCrawler tab available to both admin levels
gr.update(visible=False), # PubCrawler term panel waits for user selection
gr.update(value="Set your user in the Administrative Portal."),
gr.update(value="Current user: not selected"),
gr.update(value="Current user: not selected"),
"",
)
def upload_data_to_supabase(file_obj, species, tissue, dataset_type, request: gr.Request):
session_hash = getattr(request, "session_hash", None)
if not is_senior_admin(session_hash):
return " Senior admin access required."
if not supabase:
return "Configuration Error: Supabase connection is inactive."
if file_obj is None:
return "Please upload a valid CSV or Excel file first."
try:
if file_obj.name.endswith('.csv'):
uploaded_df = pd.read_csv(file_obj.name)
elif file_obj.name.endswith(('.xls', '.xlsx')):
uploaded_df = pd.read_excel(file_obj.name)
else:
return "Unsupported format. Please upload a .csv or .xlsx file."
uploaded_df.columns = [c.strip() for c in uploaded_df.columns]
table_name = "sra_database" if dataset_type == "SRA" else "geo_database"
for target_col in EXACT_MATCH_COLUMNS:
if target_col not in uploaded_df.columns:
uploaded_df[target_col] = None
db_ready_df = uploaded_df[EXACT_MATCH_COLUMNS].copy().dropna(subset=["Accession"])
canonical_organism = "Bos taurus" if normalize_organism(species) == "bos taurus" else species.strip()
db_ready_df["Organism"] = canonical_organism
db_ready_df["tissue"] = tissue.strip()
db_ready_df["dataset"] = dataset_type.strip()
db_ready_df = sanitize_dataframe(db_ready_df)
new_records = db_ready_df.to_dict(orient="records")
if not new_records:
return "No rows with valid Accession numbers found."
db_response = supabase.table(table_name).select("Accession,Organism,tissue").execute()
existing = db_response.data if db_response.data else []
exact_keys = set()
accession_only = {}
for r in existing:
acc = normalize_val(r.get("Accession"))
org = normalize_organism(r.get("Organism", ""))
tis = normalize_val(r.get("tissue", ""))
exact_keys.add((acc, org, tis))
if acc not in accession_only:
accession_only[acc] = []
accession_only[acc].append((r.get("Organism", ""), r.get("tissue", "")))
incoming_org_norm = normalize_organism(canonical_organism)
incoming_tis_norm = normalize_val(tissue)
true_dupes = []
cross_category_warnings = []
final_records = []
for r in new_records:
acc = normalize_val(r.get("Accession"))
key = (acc, incoming_org_norm, incoming_tis_norm)
if key in exact_keys:
true_dupes.append(acc)
else:
if acc in accession_only:
for (ex_org, ex_tis) in accession_only[acc]:
cross_category_warnings.append(
f" • {acc} already exists as Organism='{ex_org}', Tissue='{ex_tis}'"
)
final_records.append(r)
msg_parts = []
if cross_category_warnings:
warn_lines = "\n".join(cross_category_warnings)
msg_parts.append(
f" Warning — are you sure you inputted the right parameters?\n"
f"The following accessions already exist in a different category:\n{warn_lines}"
)
if true_dupes:
msg_parts.append(f"Skipped {len(true_dupes)} exact duplicate(s) (same accession + organism + tissue).")
if not final_records:
msg_parts.append("No new records to insert after duplicate check.")
return "\n".join(msg_parts)
supabase.table(table_name).insert(final_records).execute()
msg_parts.append(f" Inserted {len(final_records)} record(s) into {table_name}.")
return "\n".join(msg_parts)
except Exception as e:
return f"Upload failed: {str(e)}"
def handle_row_selection(evt: gr.SelectData, current_df):
try:
row_idx = evt.index[0]
if "Accession" in current_df.columns:
accession_id = str(current_df.iloc[row_idx].get("Accession", "")).strip()
if accession_id and accession_id != "N/A":
return accession_id, f"Selected: {accession_id}"
return "", "Selected row has no valid Accession ID."
except Exception as e:
return "", f"Selection error: {str(e)}"
def delete_record_from_supabase(accession_id, current_df, request: gr.Request):
session_hash = getattr(request, "session_hash", None)
if not is_senior_admin(session_hash):
return " Senior admin access required.", current_df, accession_id
if not supabase or not accession_id.strip() or accession_id == "N/A":
return "No valid record selected.", current_df, accession_id
try:
response = supabase.table("sra_database").delete().eq("Accession", accession_id.strip()).execute()
if not response.data:
response = supabase.table("geo_database").delete().eq("Accession", accession_id.strip()).execute()
if response.data:
return f"Deleted: {accession_id}", load_entire_table(), ""
return f"Record {accession_id} not found.", current_df, accession_id
except Exception as e:
return f"Deletion error: {str(e)}", current_df, accession_id
def _df_to_html_table(df):
if df is None or df.empty:
return "
No records found.
"
cols = list(df.columns)
header = "".join(f'
{c}
' for c in cols)
rows = ""
for i, row in df.iterrows():
bg = "#ffffff" if i % 2 == 0 else "#f8fafc"
cells = "".join(f'
Integrated Bioinformatics Platform for Animal Reproduction Research
ReproOmics Hub is an integrated bioinformatics platform developed to support research in
Animal Reproductive Technologies (ART) with a primary focus on cattle and buffalo.
The portal provides seamless access to public omics datasets, literature resources,
AI-assisted analytical tools, and bioinformatics workflows to accelerate research in
reproductive biology, embryo development, fertility, and livestock improvement.
"""
# ── Reviewed-Article helpers (kept for admin DOI box) ──────────────────────────────────
_admin_active = False
_reviewed_cache_by_user: dict[str, set] = {}
_reviewed_link_cache_by_user: dict[str, set] = {}
_supabase_reviewed_ok: bool | None = None
_reviewed_schema_error = False
def _check_reviewed_table():
global _supabase_reviewed_ok, _reviewed_schema_error
if _supabase_reviewed_ok is not None:
return _supabase_reviewed_ok
if not supabase:
_supabase_reviewed_ok = False
return False
try:
supabase.table("reviewed_articles").select("doi,user").limit(1).execute()
_supabase_reviewed_ok = True
except Exception:
_supabase_reviewed_ok = False
_reviewed_schema_error = True
return _supabase_reviewed_ok
def _normalize_reviewed_link(link):
return str(link or '').strip().rstrip('/').lower()
def load_reviewed_keys(user_name=''):
"""Return user-specific reviewed DOI and paper-link sets."""
user_name = _normalize_app_user(user_name)
dois = set()
links = set()
if not user_name:
return dois, links
if _check_reviewed_table():
try:
res = (
supabase.table('reviewed_articles')
.select('doi,link')
.eq('user', user_name)
.execute()
)
for row in (res.data or []):
doi = _extract_doi(row.get('doi', ''))
link = _normalize_reviewed_link(row.get('link', ''))
if doi:
dois.add(doi)
if link:
links.add(link)
return dois, links
except Exception:
pass
dois.update(_reviewed_cache_by_user.get(user_name, set()))
links.update(_normalize_reviewed_link(x) for x in _reviewed_link_cache_by_user.get(user_name, set()))
links.discard('')
return dois, links
def _is_reviewed_paper(doi='', link='', user_name=''):
doi_key = _extract_doi(doi)
link_key = _normalize_reviewed_link(link)
reviewed_dois, reviewed_links = load_reviewed_keys(user_name)
return bool((doi_key and doi_key in reviewed_dois) or (link_key and link_key in reviewed_links))
def load_reviewed_dois(user_name=''):
reviewed_dois, _ = load_reviewed_keys(user_name)
return reviewed_dois
def save_reviewed_article(article_id, user_name, paper_link=''):
article_id = (article_id or '').strip().lower()
user_name = _normalize_app_user(user_name)
paper_link = (paper_link or '').strip()
if not article_id:
return 'No identifier provided'
if not user_name:
return 'Please select a user first.'
if not _check_reviewed_table():
return 'reviewed_articles table is missing or does not have the required user column.'
try:
existing = (
supabase.table('reviewed_articles')
.select('doi,user')
.eq('doi', article_id)
.eq('user', user_name)
.limit(1)
.execute()
)
if existing.data:
return 'Already marked as reviewed for this user.'
payload = {'doi': article_id, 'user': user_name}
if paper_link:
payload['link'] = paper_link
try:
supabase.table('reviewed_articles').insert(payload).execute()
except Exception:
if paper_link:
return 'Could not save the paper link. Add a link column to reviewed_articles and try again.'
raise
_reviewed_cache_by_user.setdefault(user_name, set()).add(article_id)
return 'Marked as reviewed'
except Exception:
return 'Could not save the reviewed article. Check the reviewed_articles table schema and permissions.'
def save_reviewed_paper_link(paper_link, user_name):
"""Mark a direct research-paper URL as read for the current app user."""
user_name = _normalize_app_user(user_name)
paper_link = str(paper_link or '').strip()
if not user_name:
return 'Please select a user first.'
if not paper_link:
return 'Please enter a research paper link.'
parsed = urllib.parse.urlparse(paper_link)
if parsed.scheme.lower() not in ('http', 'https') or not parsed.netloc:
return 'Please enter a valid http(s) research paper link.'
doi = _extract_doi(paper_link)
if doi:
return save_reviewed_article(doi, user_name, paper_link)
if not _check_reviewed_table():
return 'reviewed_articles table is missing or does not have the required user column.'
try:
existing = (
supabase.table('reviewed_articles')
.select('doi,user,link')
.eq('user', user_name)
.eq('link', paper_link)
.limit(1)
.execute()
)
if existing.data:
return 'Already marked as reviewed for this user.'
supabase.table('reviewed_articles').insert({
'doi': None,
'user': user_name,
'link': paper_link,
}).execute()
_reviewed_link_cache_by_user.setdefault(user_name, set()).add(paper_link)
return 'Marked as reviewed'
except Exception:
return 'Could not save the paper link. Add a link column to reviewed_articles and allow DOI to be blank.'
def remove_reviewed_article(article_id, user_name):
article_id = (article_id or '').strip().lower()
user_name = _normalize_app_user(user_name)
if not article_id:
return 'No identifier provided'
if not user_name:
return 'Please select a user first.'
if not _check_reviewed_table():
return 'reviewed_articles table is missing or does not have the required user column.'
try:
if article_id.startswith(('http://', 'https://')):
supabase.table('reviewed_articles').delete().eq('link', article_id).eq('user', user_name).execute()
_reviewed_link_cache_by_user.setdefault(user_name, set()).discard(article_id)
else:
supabase.table('reviewed_articles').delete().eq('doi', article_id).eq('user', user_name).execute()
_reviewed_cache_by_user.setdefault(user_name, set()).discard(article_id)
return 'Removed from reviewed'
except Exception:
return 'Could not remove the reviewed article. Check the reviewed_articles table schema and permissions.'
def mark_article_api(doi, action, session_hash=None):
if not session_hash or session_hash not in _admin_sessions:
return "Admin login required. Please log in first."
role = get_admin_role(session_hash)
if action == "unmark" and role != "senior":
return "Senior admin access required to remove reviewed articles."
if role not in ("user", "junior", "senior"):
return "Please create or log in as a user first."
user_name = get_current_app_user(session_hash)
if not user_name:
return "Please create or log in as a user first."
article_id = (doi or "").strip().lower()
if not article_id:
return "No identifier provided"
if action == "mark":
return save_reviewed_article(article_id, user_name)
elif action == "unmark":
return remove_reviewed_article(article_id, user_name)
else:
return "Unknown action"
def list_reviewed_html(user_name=''):
rows_df = get_reviewed_table_df(user_name)
if rows_df.empty:
return "
No articles marked yet.
"
html = ""
for _, r in rows_df.iterrows():
article_id = str(r.get('DOI', ''))
at = str(r.get('Reviewed At', '') or '')[:10]
display = article_id[:80] + ('...' if len(article_id) > 80 else '')
link = f"https://doi.org/{article_id}" if not article_id.startswith('http') else article_id
html += (
f"
',
gr.update(value='Current user: not selected'),
gr.update(value='Current user: not selected'),
)
def _user_registry_ok():
if not supabase:
return False
try:
supabase.table('app_users').select('name').limit(1).execute()
return True
except Exception:
return False
def _find_app_user(user_name):
user_name = _normalize_app_user(user_name)
if not user_name or not supabase:
return None
try:
res = supabase.table('app_users').select('name').ilike('name', user_name).limit(1).execute()
return (res.data or [None])[0]
except Exception:
return None
def _activate_app_user_session(user_name, request, message):
session_hash = getattr(request, 'session_hash', None)
user_name = _normalize_app_user(user_name)
if not session_hash:
return _user_gate_failure('Could not identify this browser session. Refresh the page and try again.')
# App users do not need an admin password. Preserve senior privileges only
# when the session has separately authenticated with the senior credentials.
if session_hash not in _admin_sessions:
set_admin_active(session_hash, 'user')
elif get_admin_role(session_hash) not in ('senior', 'user', 'junior'):
set_admin_active(session_hash, 'user')
set_current_app_user(session_hash, user_name)
role = get_admin_role(session_hash)
show_junior = role in ('user', 'junior')
show_senior = role == 'senior'
reviewed = get_reviewed_table_df(user_name)
pubcrawler_terms = get_pubcrawler_terms_df(user_name)
pubcrawler_articles = get_pubcrawler_articles_html(request, user_name)
display = f'Current user: **{html_lib.escape(user_name)}**'
return (
message,
user_name,
gr.update(visible=False),
gr.update(visible=show_junior),
gr.update(visible=show_senior),
gr.update(visible=True),
gr.update(visible=True),
reviewed,
reviewed,
gr.update(visible=True), # pm_doi_col
gr.update(visible=True), # ls_doi_col
pubcrawler_terms,
gr.update(value=display),
pubcrawler_articles,
gr.update(value=display),
gr.update(value=display),
)
def create_app_user(user_name, request: gr.Request):
session_hash = getattr(request, 'session_hash', None)
if not session_hash:
return _user_gate_failure('Could not identify this browser session. Refresh the page and try again.')
if session_hash not in _admin_sessions:
set_admin_active(session_hash, 'user')
user_name = _normalize_app_user(user_name)
if not user_name:
return _user_gate_failure('Enter a user name.')
if not supabase:
return _user_gate_failure('Supabase connection is unavailable.')
if not _user_registry_ok():
return _user_gate_failure('User registry table is missing. Create the app_users table in Supabase first.')
if _find_app_user(user_name):
return _user_gate_failure('user already taken')
try:
supabase.table('app_users').insert({'name': user_name}).execute()
except Exception:
if _find_app_user(user_name):
return _user_gate_failure('user already taken')
return _user_gate_failure('Could not create user. Check the app_users table and its permissions.')
return _activate_app_user_session(user_name, request, f'User created: **{html_lib.escape(user_name)}**')
def login_as_app_user(user_name, request: gr.Request):
session_hash = getattr(request, 'session_hash', None)
if not session_hash:
return _user_gate_failure('Could not identify this browser session. Refresh the page and try again.')
if session_hash not in _admin_sessions:
set_admin_active(session_hash, 'user')
user_name = _normalize_app_user(user_name)
if not user_name:
return _user_gate_failure('Enter a user name.')
if not supabase:
return _user_gate_failure('Supabase connection is unavailable.')
if not _user_registry_ok():
return _user_gate_failure('User registry table is missing. Create the app_users table in Supabase first.')
existing = _find_app_user(user_name)
if not existing:
return _user_gate_failure('User not found. Use Create User for a new user.')
canonical = _normalize_app_user(existing.get('name', user_name))
return _activate_app_user_session(canonical, request, f'Logged in as user: **{html_lib.escape(canonical)}**')
# ── PubCrawler term management (per-user access) ─────────────────────────────
def _pubcrawler_admin_ok(request: gr.Request):
session_hash = getattr(request, 'session_hash', None)
return bool(session_hash and session_hash in _admin_sessions and get_current_app_user(session_hash)
and (get_admin_role(session_hash) in ('senior', 'junior', 'user')))
def _pubcrawler_current_user(request: gr.Request):
session_hash = getattr(request, 'session_hash', None)
return get_current_app_user(session_hash)
def get_pubcrawler_terms_df(user_name=''):
"""Load only PubCrawler terms belonging to the supplied app user.
Matching is normalized case-insensitively so records saved in an earlier
session with harmless capitalization/whitespace differences still appear.
"""
columns = ['ID', 'Term']
user_name = _normalize_app_user(user_name)
if not user_name or not supabase:
return pd.DataFrame(columns=columns)
try:
res = (
supabase.table('pubcrawler')
.select('id,term,user')
.order('id', desc=True)
.execute()
)
wanted = user_name.casefold()
rows = []
for row in (res.data or []):
stored_user = _normalize_app_user(row.get('user', ''))
if stored_user.casefold() == wanted:
rows.append({'ID': row.get('id'), 'Term': row.get('term', '')})
return pd.DataFrame(rows, columns=columns)
except Exception:
return pd.DataFrame(columns=columns)
def save_pubcrawler_term(user_name, term, request: gr.Request):
if not _pubcrawler_admin_ok(request):
return 'Admin login and user selection are required.', get_pubcrawler_terms_df(user_name), '', get_pubcrawler_articles_html(request, user_name)
user_name = _pubcrawler_current_user(request)
term = ' '.join(str(term or '').split())
if not user_name:
return 'Please select a user first.', get_pubcrawler_terms_df(), str(term or ''), ''
if not term:
return 'Enter a term before saving.', get_pubcrawler_terms_df(user_name), '', get_pubcrawler_articles_html(request, user_name)
if not supabase:
return 'Supabase connection is unavailable.', get_pubcrawler_terms_df(user_name), '', get_pubcrawler_articles_html(request, user_name)
try:
existing = (
supabase.table('pubcrawler')
.select('id')
.eq('term', term)
.eq('user', user_name)
.limit(1)
.execute()
)
if existing.data:
return 'That term is already saved for this user.', get_pubcrawler_terms_df(user_name), term, get_pubcrawler_articles_html(request, user_name)
supabase.table('pubcrawler').insert({'term': term, 'user': user_name}).execute()
return f'Saved PubCrawler term: **{html_lib.escape(term)}**', get_pubcrawler_terms_df(user_name), '', get_pubcrawler_articles_html(request, user_name)
except Exception as e:
return f'Could not save term: {e}', get_pubcrawler_terms_df(user_name), term, get_pubcrawler_articles_html(request, user_name)
def pubcrawler_row_selection(evt: gr.SelectData, current_df):
try:
row_idx = evt.index[0]
row = current_df.iloc[row_idx]
row_id = str(row.get('ID', '')).strip()
term = str(row.get('Term', '')).strip()
return row_id, f'Selected term: {term}' if term else 'No term selected.'
except Exception as e:
return '', f'Selection error: {e}'
def get_pubcrawler_articles_html(request: gr.Request, user_name=''):
"""Search only the current user's saved PubCrawler terms across the literature sources for the past 30 days."""
if not _pubcrawler_admin_ok(request):
return "
Admin login required. Please log in first.
"
user_name = _normalize_app_user(user_name)
if not user_name:
return "
Enter your name and click Set User before refreshing articles.
"
if not supabase:
return "
Supabase connection is unavailable.
"
terms_df = get_pubcrawler_terms_df(user_name)
terms = []
if not terms_df.empty:
terms = [str(x).strip() for x in terms_df['Term'].tolist() if str(x).strip()]
if not terms:
return "
No PubCrawler terms have been saved yet.
"
since = datetime.date.today() - datetime.timedelta(days=30)
since_date = since.strftime('%Y/%m/%d')
since_dash = since.strftime('%Y-%m-%d')
all_results = []
term_logs = []
for term in terms:
source_map = {
'PubMed': lambda t=term: _fetch_pubmed(t, since_date, max_results=200),
'Europe PMC': lambda t=term: _fetch_europe_pmc(t, since_dash, max_results=200),
'CrossRef': lambda t=term: _fetch_crossref(t, since_dash, max_results=100),
'Semantic Scholar': lambda t=term: _fetch_semantic_scholar(t, since_dash, max_results=100),
'bioRxiv / medRxiv': lambda t=term: _fetch_biorxiv(t, since_dash, max_results=100),
}
for source_name, fetch_fn in source_map.items():
try:
results, log = fetch_fn()
for result in results:
result = dict(result)
result['_crawler_term'] = term
all_results.append(result)
if results:
term_logs.append(f"{term}: {source_name} {len(results)}")
except Exception as e:
term_logs.append(f"{term}: {source_name} error")
if not all_results:
return f"""
No publications matching the saved PubCrawler terms were found in the past 30 days.
"""
# Deduplicate while retaining all saved terms that matched each article.
merged = {}
for r in all_results:
doi = _extract_doi(r.get('doi', ''))
title_key = str(r.get('title', '')).strip().lower()
key = ('doi', doi.lower()) if doi else ('title', title_key[:140])
if key not in merged:
merged[key] = dict(r)
merged[key]['_crawler_terms'] = [r.get('_crawler_term', '')]
else:
hit = r.get('_crawler_term', '')
if hit and hit not in merged[key]['_crawler_terms']:
merged[key]['_crawler_terms'].append(hit)
if not merged[key].get('doi') and doi:
merged[key]['doi'] = doi
results = _sort_results(list(merged.values()))
reviewed_dois, reviewed_links = load_reviewed_keys(user_name)
source_counts = {}
for r in results:
src = r.get('source', 'Unknown')
source_counts[src] = source_counts.get(src, 0) + 1
count_badges = ''.join(
f"{html_lib.escape(src)}: {cnt}"
for src, cnt in source_counts.items()
)
html = f"""
"""
for r in results:
title = html_lib.escape(str(r.get('title', 'Untitled')))
url = html_lib.escape(str(r.get('url', '') or ''))
authors = html_lib.escape(str(r.get('authors', 'Unknown Authors')))
journal = html_lib.escape(str(r.get('journal', '')))
date = html_lib.escape(str(r.get('date', 'N/A')))
doi = _extract_doi(r.get('doi', ''))
doi_esc = html_lib.escape(doi)
src = html_lib.escape(str(r.get('source', 'Unknown')))
src_color = r.get('source_color', '#2563eb')
src_bg = r.get('source_bg', '#eff6ff')
terms_hit = r.get('_crawler_terms', []) or []
term_html = ''.join(
f"{html_lib.escape(t)}"
for t in terms_hit
)
reviewed_badge = (
' Read'
if (doi and doi in reviewed_dois) or (_normalize_reviewed_link(url) and _normalize_reviewed_link(url) in reviewed_links) else ''
)
title_html = (
f'{title}'
if url else f'
{title}
'
)
html += f"""
{title_html}
{src}
{reviewed_badge}
{term_html}
{date}
Authors: {authors}
{f' •{journal}' if journal else ''}
"
return html
def refresh_pubcrawler_articles(user_name, request: gr.Request):
user_name = _pubcrawler_current_user(request)
return get_pubcrawler_articles_html(request, user_name)
def delete_pubcrawler_term(selected_id, user_name, request: gr.Request):
"""Delete only a selected PubCrawler term belonging to the current app user."""
user_name = _pubcrawler_current_user(request)
if not _pubcrawler_admin_ok(request):
return 'Admin login and user selection are required.', get_pubcrawler_terms_df(user_name), '', get_pubcrawler_articles_html(request, user_name)
if not user_name:
return 'Please select a user first.', get_pubcrawler_terms_df(), '', ''
if not selected_id:
return 'Select a term first.', get_pubcrawler_terms_df(user_name), '', get_pubcrawler_articles_html(request, user_name)
if not supabase:
return 'Supabase connection is unavailable.', get_pubcrawler_terms_df(user_name), '', get_pubcrawler_articles_html(request, user_name)
try:
supabase.table('pubcrawler').delete().eq('id', int(selected_id)).eq('user', user_name).execute()
return 'PubCrawler term removed.', get_pubcrawler_terms_df(user_name), '', get_pubcrawler_articles_html(request, user_name)
except Exception as e:
return f'Could not remove term: {e}', get_pubcrawler_terms_df(user_name), str(selected_id), get_pubcrawler_articles_html(request, user_name)
# ── Layout ────────────────────────────────────────────────────────────────────
ALL_SOURCES = ['PubMed', 'Europe PMC', 'CrossRef', 'Semantic Scholar', 'bioRxiv / medRxiv']
with gr.Blocks() as demo:
# ── Institute header ──────────────────────────────────────────────────────
gr.HTML("""
भाकृअनुप-राष्ठ्रीय डेरी अनुसंधान संस्थान
ICAR – National Dairy Research Institute
KARNAL, HARYANA
ReproOmics Hub — Bioinformatics Hub for Animal Reproduction Research
""")
# ── Paper Alert ──
gr.HTML(value=generate_paper_alert_html(), elem_id="paper-alert-host")
# ── Shared state ──
is_authenticated = gr.State(False)
current_user_state = gr.State("")
selected_accession = gr.State("")
hidden_canvas_data = gr.Textbox(visible=False)
# ── Hidden textbox for session_hash ──
session_hash_state = gr.Textbox(visible=False, value="")
# ── JavaScript to set session_hash_state ──
gr.HTML("""
""", elem_id="session-hash-js-host")
with gr.Tabs():
# ── TAB 1: Home ────────────────────────────────────────────────────────
with gr.TabItem("Home"):
with gr.Column(visible=True) as home_locked_panel:
home_view_component = gr.HTML(value=generate_view_html())
gr.HTML(HOME_CONTACT_HTML)
with gr.Column(visible=False) as home_editor_panel:
gr.Markdown("## Homepage Canvas Editor")
home_editor_component = gr.HTML(value=generate_editor_html())
with gr.Row():
save_home_btn = gr.Button(" Save Homepage", variant="primary", scale=2)
home_save_status = gr.Markdown("*Click Save to write changes.*")
# ── TAB 2: Query Database Hub ──────────────────────────────────────────
with gr.TabItem("Query Database Hub"):
with gr.Column(elem_id="qt-search-panel", visible=True) as qt_search_panel:
gr.HTML("""
Query Biological Records
Select filters and click Search to retrieve data from GEO or SRA.
""")
with gr.Column(scale=3, min_width=400):
pm_output_html = gr.HTML(
value="
Select databases and enter a topic to begin searching…
"
)
# ── Admin DOI box (visible only when authenticated) ──
with gr.Column(scale=1, min_width=220, visible=False) as pm_doi_col:
with gr.Group():
gr.HTML("""
Admin — Mark DOI as Read
Paste a DOI and mark it as reviewed. Stored in Supabase.
"
)
with gr.Row():
ls_prev_btn = gr.Button("← Prev", variant="secondary", size="sm")
ls_next_btn = gr.Button("Next →", variant="primary", size="sm")
# ── Admin DOI box (visible only when authenticated) ──
with gr.Column(scale=1, min_width=220, visible=False) as ls_doi_col:
with gr.Group():
gr.HTML("""
Admin — Mark DOI as Read
Paste a DOI and mark it as reviewed. Stored in Supabase.
""")
ls_doi_input = gr.Textbox(label="DOI", placeholder="e.g. 10.1093/nar/gkac123")
ls_doi_btn = gr.Button("Mark as Read", variant="primary", size="sm")
ls_doi_status = gr.Markdown("")
# ── Event wiring ──────────────────────────────────────────────
ls_search_btn.click(
fn=ls_init_search,
inputs=[ls_topic_input, ls_sources_input, ls_year_from, ls_year_to, current_user_state],
outputs=[ls_output_html, ls_nav_html, ls_state],
)
ls_doi_search_btn.click(
fn=search_by_doi,
inputs=[ls_doi_search_input, current_user_state],
outputs=ls_output_html
)
ls_prev_btn.click(
fn=ls_prev_page,
inputs=[ls_state, current_user_state],
outputs=[ls_output_html, ls_nav_html, ls_state],
)
ls_next_btn.click(
fn=ls_next_page,
inputs=[ls_state, current_user_state],
outputs=[ls_output_html, ls_nav_html, ls_state],
)
def ls_mark_doi_and_refresh(doi, user_name, state, request: gr.Request):
# Mark the DOI for the active app user, then immediately re-render
# the current Literature Search results so the Read badge appears.
msg = manual_mark_doi(doi, request)
current_user = get_current_app_user(getattr(request, 'session_hash', None)) or _normalize_app_user(user_name)
return msg, _ls_render_page(state, current_user), _ls_render_nav(state), state
ls_doi_btn.click(
fn=ls_mark_doi_and_refresh,
inputs=[ls_doi_input, current_user_state, ls_state],
outputs=[ls_doi_status, ls_output_html, ls_nav_html, ls_state]
)
# Remove all mark/review UI and JavaScript.
# ── TAB 5: AI Tools Directory ──────────────────────────────────────────
with gr.TabItem("AI Tools"):
gr.HTML("""
AI / ML / DL Tools
Curated AI, Machine Learning & Deep Learning tools for cattle & buffalo reproduction research.
Click any card to open the tool or source paper.
AI / ML / DL Tools for Cattle & Buffalo Reproduction
""")
# ── TAB 6: PubCrawler ───────────────────────────────────────────────────
# Available to both senior and junior admins after a shared app user is selected.
with gr.TabItem("PubCrawler", visible=False) as pubcrawler_tab:
gr.HTML("""
PubCrawler
PubCrawler uses the same app user selected in the Administrative Portal. Only that user's saved terms are loaded.
""")
pubcrawler_user_display = gr.Markdown("Set your user in the Administrative Portal.")
gr.HTML("""
Search additional biomedical literature directly in NCBI or PubMed.
""")
with gr.Column():
gr.Markdown("### Mark a Research Paper as Read")
gr.Markdown("Use either a DOI or a direct research-paper link. The entry is saved only for the current app user.")
with gr.Row():
with gr.Column(scale=1):
pubcrawler_doi_input = gr.Textbox(label="DOI", placeholder="e.g. 10.1093/nar/gkac123")
pubcrawler_doi_btn = gr.Button("Mark DOI as Read", variant="primary")
with gr.Column(scale=1):
pubcrawler_link_input = gr.Textbox(label="Research Paper Link", placeholder="https://pubmed.ncbi.nlm.nih.gov/...")
pubcrawler_link_btn = gr.Button("Mark Link as Read", variant="primary")
pubcrawler_mark_status = gr.Markdown("")
with gr.Row():
with gr.Column(scale=2, min_width=320, visible=True) as pubcrawler_term_panel:
gr.Markdown("### Add a crawler term")
pubcrawler_term_input = gr.Textbox(
label="Term",
placeholder="e.g. bovine oocyte CRISPR"
)
with gr.Row():
pubcrawler_save_btn = gr.Button("Save Term", variant="primary")
pubcrawler_refresh_btn = gr.Button("Refresh", variant="secondary")
pubcrawler_status = gr.Markdown("")
with gr.Column(scale=3, min_width=420):
gr.Markdown("### Your Saved PubCrawler Terms")
pubcrawler_table = gr.Dataframe(
value=get_pubcrawler_terms_df(),
headers=['ID', 'Term'],
datatype=['number', 'str'],
interactive=False,
wrap=True,
max_height=300,
)
selected_pubcrawler_id = gr.State("")
with gr.Row():
pubcrawler_delete_btn = gr.Button("Remove Selected Term", variant="stop")
pubcrawler_selection_status = gr.Markdown("")
gr.Markdown("### Articles Matching Your Saved Terms — Past 30 Days")
with gr.Row():
pubcrawler_articles_refresh_btn = gr.Button("Refresh Articles", variant="secondary", scale=1)
pubcrawler_articles_status = gr.Markdown("")
pubcrawler_articles_html = gr.HTML(
value="
Set your user in the Administrative Portal.
"
)
# ── TAB 6: Administrative Portal ───────────────────────────────────────
with gr.TabItem("Administrative Portal"):
with gr.Column(visible=True) as login_panel:
gr.Markdown("### Senior Admin Login")
gr.Markdown("Optional. Ordinary users can create or log in to an app user below without an admin password.")
user_box = gr.Textbox(label="Username", placeholder="Enter username...")
pass_box = gr.Textbox(label="Password", type="password", placeholder="Enter password...")
login_btn = gr.Button("Unlock Admin Console", variant="primary")
with gr.Column(visible=True) as user_gate_panel:
with gr.Row():
gr.Markdown("## Select User")
user_gate_logout_btn = gr.Button("Log Out", variant="secondary")
gr.Markdown("Create a new user or log in as an existing user. This user will be used for both PubCrawler terms and Mark as Read.")
app_user_input = gr.Textbox(label="User", placeholder="Enter user name...")
with gr.Row():
create_user_btn = gr.Button("Create User", variant="primary")
login_user_btn = gr.Button("Login as User", variant="secondary")
app_user_status = gr.Markdown("")
with gr.Column(visible=False) as junior_access_panel:
gr.Markdown("## User Tools — DOI Review")
junior_current_user = gr.Markdown("Current user: not selected")
gr.Markdown("Your DOI review and PubCrawler workspace. Dataset uploads, record deletion, homepage editing, and removing reviewed articles require senior admin access.")
with gr.Row():
junior_doi_input = gr.Textbox(label="DOI", placeholder="e.g. 10.1093/nar/gkac123", scale=4)
junior_doi_btn = gr.Button("Mark as Read", variant="primary", scale=1)
junior_doi_status = gr.Markdown("")
junior_logout_btn = gr.Button("Log Out", variant="secondary")
gr.Markdown("### Recently Marked DOIs")
junior_reviewed_table = gr.Dataframe(value=get_reviewed_table_df(), headers=['DOI', 'Reviewed At', 'User', 'Link'], datatype=['str', 'str', 'str', 'str'], interactive=False, wrap=True, max_height=320)
with gr.Column(visible=False) as upload_panel:
senior_current_user = gr.Markdown("Current user: not selected")
with gr.Row():
gr.Markdown("## Admin Console")
senior_logout_btn = gr.Button("Log Out", variant="secondary", scale=1)
with gr.Row():
with gr.Column(scale=2):
gr.Markdown("### Upload Records")
file_input = gr.File(label="Upload CSV or Excel", file_types=[".csv", ".xlsx"])
upload_species = gr.Dropdown(choices=["Bos taurus", "Buffalo"], label="Animal Type", value="Bos taurus")
upload_tissue = gr.Dropdown(choices=["Oocyte", "Blastocyst", "Sperm"], label="Tissue Type", value="Oocyte")
upload_dataset = gr.Radio(choices=["GEO", "SRA"], label="Dataset Type", value="SRA")
submit_btn = gr.Button("Push Records", variant="primary")
gr.Markdown("---")
gr.Markdown("### Delete Record")
delete_btn = gr.Button("Delete Selected Row", variant="stop")
status_output = gr.Textbox(label="Console Log", placeholder="Awaiting actions...")
with gr.Column(scale=3):
gr.Markdown("### Filter & Browse Records")
admin_species = gr.Dropdown(choices=["Bos taurus", "Buffalo"], label="Animal Type", value="Bos taurus")
admin_tissue = gr.Dropdown(choices=["Oocyte", "Blastocyst", "Sperm"], label="Tissue Type", value="Oocyte")
admin_tool = gr.Radio(choices=["GEO", "SRA"], label="Source", value="SRA")
with gr.Row():
admin_search_btn = gr.Button("Run Filter", variant="secondary")
admin_reset_btn = gr.Button("Reload All", variant="secondary")
gr.Markdown("### Database Table (All Records)")
admin_table_view = gr.Dataframe(value=BLANK_TEMPLATE_DF, interactive=False, wrap=False)
# ── Manual DOI mark as read ──────────────────────
gr.Markdown("### Manual Mark as Read")
gr.HTML("""
Paste a DOI (e.g., 10.1093/nar/gkac123) and click the button.
The article will be marked as reviewed and stored in Supabase.