bep40 commited on
Commit
6505e8d
·
verified ·
1 Parent(s): d496a66

Improve topic posts: use Google News RSS web context before fallback

Browse files
Files changed (1) hide show
  1. main.py +17 -6
main.py CHANGED
@@ -677,16 +677,27 @@ def api_article(url:str=Query(...)):
677
  else:data=None
678
  return JSONResponse(data if data else{"error":"not supported"})
679
  def _web_context(topic):
 
 
680
  try:
681
- q=quote(topic)
682
- r=requests.get("https://duckduckgo.com/html/?q="+q,headers=HEADERS,timeout=12);r.encoding="utf-8"
683
- soup=BeautifulSoup(r.text,"lxml");bits=[]
 
 
 
 
 
 
 
 
 
684
  for res in soup.select(".result")[:6]:
685
  t=res.select_one(".result__title");sn=res.select_one(".result__snippet")
686
  line=((t.get_text(" ",strip=True) if t else "")+" — "+(sn.get_text(" ",strip=True) if sn else "")).strip(" —")
687
- if line:bits.append(line)
688
- return "\n".join(bits)
689
- except:return ""
690
 
691
  def _scrape_generic_article(url):
692
  try:
 
677
  else:data=None
678
  return JSONResponse(data if data else{"error":"not supported"})
679
  def _web_context(topic):
680
+ """Collect real web/news context for a topic."""
681
+ bits=[]
682
  try:
683
+ rss="https://news.google.com/rss/search?q="+quote(topic)+"&hl=vi&gl=VN&ceid=VN:vi"
684
+ r=requests.get(rss,headers=HEADERS,timeout=12);r.encoding="utf-8"
685
+ soup=BeautifulSoup(r.text,"xml")
686
+ for it in soup.find_all("item")[:8]:
687
+ title=it.find("title").get_text(" ",strip=True) if it.find("title") else ""
688
+ src=it.find("source").get_text(" ",strip=True) if it.find("source") else ""
689
+ if title:bits.append((title+(" — "+src if src else ""))[:280])
690
+ except:pass
691
+ if bits:return "\n".join(bits)
692
+ try:
693
+ r=requests.get("https://html.duckduckgo.com/html/?q="+quote(topic),headers=HEADERS,timeout=12);r.encoding="utf-8"
694
+ soup=BeautifulSoup(r.text,"lxml")
695
  for res in soup.select(".result")[:6]:
696
  t=res.select_one(".result__title");sn=res.select_one(".result__snippet")
697
  line=((t.get_text(" ",strip=True) if t else "")+" — "+(sn.get_text(" ",strip=True) if sn else "")).strip(" —")
698
+ if line:bits.append(line[:280])
699
+ except:pass
700
+ return "\n".join(bits)
701
 
702
  def _scrape_generic_article(url):
703
  try: