bep40 commited on
Commit
308f50b
·
verified ·
1 Parent(s): 59ce1f9

Add vnreview.vn AI scraper + /api/vnreview_ai endpoint

Browse files
Files changed (1) hide show
  1. main.py +30 -0
main.py CHANGED
@@ -340,6 +340,34 @@ def scrape_dantri_congnghe():
340
  if len(arts)>=15:break
341
  return arts
342
  except:return[]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
343
  def scrape_dantri_article(url):
344
  try:
345
  r=requests.get(url,headers=HEADERS,timeout=15);r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
@@ -427,6 +455,8 @@ def api_categories():
427
  return JSONResponse(cats)
428
  @app.get("/api/dantri_hot")
429
  def api_dantri_hot():return JSONResponse(_cached("dantri_hot",scrape_dantri_hot))
 
 
430
  @app.get("/api/article")
431
  def api_article(url:str=Query(...)):
432
  if"vnexpress.net" in url:data=scrape_vne_article(url)
 
340
  if len(arts)>=15:break
341
  return arts
342
  except:return[]
343
+ def scrape_vnreview_ai():
344
+ """Scrape AI articles from vnreview.vn"""
345
+ try:
346
+ r=requests.get("https://vnreview.vn/chuyen-muc/ai-pho-thong.28/",headers=HEADERS,timeout=15)
347
+ if r.status_code!=200:return[]
348
+ r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
349
+ import re as _re
350
+ articles=[];seen=set()
351
+ for img in soup.find_all("img"):
352
+ src=img.get("src","") or img.get("data-src","")
353
+ if "cdn.vnreview.vn" not in src or "avatars" in src or "logo" in src:continue
354
+ parent=img.parent;link=None
355
+ for _ in range(5):
356
+ if parent is None:break
357
+ link=parent.find("a",href=_re.compile(r"/threads/"))
358
+ if link:break
359
+ parent=parent.parent
360
+ if not link:continue
361
+ href=link.get("href","")
362
+ if not href.startswith("http"):href="https://vnreview.vn"+href
363
+ if href in seen:continue
364
+ title=link.get("title","") or link.get_text(strip=True) or img.get("alt","")
365
+ if not title or len(title)<10:continue
366
+ seen.add(href);articles.append({"title":title,"link":href,"img":src,"source":"vnreview"})
367
+ if len(articles)>=12:break
368
+ return articles
369
+ except:return[]
370
+
371
  def scrape_dantri_article(url):
372
  try:
373
  r=requests.get(url,headers=HEADERS,timeout=15);r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
 
455
  return JSONResponse(cats)
456
  @app.get("/api/dantri_hot")
457
  def api_dantri_hot():return JSONResponse(_cached("dantri_hot",scrape_dantri_hot))
458
+ @app.get("/api/vnreview_ai")
459
+ def api_vnreview_ai():return JSONResponse(_cached("vnreview_ai",scrape_vnreview_ai,ttl=_cache_ttl))
460
  @app.get("/api/article")
461
  def api_article(url:str=Query(...)):
462
  if"vnexpress.net" in url:data=scrape_vne_article(url)