Spaces:
Running
Running
Add vnreview.vn AI scraper + /api/vnreview_ai endpoint
Browse files
main.py
CHANGED
|
@@ -340,6 +340,34 @@ def scrape_dantri_congnghe():
|
|
| 340 |
if len(arts)>=15:break
|
| 341 |
return arts
|
| 342 |
except:return[]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 343 |
def scrape_dantri_article(url):
|
| 344 |
try:
|
| 345 |
r=requests.get(url,headers=HEADERS,timeout=15);r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
|
|
@@ -427,6 +455,8 @@ def api_categories():
|
|
| 427 |
return JSONResponse(cats)
|
| 428 |
@app.get("/api/dantri_hot")
|
| 429 |
def api_dantri_hot():return JSONResponse(_cached("dantri_hot",scrape_dantri_hot))
|
|
|
|
|
|
|
| 430 |
@app.get("/api/article")
|
| 431 |
def api_article(url:str=Query(...)):
|
| 432 |
if"vnexpress.net" in url:data=scrape_vne_article(url)
|
|
|
|
| 340 |
if len(arts)>=15:break
|
| 341 |
return arts
|
| 342 |
except:return[]
|
| 343 |
+
def scrape_vnreview_ai():
|
| 344 |
+
"""Scrape AI articles from vnreview.vn"""
|
| 345 |
+
try:
|
| 346 |
+
r=requests.get("https://vnreview.vn/chuyen-muc/ai-pho-thong.28/",headers=HEADERS,timeout=15)
|
| 347 |
+
if r.status_code!=200:return[]
|
| 348 |
+
r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
|
| 349 |
+
import re as _re
|
| 350 |
+
articles=[];seen=set()
|
| 351 |
+
for img in soup.find_all("img"):
|
| 352 |
+
src=img.get("src","") or img.get("data-src","")
|
| 353 |
+
if "cdn.vnreview.vn" not in src or "avatars" in src or "logo" in src:continue
|
| 354 |
+
parent=img.parent;link=None
|
| 355 |
+
for _ in range(5):
|
| 356 |
+
if parent is None:break
|
| 357 |
+
link=parent.find("a",href=_re.compile(r"/threads/"))
|
| 358 |
+
if link:break
|
| 359 |
+
parent=parent.parent
|
| 360 |
+
if not link:continue
|
| 361 |
+
href=link.get("href","")
|
| 362 |
+
if not href.startswith("http"):href="https://vnreview.vn"+href
|
| 363 |
+
if href in seen:continue
|
| 364 |
+
title=link.get("title","") or link.get_text(strip=True) or img.get("alt","")
|
| 365 |
+
if not title or len(title)<10:continue
|
| 366 |
+
seen.add(href);articles.append({"title":title,"link":href,"img":src,"source":"vnreview"})
|
| 367 |
+
if len(articles)>=12:break
|
| 368 |
+
return articles
|
| 369 |
+
except:return[]
|
| 370 |
+
|
| 371 |
def scrape_dantri_article(url):
|
| 372 |
try:
|
| 373 |
r=requests.get(url,headers=HEADERS,timeout=15);r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
|
|
|
|
| 455 |
return JSONResponse(cats)
|
| 456 |
@app.get("/api/dantri_hot")
|
| 457 |
def api_dantri_hot():return JSONResponse(_cached("dantri_hot",scrape_dantri_hot))
|
| 458 |
+
@app.get("/api/vnreview_ai")
|
| 459 |
+
def api_vnreview_ai():return JSONResponse(_cached("vnreview_ai",scrape_vnreview_ai,ttl=_cache_ttl))
|
| 460 |
@app.get("/api/article")
|
| 461 |
def api_article(url:str=Query(...)):
|
| 462 |
if"vnexpress.net" in url:data=scrape_vne_article(url)
|