Spaces:
Running
Running
Fix: Công nghệ scraper accepts articles without images from DanTri
Browse files
main.py
CHANGED
|
@@ -321,7 +321,25 @@ def _scrape_dantri_homepage(cat_filter=None):
|
|
| 321 |
return arts
|
| 322 |
except:return[]
|
| 323 |
def scrape_dantri_hot():return _scrape_dantri_homepage()
|
| 324 |
-
def scrape_dantri_congnghe():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
def scrape_dantri_article(url):
|
| 326 |
try:
|
| 327 |
r=requests.get(url,headers=HEADERS,timeout=15);r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
|
|
|
|
| 321 |
return arts
|
| 322 |
except:return[]
|
| 323 |
def scrape_dantri_hot():return _scrape_dantri_homepage()
|
| 324 |
+
def scrape_dantri_congnghe():
|
| 325 |
+
try:
|
| 326 |
+
soup=_get("https://dantri.com.vn/");arts=[];seen=set()
|
| 327 |
+
for a in soup.find_all("a",href=True):
|
| 328 |
+
href=a.get("href","");title=a.get("title","") or a.get_text(strip=True)
|
| 329 |
+
if not title or len(title)<15 or"javascript:" in href:continue
|
| 330 |
+
if not href.startswith("http"):href="https://dantri.com.vn"+href
|
| 331 |
+
if href in seen or not href.endswith(".htm"):continue
|
| 332 |
+
if"/cong-nghe/" not in href:continue
|
| 333 |
+
img_tag=a.find("img")
|
| 334 |
+
if not img_tag and a.parent:img_tag=a.parent.find("img")
|
| 335 |
+
img_src=""
|
| 336 |
+
if img_tag:img_src=img_tag.get("data-src","") or img_tag.get("src","")
|
| 337 |
+
if img_src and "cdn" in img_src:img_src="/api/proxy/img?url="+quote(img_src,safe="")
|
| 338 |
+
else:img_src=""
|
| 339 |
+
seen.add(href);arts.append({"title":title,"link":href,"img":img_src,"source":"dantri"})
|
| 340 |
+
if len(arts)>=15:break
|
| 341 |
+
return arts
|
| 342 |
+
except:return[]
|
| 343 |
def scrape_dantri_article(url):
|
| 344 |
try:
|
| 345 |
r=requests.get(url,headers=HEADERS,timeout=15);r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
|