bep40 commited on
Commit
79b33c4
·
verified ·
1 Parent(s): 840f64e

Fix: Công nghệ scraper accepts articles without images from DanTri

Browse files
Files changed (1) hide show
  1. main.py +19 -1
main.py CHANGED
@@ -321,7 +321,25 @@ def _scrape_dantri_homepage(cat_filter=None):
321
  return arts
322
  except:return[]
323
  def scrape_dantri_hot():return _scrape_dantri_homepage()
324
- def scrape_dantri_congnghe():return _scrape_dantri_homepage("cong-nghe")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
325
  def scrape_dantri_article(url):
326
  try:
327
  r=requests.get(url,headers=HEADERS,timeout=15);r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
 
321
  return arts
322
  except:return[]
323
  def scrape_dantri_hot():return _scrape_dantri_homepage()
324
+ def scrape_dantri_congnghe():
325
+ try:
326
+ soup=_get("https://dantri.com.vn/");arts=[];seen=set()
327
+ for a in soup.find_all("a",href=True):
328
+ href=a.get("href","");title=a.get("title","") or a.get_text(strip=True)
329
+ if not title or len(title)<15 or"javascript:" in href:continue
330
+ if not href.startswith("http"):href="https://dantri.com.vn"+href
331
+ if href in seen or not href.endswith(".htm"):continue
332
+ if"/cong-nghe/" not in href:continue
333
+ img_tag=a.find("img")
334
+ if not img_tag and a.parent:img_tag=a.parent.find("img")
335
+ img_src=""
336
+ if img_tag:img_src=img_tag.get("data-src","") or img_tag.get("src","")
337
+ if img_src and "cdn" in img_src:img_src="/api/proxy/img?url="+quote(img_src,safe="")
338
+ else:img_src=""
339
+ seen.add(href);arts.append({"title":title,"link":href,"img":img_src,"source":"dantri"})
340
+ if len(arts)>=15:break
341
+ return arts
342
+ except:return[]
343
  def scrape_dantri_article(url):
344
  try:
345
  r=requests.get(url,headers=HEADERS,timeout=15);r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")