Spaces:
Running
Running
fix: hash routing works, shorts fallback scraper, highlight img from og, video comments+views
Browse files
main.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""VNEWS - FastAPI backend
|
| 2 |
import hashlib, re, time
|
| 3 |
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 4 |
from fastapi import FastAPI, Query
|
|
@@ -8,305 +8,274 @@ import requests
|
|
| 8 |
from bs4 import BeautifulSoup
|
| 9 |
|
| 10 |
app = FastAPI()
|
| 11 |
-
|
| 12 |
-
HEADERS = {
|
| 13 |
-
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
| 14 |
-
"Accept-Language": "vi-VN,vi;q=0.9,en;q=0.8",
|
| 15 |
-
}
|
| 16 |
BASE_BDP = "https://bongdaplus.vn"
|
| 17 |
BASE_24H = "https://www.24h.com.vn"
|
| 18 |
SPACE_URL = "https://bep40-vnews.hf.space"
|
| 19 |
-
|
| 20 |
_cache = {}
|
| 21 |
_cache_ttl = 300
|
| 22 |
|
| 23 |
def _cached(key, fn):
|
| 24 |
-
now
|
| 25 |
-
if key in _cache and now
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
_cache[key] = {"d": data, "t": now}
|
| 30 |
-
return data
|
| 31 |
|
| 32 |
-
def _get(url,
|
| 33 |
-
h
|
| 34 |
-
|
| 35 |
-
return BeautifulSoup(r.text, "lxml")
|
| 36 |
|
| 37 |
-
# ═══ Scrapers ═══
|
| 38 |
def scrape_vne(cat_url):
|
| 39 |
try:
|
| 40 |
-
soup
|
| 41 |
for it in soup.select("article.item-news")[:15]:
|
| 42 |
-
a
|
| 43 |
-
if not a:
|
| 44 |
-
t
|
| 45 |
-
if not t or not lk:
|
| 46 |
-
im
|
| 47 |
-
img = (im.get("data-src") or im.get("src","")) if im else ""
|
| 48 |
if img and 'blank' in img:
|
| 49 |
-
src
|
| 50 |
-
if src:
|
| 51 |
-
desc
|
| 52 |
arts.append({"title":t,"link":lk,"img":img,"summary":(desc.get_text(strip=True)[:150] if desc else ""),"source":"vne"})
|
| 53 |
return arts
|
| 54 |
-
except:
|
| 55 |
|
| 56 |
def scrape_vne_article(url):
|
| 57 |
try:
|
| 58 |
-
soup
|
| 59 |
-
|
| 60 |
-
title = h1.get_text(strip=True) if h1 else ""
|
| 61 |
-
desc = soup.select_one("p.description")
|
| 62 |
-
summary = desc.get_text(strip=True) if desc else ""
|
| 63 |
-
cd = soup.select_one("article.fck_detail")
|
| 64 |
-
body = []
|
| 65 |
if cd:
|
| 66 |
for ch in cd.children:
|
| 67 |
-
if not hasattr(ch,'name') or not ch.name:
|
| 68 |
-
if ch.name
|
| 69 |
-
t
|
| 70 |
-
if t:
|
| 71 |
-
elif ch.name
|
| 72 |
-
im
|
| 73 |
if im:
|
| 74 |
-
s
|
| 75 |
-
cap
|
| 76 |
-
if s:
|
| 77 |
-
elif ch.name in
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
except: return None
|
| 81 |
|
| 82 |
def scrape_bdp(url):
|
| 83 |
try:
|
| 84 |
-
soup
|
| 85 |
-
for sel in
|
| 86 |
for it in soup.select(sel):
|
| 87 |
-
tag
|
| 88 |
-
if not tag:
|
| 89 |
-
t=tag.get_text(strip=True);
|
| 90 |
-
if not t or len(t)<5:
|
| 91 |
-
if lk and not lk.startswith("http"):
|
| 92 |
-
if lk in seen:
|
| 93 |
-
im=it.find("img");
|
| 94 |
-
seen.add(lk);
|
| 95 |
return arts[:15]
|
| 96 |
-
except:
|
| 97 |
|
| 98 |
def scrape_bdp_videos():
|
| 99 |
try:
|
| 100 |
-
soup
|
| 101 |
-
for a in soup.find_all("a",
|
| 102 |
-
href
|
| 103 |
-
if
|
| 104 |
-
if not href.startswith("http"):
|
| 105 |
-
if href in seen:
|
| 106 |
-
title
|
| 107 |
-
if not title or len(title)<5:
|
| 108 |
-
img_tag
|
| 109 |
-
img
|
| 110 |
-
seen.add(href);
|
| 111 |
return arts[:20]
|
| 112 |
-
except:
|
| 113 |
|
| 114 |
def scrape_24h_highlights():
|
|
|
|
| 115 |
try:
|
| 116 |
-
soup
|
| 117 |
-
for a in soup.find_all("a",
|
| 118 |
-
href
|
| 119 |
-
title
|
| 120 |
-
if not
|
| 121 |
-
if not href.
|
| 122 |
-
if href in seen or "video" not in href.lower() or href.endswith("-c953.html"): continue
|
| 123 |
seen.add(href)
|
| 124 |
-
img
|
| 125 |
-
img_src
|
| 126 |
-
if img:
|
| 127 |
-
|
| 128 |
-
if "base64" in img_src: img_src = ""
|
| 129 |
arts.append({"title":title,"link":href,"img":img_src,"source":"24h"})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
return arts[:30]
|
| 131 |
-
except:
|
| 132 |
|
| 133 |
def scrape_24h_shorts():
|
|
|
|
| 134 |
try:
|
| 135 |
-
|
|
|
|
|
|
|
|
|
|
| 136 |
for art in soup.find_all("article"):
|
| 137 |
-
a
|
| 138 |
-
if not a:
|
| 139 |
-
href
|
| 140 |
-
if not href.startswith("http"):
|
| 141 |
-
if href in seen:
|
| 142 |
-
img_tag
|
| 143 |
-
title
|
| 144 |
-
if not title or len(title)<10:
|
| 145 |
-
img_src
|
| 146 |
-
if img_tag:
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
return arts[:20]
|
| 151 |
-
except:
|
| 152 |
|
| 153 |
def scrape_bbc_vietnamese():
|
| 154 |
-
"""Scrape BBC Vietnamese - uses UK-based server headers."""
|
| 155 |
try:
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
"Accept": "text/html,application/xhtml+xml",
|
| 160 |
-
}
|
| 161 |
-
r = requests.get("https://www.bbc.com/vietnamese", headers=bbc_headers, timeout=15)
|
| 162 |
-
r.encoding = "utf-8"
|
| 163 |
-
soup = BeautifulSoup(r.text, "lxml")
|
| 164 |
-
arts = []; seen = set()
|
| 165 |
-
# BBC uses data-testid or specific class patterns
|
| 166 |
for a in soup.select("a[href*='/vietnamese/']"):
|
| 167 |
-
href
|
| 168 |
-
if not href or href
|
| 169 |
-
if href.
|
| 170 |
-
if
|
| 171 |
-
|
| 172 |
-
title
|
| 173 |
-
if
|
| 174 |
-
|
| 175 |
-
if any(x in title.lower() for x in ["đăng nhập","trang chủ","bbc news"]): continue
|
| 176 |
-
img = ""
|
| 177 |
-
# Look for image in parent/sibling
|
| 178 |
-
container = a.parent
|
| 179 |
for _ in range(3):
|
| 180 |
if container:
|
| 181 |
-
im
|
| 182 |
-
if im:
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
seen.add(href)
|
| 187 |
-
arts.append({"title":title,"link":href,"img":img,"source":"bbc","summary":""})
|
| 188 |
-
if len(arts) >= 15: break
|
| 189 |
return arts
|
| 190 |
-
except:
|
| 191 |
|
| 192 |
def scrape_bbc_article(url):
|
| 193 |
try:
|
| 194 |
-
|
| 195 |
-
r
|
| 196 |
-
soup
|
| 197 |
-
|
| 198 |
-
title = h1.get_text(strip=True) if h1 else ""
|
| 199 |
-
body = []
|
| 200 |
-
# BBC article paragraphs
|
| 201 |
for p in soup.select("[data-component='text-block'] p, article p, main p"):
|
| 202 |
-
t
|
| 203 |
-
if t and len(t)
|
| 204 |
-
body.append({"type":"p","text":t})
|
| 205 |
-
# Images
|
| 206 |
for img in soup.select("main img, article img"):
|
| 207 |
-
src
|
| 208 |
-
if src and
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
except: return None
|
| 212 |
|
| 213 |
def extract_video_url(article_url):
|
| 214 |
-
"""Extract m3u8 video URL + probe for multi-part."""
|
| 215 |
try:
|
| 216 |
-
r
|
| 217 |
-
r.
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
all_urls =
|
| 221 |
-
full
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
if
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
base_url = primary[0]
|
| 231 |
-
parts = [base_url]
|
| 232 |
-
is_720p = '_720p.m3u8' in base_url
|
| 233 |
-
name = base_url.replace('_720p.m3u8','').replace('.m3u8','')
|
| 234 |
-
m = re.search(r'(\D)(\d{1,2})$', name)
|
| 235 |
if m:
|
| 236 |
-
cur
|
| 237 |
-
for i in range(cur+1,
|
| 238 |
-
nn
|
| 239 |
-
for sfx in
|
| 240 |
-
pu
|
| 241 |
-
if pu in parts:
|
| 242 |
try:
|
| 243 |
-
if requests.head(pu,
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
else: break
|
| 247 |
-
# Pattern B: CDN id increment
|
| 248 |
if len(parts)==1:
|
| 249 |
-
m2
|
| 250 |
if m2:
|
| 251 |
-
pfx,cid,mid,pn,sfx
|
| 252 |
for i in range(1,5):
|
| 253 |
-
pu
|
| 254 |
try:
|
| 255 |
-
if requests.head(pu,
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
return {"src":parts[0],"poster":poster,"all_parts":parts}
|
| 262 |
-
return {"src":primary[0],"poster":poster}
|
| 263 |
-
except: return None
|
| 264 |
|
| 265 |
def extract_bdp_video(url):
|
| 266 |
try:
|
| 267 |
-
m
|
| 268 |
-
if not m:
|
| 269 |
-
|
| 270 |
-
r
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
source = video.find("source")
|
| 276 |
-
return {"src":source.get("src","") if source else "","poster":video.get("poster","")}
|
| 277 |
-
except: return None
|
| 278 |
|
| 279 |
# ═══ API ═══
|
| 280 |
-
VNE_CATS
|
| 281 |
-
|
| 282 |
-
"the-gioi":("https://vnexpress.net/the-gioi","Thế Giới"),
|
| 283 |
-
"kinh-doanh":("https://vnexpress.net/kinh-doanh","Kinh Doanh"),
|
| 284 |
-
"cong-nghe":("https://vnexpress.net/so-hoa","Công Nghệ"),
|
| 285 |
-
"the-thao":("https://vnexpress.net/the-thao","Thể Thao"),
|
| 286 |
-
"giai-tri":("https://vnexpress.net/giai-tri","Giải Trí"),
|
| 287 |
-
"suc-khoe":("https://vnexpress.net/suc-khoe","Sức Khỏe"),
|
| 288 |
-
}
|
| 289 |
-
BDP_CATS = {
|
| 290 |
-
"ngoai-hang-anh":("https://bongdaplus.vn/ngoai-hang-anh","Ngoại Hạng Anh"),
|
| 291 |
-
"la-liga":("https://bongdaplus.vn/la-liga","La Liga"),
|
| 292 |
-
"champions-league":("https://bongdaplus.vn/champions-league-cup-c1","Champions League"),
|
| 293 |
-
"bong-da-vn":("https://bongdaplus.vn/bong-da-viet-nam","Bóng Đá VN"),
|
| 294 |
-
}
|
| 295 |
|
| 296 |
@app.get("/api/homepage")
|
| 297 |
def api_homepage():
|
| 298 |
def _f():
|
| 299 |
articles=[]
|
| 300 |
with ThreadPoolExecutor(6) as ex:
|
| 301 |
-
futs={ex.submit(scrape_vne,VNE_CATS[k][0]):VNE_CATS[k][1] for k in
|
| 302 |
futs[ex.submit(scrape_bbc_vietnamese)]="BBC"
|
| 303 |
for f in as_completed(futs):
|
| 304 |
try:
|
| 305 |
-
for a in f.result():
|
| 306 |
-
except:
|
| 307 |
try:
|
| 308 |
-
for a in scrape_bdp(f"{BASE_BDP}/tin-moi")[:6]:
|
| 309 |
-
except:
|
| 310 |
return articles
|
| 311 |
return JSONResponse(_cached("homepage",_f))
|
| 312 |
|
|
@@ -314,14 +283,8 @@ def api_homepage():
|
|
| 314 |
def api_category(cat_id:str):
|
| 315 |
def _f():
|
| 316 |
if cat_id=="bbc":return scrape_bbc_vietnamese()
|
| 317 |
-
if cat_id in VNE_CATS:
|
| 318 |
-
|
| 319 |
-
for a in arts:a["group"]=VNE_CATS[cat_id][1]
|
| 320 |
-
return arts
|
| 321 |
-
if cat_id in BDP_CATS:
|
| 322 |
-
arts=scrape_bdp(BDP_CATS[cat_id][0])
|
| 323 |
-
for a in arts:a["group"]=BDP_CATS[cat_id][1]
|
| 324 |
-
return arts
|
| 325 |
return[]
|
| 326 |
return JSONResponse(_cached(f"cat_{cat_id}",_f))
|
| 327 |
|
|
@@ -346,28 +309,25 @@ def api_bdp_videos():
|
|
| 346 |
|
| 347 |
@app.get("/api/video_url")
|
| 348 |
def api_video_url(url:str=Query(...)):
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
else:
|
| 353 |
-
return JSONResponse(v if v else {"error":"not found"})
|
| 354 |
|
| 355 |
@app.get("/api/article")
|
| 356 |
def api_article(url:str=Query(...)):
|
| 357 |
-
if "vnexpress.net" in url:
|
| 358 |
-
elif "bbc.com" in url:
|
| 359 |
-
else:
|
| 360 |
-
return JSONResponse(data if data else
|
| 361 |
|
| 362 |
@app.get("/share/{path:path}")
|
| 363 |
async def share_page(path:str,img:str=Query(default=""),title:str=Query(default="VNEWS")):
|
| 364 |
og_image=img or "https://s1.vnecdn.net/vnexpress/restruct/i/v9505/logo_default.jpg"
|
| 365 |
-
|
| 366 |
-
return HTMLResponse(content=html)
|
| 367 |
|
| 368 |
@app.get("/")
|
| 369 |
async def index():
|
| 370 |
-
with open("/app/static/index.html","r",encoding="utf-8") as f:
|
| 371 |
-
return HTMLResponse(content=f.read())
|
| 372 |
|
| 373 |
app.mount("/static",StaticFiles(directory="/app/static"),name="static")
|
|
|
|
| 1 |
+
"""VNEWS - FastAPI backend."""
|
| 2 |
import hashlib, re, time
|
| 3 |
from concurrent.futures import ThreadPoolExecutor, as_completed
|
| 4 |
from fastapi import FastAPI, Query
|
|
|
|
| 8 |
from bs4 import BeautifulSoup
|
| 9 |
|
| 10 |
app = FastAPI()
|
| 11 |
+
HEADERS = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36","Accept-Language":"vi-VN,vi;q=0.9,en;q=0.8"}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
BASE_BDP = "https://bongdaplus.vn"
|
| 13 |
BASE_24H = "https://www.24h.com.vn"
|
| 14 |
SPACE_URL = "https://bep40-vnews.hf.space"
|
|
|
|
| 15 |
_cache = {}
|
| 16 |
_cache_ttl = 300
|
| 17 |
|
| 18 |
def _cached(key, fn):
|
| 19 |
+
now=time.time()
|
| 20 |
+
if key in _cache and now-_cache[key]["t"]<_cache_ttl: return _cache[key]["d"]
|
| 21 |
+
try: data=fn()
|
| 22 |
+
except: data=_cache.get(key,{}).get("d",[])
|
| 23 |
+
_cache[key]={"d":data,"t":now}; return data
|
|
|
|
|
|
|
| 24 |
|
| 25 |
+
def _get(url,headers=None):
|
| 26 |
+
h=headers or HEADERS; r=requests.get(url,headers=h,timeout=15); r.encoding="utf-8"
|
| 27 |
+
return BeautifulSoup(r.text,"lxml")
|
|
|
|
| 28 |
|
|
|
|
| 29 |
def scrape_vne(cat_url):
|
| 30 |
try:
|
| 31 |
+
soup=_get(cat_url);arts=[]
|
| 32 |
for it in soup.select("article.item-news")[:15]:
|
| 33 |
+
a=it.select_one("h2.title-news a") or it.select_one("h3.title-news a")
|
| 34 |
+
if not a:continue
|
| 35 |
+
t=a.get("title","") or a.get_text(strip=True);lk=a.get("href","")
|
| 36 |
+
if not t or not lk:continue
|
| 37 |
+
im=it.find("img");img=(im.get("data-src") or im.get("src","")) if im else ""
|
|
|
|
| 38 |
if img and 'blank' in img:
|
| 39 |
+
src=it.find("source")
|
| 40 |
+
if src:img=src.get("srcset","").split(",")[0].strip().split(" ")[0]
|
| 41 |
+
desc=it.select_one("p.description")
|
| 42 |
arts.append({"title":t,"link":lk,"img":img,"summary":(desc.get_text(strip=True)[:150] if desc else ""),"source":"vne"})
|
| 43 |
return arts
|
| 44 |
+
except:return[]
|
| 45 |
|
| 46 |
def scrape_vne_article(url):
|
| 47 |
try:
|
| 48 |
+
soup=_get(url);h1=soup.select_one("h1.title-detail");desc=soup.select_one("p.description")
|
| 49 |
+
cd=soup.select_one("article.fck_detail");body=[]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
if cd:
|
| 51 |
for ch in cd.children:
|
| 52 |
+
if not hasattr(ch,'name') or not ch.name:continue
|
| 53 |
+
if ch.name=="p":
|
| 54 |
+
t=ch.get_text(strip=True)
|
| 55 |
+
if t:body.append({"type":"p","text":t})
|
| 56 |
+
elif ch.name=="figure":
|
| 57 |
+
im=ch.find("img")
|
| 58 |
if im:
|
| 59 |
+
s=im.get("data-src") or im.get("src","")
|
| 60 |
+
cap=ch.find("figcaption")
|
| 61 |
+
if s:body.append({"type":"img","src":s,"alt":cap.get_text(strip=True) if cap else ""})
|
| 62 |
+
elif ch.name in("h2","h3"):body.append({"type":"heading","text":ch.get_text(strip=True)})
|
| 63 |
+
return{"title":h1.get_text(strip=True) if h1 else "","summary":desc.get_text(strip=True) if desc else "","body":body,"source":"vne","url":url}
|
| 64 |
+
except:return None
|
|
|
|
| 65 |
|
| 66 |
def scrape_bdp(url):
|
| 67 |
try:
|
| 68 |
+
soup=_get(url);arts=[];seen=set()
|
| 69 |
+
for sel in["div.news.fst","div.sld-itm.news","li.news"]:
|
| 70 |
for it in soup.select(sel):
|
| 71 |
+
tag=it.find("a",class_="title") or it.find("a",href=True)
|
| 72 |
+
if not tag:continue
|
| 73 |
+
t=tag.get_text(strip=True);lk=tag.get("href","")
|
| 74 |
+
if not t or len(t)<5:continue
|
| 75 |
+
if lk and not lk.startswith("http"):lk=BASE_BDP+lk
|
| 76 |
+
if lk in seen:continue
|
| 77 |
+
im=it.find("img");img=(im.get("data-src") or im.get("src","")) if im else ""
|
| 78 |
+
seen.add(lk);arts.append({"title":t,"link":lk,"img":img,"source":"bdp","summary":""})
|
| 79 |
return arts[:15]
|
| 80 |
+
except:return[]
|
| 81 |
|
| 82 |
def scrape_bdp_videos():
|
| 83 |
try:
|
| 84 |
+
soup=_get(f"{BASE_BDP}/video");arts=[];seen=set()
|
| 85 |
+
for a in soup.find_all("a",href=True):
|
| 86 |
+
href=a.get("href","")
|
| 87 |
+
if"/video/" not in href or href in("/video/","/video/ban-thang-dep","/video/highlight"):continue
|
| 88 |
+
if not href.startswith("http"):href=BASE_BDP+href
|
| 89 |
+
if href in seen:continue
|
| 90 |
+
title=re.sub(r'^\d{2}:\d{2}','',a.get_text(strip=True)).strip()
|
| 91 |
+
if not title or len(title)<5:continue
|
| 92 |
+
img_tag=a.find("img") or(a.parent.find("img") if a.parent else None)
|
| 93 |
+
img=(img_tag.get("data-src") or img_tag.get("src","")) if img_tag else ""
|
| 94 |
+
seen.add(href);arts.append({"title":title,"link":href,"img":img,"source":"bdp"})
|
| 95 |
return arts[:20]
|
| 96 |
+
except:return[]
|
| 97 |
|
| 98 |
def scrape_24h_highlights():
|
| 99 |
+
"""Scrape highlights + fetch og:image for items missing img."""
|
| 100 |
try:
|
| 101 |
+
soup=_get(f"{BASE_24H}/video-highlight-c953.html");arts=[];seen=set()
|
| 102 |
+
for a in soup.find_all("a",href=True):
|
| 103 |
+
href=a.get("href","");title=a.get("title","") or a.get_text(strip=True)
|
| 104 |
+
if not title or len(title)<15 or "javascript:" in href:continue
|
| 105 |
+
if not href.startswith("http"):href=BASE_24H+href
|
| 106 |
+
if href in seen or "video" not in href.lower() or href.endswith("-c953.html"):continue
|
|
|
|
| 107 |
seen.add(href)
|
| 108 |
+
img=a.find("img")
|
| 109 |
+
img_src=""
|
| 110 |
+
if img:img_src=img.get("data-original") or img.get("data-src") or img.get("src","")
|
| 111 |
+
if img_src and "base64" in img_src:img_src=""
|
|
|
|
| 112 |
arts.append({"title":title,"link":href,"img":img_src,"source":"24h"})
|
| 113 |
+
# Fetch og:image for items without img (parallel, max 10)
|
| 114 |
+
def _fetch_og(art):
|
| 115 |
+
if art["img"]:return art
|
| 116 |
+
try:
|
| 117 |
+
r=requests.get(art["link"],headers=HEADERS,timeout=8);r.encoding="utf-8"
|
| 118 |
+
s=BeautifulSoup(r.text,"lxml")
|
| 119 |
+
og=s.find("meta",property="og:image")
|
| 120 |
+
if og:art["img"]=og.get("content","")
|
| 121 |
+
except:pass
|
| 122 |
+
return art
|
| 123 |
+
need_img=[a for a in arts if not a["img"]][:10]
|
| 124 |
+
if need_img:
|
| 125 |
+
with ThreadPoolExecutor(5) as ex:
|
| 126 |
+
list(ex.map(_fetch_og,need_img))
|
| 127 |
return arts[:30]
|
| 128 |
+
except:return[]
|
| 129 |
|
| 130 |
def scrape_24h_shorts():
|
| 131 |
+
"""Scrape shorts - try <article> first, fallback to <a> with images."""
|
| 132 |
try:
|
| 133 |
+
r=requests.get(f"{BASE_24H}/video/video-tin-tuc-cvd769.html",headers=HEADERS,timeout=15)
|
| 134 |
+
r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
|
| 135 |
+
arts=[];seen=set()
|
| 136 |
+
# Method 1: <article> tags
|
| 137 |
for art in soup.find_all("article"):
|
| 138 |
+
a=art.find("a",href=True)
|
| 139 |
+
if not a:continue
|
| 140 |
+
href=a.get("href","")
|
| 141 |
+
if not href.startswith("http"):href=BASE_24H+href
|
| 142 |
+
if href in seen:continue
|
| 143 |
+
img_tag=art.find("img")
|
| 144 |
+
title=(img_tag.get("alt","") if img_tag else "") or a.get("title","") or a.get_text(strip=True)
|
| 145 |
+
if not title or len(title)<10:continue
|
| 146 |
+
img_src=""
|
| 147 |
+
if img_tag:img_src=img_tag.get("data-original") or img_tag.get("data-src") or img_tag.get("src","")
|
| 148 |
+
if img_src and "base64" in img_src:img_src=""
|
| 149 |
+
seen.add(href);arts.append({"title":title,"link":href,"img":img_src,"source":"24h-shorts"})
|
| 150 |
+
# Method 2: fallback - all <a> with <img> inside
|
| 151 |
+
if len(arts)<3:
|
| 152 |
+
for a in soup.find_all("a",href=True):
|
| 153 |
+
href=a.get("href","")
|
| 154 |
+
if not href.endswith(".html") or "javascript:" in href or "-cvd" in href:continue
|
| 155 |
+
if not href.startswith("http"):href=BASE_24H+href
|
| 156 |
+
if href in seen:continue
|
| 157 |
+
img=a.find("img")
|
| 158 |
+
if not img:
|
| 159 |
+
p=a.parent
|
| 160 |
+
if p:img=p.find("img")
|
| 161 |
+
if not img:continue
|
| 162 |
+
img_src=img.get("data-original") or img.get("data-src") or img.get("src","")
|
| 163 |
+
if not img_src or "base64" in img_src:continue
|
| 164 |
+
title=img.get("alt","") or a.get("title","") or a.get_text(strip=True)
|
| 165 |
+
if not title or len(title)<8:continue
|
| 166 |
+
seen.add(href);arts.append({"title":title,"link":href,"img":img_src,"source":"24h-shorts"})
|
| 167 |
return arts[:20]
|
| 168 |
+
except:return[]
|
| 169 |
|
| 170 |
def scrape_bbc_vietnamese():
|
|
|
|
| 171 |
try:
|
| 172 |
+
bbc_h={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36","Accept-Language":"en-GB,en;q=0.9"}
|
| 173 |
+
r=requests.get("https://www.bbc.com/vietnamese",headers=bbc_h,timeout=15);r.encoding="utf-8"
|
| 174 |
+
soup=BeautifulSoup(r.text,"lxml");arts=[];seen=set()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
for a in soup.select("a[href*='/vietnamese/']"):
|
| 176 |
+
href=a.get("href","")
|
| 177 |
+
if not href or href=="/vietnamese" or href.count("/")<3:continue
|
| 178 |
+
if not href.startswith("http"):href="https://www.bbc.com"+href
|
| 179 |
+
if href in seen:continue
|
| 180 |
+
title=a.get_text(strip=True)
|
| 181 |
+
if not title or len(title)<15:continue
|
| 182 |
+
if any(x in title.lower() for x in["đăng nhập","trang chủ","bbc news"]):continue
|
| 183 |
+
img="";container=a.parent
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
for _ in range(3):
|
| 185 |
if container:
|
| 186 |
+
im=container.find("img")
|
| 187 |
+
if im:img=im.get("src","") or im.get("data-src","");break
|
| 188 |
+
container=container.parent
|
| 189 |
+
seen.add(href);arts.append({"title":title,"link":href,"img":img,"source":"bbc","summary":""})
|
| 190 |
+
if len(arts)>=15:break
|
|
|
|
|
|
|
|
|
|
| 191 |
return arts
|
| 192 |
+
except:return[]
|
| 193 |
|
| 194 |
def scrape_bbc_article(url):
|
| 195 |
try:
|
| 196 |
+
bbc_h={"User-Agent":"Mozilla/5.0","Accept-Language":"en-GB,en;q=0.9"}
|
| 197 |
+
r=requests.get(url,headers=bbc_h,timeout=15);r.encoding="utf-8"
|
| 198 |
+
soup=BeautifulSoup(r.text,"lxml");h1=soup.find("h1")
|
| 199 |
+
body=[]
|
|
|
|
|
|
|
|
|
|
| 200 |
for p in soup.select("[data-component='text-block'] p, article p, main p"):
|
| 201 |
+
t=p.get_text(strip=True)
|
| 202 |
+
if t and len(t)>20:body.append({"type":"p","text":t})
|
|
|
|
|
|
|
| 203 |
for img in soup.select("main img, article img"):
|
| 204 |
+
src=img.get("src","")
|
| 205 |
+
if src and("ichef" in src or "bbci" in src):body.append({"type":"img","src":src,"alt":img.get("alt","")})
|
| 206 |
+
return{"title":h1.get_text(strip=True) if h1 else "","summary":"","body":body,"source":"bbc","url":url}
|
| 207 |
+
except:return None
|
|
|
|
| 208 |
|
| 209 |
def extract_video_url(article_url):
|
|
|
|
| 210 |
try:
|
| 211 |
+
r=requests.get(article_url,headers={**HEADERS,"Referer":"https://www.24h.com.vn/"},timeout=10);r.encoding="utf-8"
|
| 212 |
+
m3u8s=re.findall(r'(https?://cdn\.24h\.com\.vn/[^\s"\'\\<>]+\.m3u8)',r.text)
|
| 213 |
+
esc=[u.replace('\\//','/').replace('\\/','/') for u in re.findall(r'(https?:\\/\\/cdn\.24h\.com\.vn\\/[^\s"\'<>]+\.m3u8)',r.text)]
|
| 214 |
+
all_urls=list(dict.fromkeys(m3u8s+esc))
|
| 215 |
+
full=[u for u in all_urls if'_720p' not in u];p720=[u for u in all_urls if'_720p' in u]
|
| 216 |
+
primary=full or p720
|
| 217 |
+
if not primary:return None
|
| 218 |
+
soup=BeautifulSoup(r.text,"lxml");og=soup.find("meta",property="og:image")
|
| 219 |
+
poster=og.get("content","") if og else ""
|
| 220 |
+
# Multi-part probe
|
| 221 |
+
base_url=primary[0];parts=[base_url]
|
| 222 |
+
is_720p='_720p.m3u8' in base_url
|
| 223 |
+
name=base_url.replace('_720p.m3u8','').replace('.m3u8','')
|
| 224 |
+
m=re.search(r'(\D)(\d{1,2})$',name)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 225 |
if m:
|
| 226 |
+
cur=int(m.group(2));w=len(m.group(2));bn=name[:m.start(2)]
|
| 227 |
+
for i in range(cur+1,cur+5):
|
| 228 |
+
nn=f"{bn}{i:02d}" if w==2 else f"{bn}{i}"
|
| 229 |
+
for sfx in[('_720p.m3u8' if is_720p else '.m3u8'),'_720p.m3u8','.m3u8']:
|
| 230 |
+
pu=nn+sfx
|
| 231 |
+
if pu in parts:continue
|
| 232 |
try:
|
| 233 |
+
if requests.head(pu,headers=HEADERS,timeout=3,allow_redirects=True).status_code==200:parts.append(pu);break
|
| 234 |
+
except:continue
|
| 235 |
+
else:break
|
|
|
|
|
|
|
| 236 |
if len(parts)==1:
|
| 237 |
+
m2=re.match(r'^(.+-)(\d{7,})(-.+?)(\d{1,2})((?:_720p)?\.m3u8)$',base_url)
|
| 238 |
if m2:
|
| 239 |
+
pfx,cid,mid,pn,sfx=m2.group(1),int(m2.group(2)),m2.group(3),int(m2.group(4)),m2.group(5)
|
| 240 |
for i in range(1,5):
|
| 241 |
+
pu=f"{pfx}{cid+i}{mid}{pn+i}{sfx}"
|
| 242 |
try:
|
| 243 |
+
if requests.head(pu,headers=HEADERS,timeout=3,allow_redirects=True).status_code==200:parts.append(pu)
|
| 244 |
+
else:break
|
| 245 |
+
except:break
|
| 246 |
+
if len(parts)>1:return{"src":parts[0],"poster":poster,"all_parts":parts}
|
| 247 |
+
return{"src":primary[0],"poster":poster}
|
| 248 |
+
except:return None
|
|
|
|
|
|
|
|
|
|
| 249 |
|
| 250 |
def extract_bdp_video(url):
|
| 251 |
try:
|
| 252 |
+
m=re.search(r'-(\d{6,})\.html',url)
|
| 253 |
+
if not m:return None
|
| 254 |
+
r=requests.get(f"{BASE_BDP}/video-embed/{m.group(1)}.html",headers=HEADERS,timeout=10);r.encoding="utf-8"
|
| 255 |
+
soup=BeautifulSoup(r.text,"lxml");video=soup.select_one("video#videoPlayer")
|
| 256 |
+
if not video:return None
|
| 257 |
+
source=video.find("source")
|
| 258 |
+
return{"src":source.get("src","") if source else "","poster":video.get("poster","")}
|
| 259 |
+
except:return None
|
|
|
|
|
|
|
|
|
|
| 260 |
|
| 261 |
# ═══ API ═══
|
| 262 |
+
VNE_CATS={"thoi-su":("https://vnexpress.net/thoi-su","Thời Sự"),"the-gioi":("https://vnexpress.net/the-gioi","Thế Giới"),"kinh-doanh":("https://vnexpress.net/kinh-doanh","Kinh Doanh"),"cong-nghe":("https://vnexpress.net/so-hoa","Công Nghệ"),"the-thao":("https://vnexpress.net/the-thao","Thể Thao"),"giai-tri":("https://vnexpress.net/giai-tri","Giải Trí"),"suc-khoe":("https://vnexpress.net/suc-khoe","Sức Khỏe")}
|
| 263 |
+
BDP_CATS={"ngoai-hang-anh":("https://bongdaplus.vn/ngoai-hang-anh","Ngoại Hạng Anh"),"la-liga":("https://bongdaplus.vn/la-liga","La Liga"),"champions-league":("https://bongdaplus.vn/champions-league-cup-c1","Champions League"),"bong-da-vn":("https://bongdaplus.vn/bong-da-viet-nam","Bóng Đá VN")}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 264 |
|
| 265 |
@app.get("/api/homepage")
|
| 266 |
def api_homepage():
|
| 267 |
def _f():
|
| 268 |
articles=[]
|
| 269 |
with ThreadPoolExecutor(6) as ex:
|
| 270 |
+
futs={ex.submit(scrape_vne,VNE_CATS[k][0]):VNE_CATS[k][1] for k in["thoi-su","the-gioi","kinh-doanh","the-thao","giai-tri"]}
|
| 271 |
futs[ex.submit(scrape_bbc_vietnamese)]="BBC"
|
| 272 |
for f in as_completed(futs):
|
| 273 |
try:
|
| 274 |
+
for a in f.result():a["group"]=futs[f];articles.append(a)
|
| 275 |
+
except:pass
|
| 276 |
try:
|
| 277 |
+
for a in scrape_bdp(f"{BASE_BDP}/tin-moi")[:6]:a["group"]="Bóng Đá";articles.append(a)
|
| 278 |
+
except:pass
|
| 279 |
return articles
|
| 280 |
return JSONResponse(_cached("homepage",_f))
|
| 281 |
|
|
|
|
| 283 |
def api_category(cat_id:str):
|
| 284 |
def _f():
|
| 285 |
if cat_id=="bbc":return scrape_bbc_vietnamese()
|
| 286 |
+
if cat_id in VNE_CATS:arts=scrape_vne(VNE_CATS[cat_id][0]);[a.update({"group":VNE_CATS[cat_id][1]}) for a in arts];return arts
|
| 287 |
+
if cat_id in BDP_CATS:arts=scrape_bdp(BDP_CATS[cat_id][0]);[a.update({"group":BDP_CATS[cat_id][1]}) for a in arts];return arts
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 288 |
return[]
|
| 289 |
return JSONResponse(_cached(f"cat_{cat_id}",_f))
|
| 290 |
|
|
|
|
| 309 |
|
| 310 |
@app.get("/api/video_url")
|
| 311 |
def api_video_url(url:str=Query(...)):
|
| 312 |
+
if "24h.com.vn" in url:v=extract_video_url(url)
|
| 313 |
+
elif "bongdaplus.vn" in url:v=extract_bdp_video(url)
|
| 314 |
+
else:v=None
|
| 315 |
+
return JSONResponse(v if v else{"error":"not found"})
|
|
|
|
| 316 |
|
| 317 |
@app.get("/api/article")
|
| 318 |
def api_article(url:str=Query(...)):
|
| 319 |
+
if "vnexpress.net" in url:data=scrape_vne_article(url)
|
| 320 |
+
elif "bbc.com" in url:data=scrape_bbc_article(url)
|
| 321 |
+
else:data=None
|
| 322 |
+
return JSONResponse(data if data else{"error":"not supported"})
|
| 323 |
|
| 324 |
@app.get("/share/{path:path}")
|
| 325 |
async def share_page(path:str,img:str=Query(default=""),title:str=Query(default="VNEWS")):
|
| 326 |
og_image=img or "https://s1.vnecdn.net/vnexpress/restruct/i/v9505/logo_default.jpg"
|
| 327 |
+
return HTMLResponse(f'<!DOCTYPE html><html><head><meta charset="utf-8"><title>{title}</title><meta property="og:title" content="{title}"><meta property="og:image" content="{og_image}"><meta property="og:type" content="article"><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="{og_image}"><meta http-equiv="refresh" content="0;url={SPACE_URL}/#/{path}"></head><body>Redirecting...</body></html>')
|
|
|
|
| 328 |
|
| 329 |
@app.get("/")
|
| 330 |
async def index():
|
| 331 |
+
with open("/app/static/index.html","r",encoding="utf-8") as f:return HTMLResponse(content=f.read())
|
|
|
|
| 332 |
|
| 333 |
app.mount("/static",StaticFiles(directory="/app/static"),name="static")
|