bep40 commited on
Commit
4466749
·
verified ·
1 Parent(s): ffea884

fix: hash routing works, shorts fallback scraper, highlight img from og, video comments+views

Browse files
Files changed (1) hide show
  1. main.py +204 -244
main.py CHANGED
@@ -1,4 +1,4 @@
1
- """VNEWS - FastAPI backend with news scrapers and API."""
2
  import hashlib, re, time
3
  from concurrent.futures import ThreadPoolExecutor, as_completed
4
  from fastapi import FastAPI, Query
@@ -8,305 +8,274 @@ import requests
8
  from bs4 import BeautifulSoup
9
 
10
  app = FastAPI()
11
-
12
- HEADERS = {
13
- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
14
- "Accept-Language": "vi-VN,vi;q=0.9,en;q=0.8",
15
- }
16
  BASE_BDP = "https://bongdaplus.vn"
17
  BASE_24H = "https://www.24h.com.vn"
18
  SPACE_URL = "https://bep40-vnews.hf.space"
19
-
20
  _cache = {}
21
  _cache_ttl = 300
22
 
23
  def _cached(key, fn):
24
- now = time.time()
25
- if key in _cache and now - _cache[key]["t"] < _cache_ttl:
26
- return _cache[key]["d"]
27
- try: data = fn()
28
- except: data = _cache.get(key, {}).get("d", [])
29
- _cache[key] = {"d": data, "t": now}
30
- return data
31
 
32
- def _get(url, headers=None):
33
- h = headers or HEADERS
34
- r = requests.get(url, headers=h, timeout=15); r.encoding="utf-8"
35
- return BeautifulSoup(r.text, "lxml")
36
 
37
- # ═══ Scrapers ═══
38
  def scrape_vne(cat_url):
39
  try:
40
- soup = _get(cat_url); arts = []
41
  for it in soup.select("article.item-news")[:15]:
42
- a = it.select_one("h2.title-news a") or it.select_one("h3.title-news a")
43
- if not a: continue
44
- t = a.get("title","") or a.get_text(strip=True); lk = a.get("href","")
45
- if not t or not lk: continue
46
- im = it.find("img")
47
- img = (im.get("data-src") or im.get("src","")) if im else ""
48
  if img and 'blank' in img:
49
- src = it.find("source")
50
- if src: img = src.get("srcset","").split(",")[0].strip().split(" ")[0]
51
- desc = it.select_one("p.description")
52
  arts.append({"title":t,"link":lk,"img":img,"summary":(desc.get_text(strip=True)[:150] if desc else ""),"source":"vne"})
53
  return arts
54
- except: return []
55
 
56
  def scrape_vne_article(url):
57
  try:
58
- soup = _get(url)
59
- h1 = soup.select_one("h1.title-detail")
60
- title = h1.get_text(strip=True) if h1 else ""
61
- desc = soup.select_one("p.description")
62
- summary = desc.get_text(strip=True) if desc else ""
63
- cd = soup.select_one("article.fck_detail")
64
- body = []
65
  if cd:
66
  for ch in cd.children:
67
- if not hasattr(ch,'name') or not ch.name: continue
68
- if ch.name == "p":
69
- t = ch.get_text(strip=True)
70
- if t: body.append({"type":"p","text":t})
71
- elif ch.name == "figure":
72
- im = ch.find("img")
73
  if im:
74
- s = im.get("data-src") or im.get("src","")
75
- cap = ch.find("figcaption")
76
- if s: body.append({"type":"img","src":s,"alt":cap.get_text(strip=True) if cap else ""})
77
- elif ch.name in ("h2","h3"):
78
- body.append({"type":"heading","text":ch.get_text(strip=True)})
79
- return {"title":title,"summary":summary,"body":body,"source":"vne","url":url}
80
- except: return None
81
 
82
  def scrape_bdp(url):
83
  try:
84
- soup = _get(url); arts = []; seen = set()
85
- for sel in ["div.news.fst","div.sld-itm.news","li.news"]:
86
  for it in soup.select(sel):
87
- tag = it.find("a",class_="title") or it.find("a",href=True)
88
- if not tag: continue
89
- t=tag.get_text(strip=True); lk=tag.get("href","")
90
- if not t or len(t)<5: continue
91
- if lk and not lk.startswith("http"): lk=BASE_BDP+lk
92
- if lk in seen: continue
93
- im=it.find("img"); img=(im.get("data-src") or im.get("src","")) if im else ""
94
- seen.add(lk); arts.append({"title":t,"link":lk,"img":img,"source":"bdp","summary":""})
95
  return arts[:15]
96
- except: return []
97
 
98
  def scrape_bdp_videos():
99
  try:
100
- soup = _get(f"{BASE_BDP}/video"); arts = []; seen = set()
101
- for a in soup.find_all("a", href=True):
102
- href = a.get("href","")
103
- if "/video/" not in href or href in ("/video/","/video/ban-thang-dep","/video/highlight"): continue
104
- if not href.startswith("http"): href = BASE_BDP + href
105
- if href in seen: continue
106
- title = re.sub(r'^\d{2}:\d{2}','',a.get_text(strip=True)).strip()
107
- if not title or len(title)<5: continue
108
- img_tag = a.find("img") or (a.parent.find("img") if a.parent else None)
109
- img = (img_tag.get("data-src") or img_tag.get("src","")) if img_tag else ""
110
- seen.add(href); arts.append({"title":title,"link":href,"img":img,"source":"bdp"})
111
  return arts[:20]
112
- except: return []
113
 
114
  def scrape_24h_highlights():
 
115
  try:
116
- soup = _get(f"{BASE_24H}/video-highlight-c953.html"); arts = []; seen = set()
117
- for a in soup.find_all("a", href=True):
118
- href = a.get("href","")
119
- title = a.get("title","") or a.get_text(strip=True)
120
- if not title or len(title)<15 or "javascript:" in href: continue
121
- if not href.startswith("http"): href = BASE_24H + href
122
- if href in seen or "video" not in href.lower() or href.endswith("-c953.html"): continue
123
  seen.add(href)
124
- img = a.find("img")
125
- img_src = ""
126
- if img:
127
- img_src = img.get("data-original") or img.get("data-src") or img.get("src","")
128
- if "base64" in img_src: img_src = ""
129
  arts.append({"title":title,"link":href,"img":img_src,"source":"24h"})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
130
  return arts[:30]
131
- except: return []
132
 
133
  def scrape_24h_shorts():
 
134
  try:
135
- soup = _get(f"{BASE_24H}/video/video-tin-tuc-cvd769.html"); arts = []; seen = set()
 
 
 
136
  for art in soup.find_all("article"):
137
- a = art.find("a", href=True)
138
- if not a: continue
139
- href = a.get("href","")
140
- if not href.startswith("http"): href = BASE_24H + href
141
- if href in seen: continue
142
- img_tag = art.find("img")
143
- title = (img_tag.get("alt","") if img_tag else "") or a.get("title","") or a.get_text(strip=True)
144
- if not title or len(title)<10: continue
145
- img_src = ""
146
- if img_tag:
147
- img_src = img_tag.get("data-original") or img_tag.get("data-src") or img_tag.get("src","")
148
- if "base64" in img_src: img_src = ""
149
- seen.add(href); arts.append({"title":title,"link":href,"img":img_src,"source":"24h-shorts"})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
150
  return arts[:20]
151
- except: return []
152
 
153
  def scrape_bbc_vietnamese():
154
- """Scrape BBC Vietnamese - uses UK-based server headers."""
155
  try:
156
- bbc_headers = {
157
- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
158
- "Accept-Language": "en-GB,en;q=0.9,vi;q=0.8",
159
- "Accept": "text/html,application/xhtml+xml",
160
- }
161
- r = requests.get("https://www.bbc.com/vietnamese", headers=bbc_headers, timeout=15)
162
- r.encoding = "utf-8"
163
- soup = BeautifulSoup(r.text, "lxml")
164
- arts = []; seen = set()
165
- # BBC uses data-testid or specific class patterns
166
  for a in soup.select("a[href*='/vietnamese/']"):
167
- href = a.get("href","")
168
- if not href or href == "/vietnamese" or href == "/vietnamese/": continue
169
- if href.count("/") < 3: continue
170
- if not href.startswith("http"): href = "https://www.bbc.com" + href
171
- if href in seen: continue
172
- title = a.get_text(strip=True)
173
- if not title or len(title) < 15: continue
174
- # Skip navigation/footer links
175
- if any(x in title.lower() for x in ["đăng nhập","trang chủ","bbc news"]): continue
176
- img = ""
177
- # Look for image in parent/sibling
178
- container = a.parent
179
  for _ in range(3):
180
  if container:
181
- im = container.find("img")
182
- if im:
183
- img = im.get("src","") or im.get("data-src","")
184
- break
185
- container = container.parent
186
- seen.add(href)
187
- arts.append({"title":title,"link":href,"img":img,"source":"bbc","summary":""})
188
- if len(arts) >= 15: break
189
  return arts
190
- except: return []
191
 
192
  def scrape_bbc_article(url):
193
  try:
194
- bbc_headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36","Accept-Language":"en-GB,en;q=0.9"}
195
- r = requests.get(url, headers=bbc_headers, timeout=15); r.encoding="utf-8"
196
- soup = BeautifulSoup(r.text,"lxml")
197
- h1 = soup.find("h1")
198
- title = h1.get_text(strip=True) if h1 else ""
199
- body = []
200
- # BBC article paragraphs
201
  for p in soup.select("[data-component='text-block'] p, article p, main p"):
202
- t = p.get_text(strip=True)
203
- if t and len(t) > 20 and t != title:
204
- body.append({"type":"p","text":t})
205
- # Images
206
  for img in soup.select("main img, article img"):
207
- src = img.get("src","")
208
- if src and ("ichef" in src or "bbci" in src):
209
- body.append({"type":"img","src":src,"alt":img.get("alt","")})
210
- return {"title":title,"summary":"","body":body,"source":"bbc","url":url}
211
- except: return None
212
 
213
  def extract_video_url(article_url):
214
- """Extract m3u8 video URL + probe for multi-part."""
215
  try:
216
- r = requests.get(article_url, headers={**HEADERS,"Referer":"https://www.24h.com.vn/"}, timeout=10)
217
- r.encoding="utf-8"
218
- m3u8s = re.findall(r'(https?://cdn\.24h\.com\.vn/[^\s"\'\\<>]+\.m3u8)', r.text)
219
- esc = [u.replace('\\//','/').replace('\\/','/') for u in re.findall(r'(https?:\\/\\/cdn\.24h\.com\.vn\\/[^\s"\'<>]+\.m3u8)', r.text)]
220
- all_urls = list(dict.fromkeys(m3u8s + esc))
221
- full = [u for u in all_urls if '_720p' not in u]
222
- p720 = [u for u in all_urls if '_720p' in u]
223
- primary = full or p720
224
- if not primary: return None
225
- soup = BeautifulSoup(r.text,"lxml")
226
- og = soup.find("meta",property="og:image")
227
- poster = og.get("content","") if og else ""
228
-
229
- # Probe for multi-part
230
- base_url = primary[0]
231
- parts = [base_url]
232
- is_720p = '_720p.m3u8' in base_url
233
- name = base_url.replace('_720p.m3u8','').replace('.m3u8','')
234
- m = re.search(r'(\D)(\d{1,2})$', name)
235
  if m:
236
- cur = int(m.group(2)); w = len(m.group(2)); base_name = name[:m.start(2)]
237
- for i in range(cur+1, cur+5):
238
- nn = f"{base_name}{i:02d}" if w==2 else f"{base_name}{i}"
239
- for sfx in [('_720p.m3u8' if is_720p else '.m3u8'),'_720p.m3u8','.m3u8']:
240
- pu = nn+sfx
241
- if pu in parts: continue
242
  try:
243
- if requests.head(pu, headers=HEADERS, timeout=3, allow_redirects=True).status_code==200:
244
- parts.append(pu); break
245
- except: continue
246
- else: break
247
- # Pattern B: CDN id increment
248
  if len(parts)==1:
249
- m2 = re.match(r'^(.+-)(\d{7,})(-.+?)(\d{1,2})((?:_720p)?\.m3u8)$', base_url)
250
  if m2:
251
- pfx,cid,mid,pn,sfx = m2.group(1),int(m2.group(2)),m2.group(3),int(m2.group(4)),m2.group(5)
252
  for i in range(1,5):
253
- pu = f"{pfx}{cid+i}{mid}{pn+i}{sfx}"
254
  try:
255
- if requests.head(pu, headers=HEADERS, timeout=3, allow_redirects=True).status_code==200:
256
- parts.append(pu)
257
- else: break
258
- except: break
259
-
260
- if len(parts)>1:
261
- return {"src":parts[0],"poster":poster,"all_parts":parts}
262
- return {"src":primary[0],"poster":poster}
263
- except: return None
264
 
265
  def extract_bdp_video(url):
266
  try:
267
- m = re.search(r'-(\d{6,})\.html', url)
268
- if not m: return None
269
- vid_id = m.group(1)
270
- r = requests.get(f"{BASE_BDP}/video-embed/{vid_id}.html", headers=HEADERS, timeout=10)
271
- r.encoding="utf-8"
272
- soup = BeautifulSoup(r.text,"lxml")
273
- video = soup.select_one("video#videoPlayer")
274
- if not video: return None
275
- source = video.find("source")
276
- return {"src":source.get("src","") if source else "","poster":video.get("poster","")}
277
- except: return None
278
 
279
  # ═══ API ═══
280
- VNE_CATS = {
281
- "thoi-su":("https://vnexpress.net/thoi-su","Thời Sự"),
282
- "the-gioi":("https://vnexpress.net/the-gioi","Thế Giới"),
283
- "kinh-doanh":("https://vnexpress.net/kinh-doanh","Kinh Doanh"),
284
- "cong-nghe":("https://vnexpress.net/so-hoa","Công Nghệ"),
285
- "the-thao":("https://vnexpress.net/the-thao","Thể Thao"),
286
- "giai-tri":("https://vnexpress.net/giai-tri","Giải Trí"),
287
- "suc-khoe":("https://vnexpress.net/suc-khoe","Sức Khỏe"),
288
- }
289
- BDP_CATS = {
290
- "ngoai-hang-anh":("https://bongdaplus.vn/ngoai-hang-anh","Ngoại Hạng Anh"),
291
- "la-liga":("https://bongdaplus.vn/la-liga","La Liga"),
292
- "champions-league":("https://bongdaplus.vn/champions-league-cup-c1","Champions League"),
293
- "bong-da-vn":("https://bongdaplus.vn/bong-da-viet-nam","Bóng Đá VN"),
294
- }
295
 
296
  @app.get("/api/homepage")
297
  def api_homepage():
298
  def _f():
299
  articles=[]
300
  with ThreadPoolExecutor(6) as ex:
301
- futs={ex.submit(scrape_vne,VNE_CATS[k][0]):VNE_CATS[k][1] for k in ["thoi-su","the-gioi","kinh-doanh","the-thao","giai-tri"]}
302
  futs[ex.submit(scrape_bbc_vietnamese)]="BBC"
303
  for f in as_completed(futs):
304
  try:
305
- for a in f.result(): a["group"]=futs[f]; articles.append(a)
306
- except: pass
307
  try:
308
- for a in scrape_bdp(f"{BASE_BDP}/tin-moi")[:6]: a["group"]="Bóng Đá"; articles.append(a)
309
- except: pass
310
  return articles
311
  return JSONResponse(_cached("homepage",_f))
312
 
@@ -314,14 +283,8 @@ def api_homepage():
314
  def api_category(cat_id:str):
315
  def _f():
316
  if cat_id=="bbc":return scrape_bbc_vietnamese()
317
- if cat_id in VNE_CATS:
318
- arts=scrape_vne(VNE_CATS[cat_id][0])
319
- for a in arts:a["group"]=VNE_CATS[cat_id][1]
320
- return arts
321
- if cat_id in BDP_CATS:
322
- arts=scrape_bdp(BDP_CATS[cat_id][0])
323
- for a in arts:a["group"]=BDP_CATS[cat_id][1]
324
- return arts
325
  return[]
326
  return JSONResponse(_cached(f"cat_{cat_id}",_f))
327
 
@@ -346,28 +309,25 @@ def api_bdp_videos():
346
 
347
  @app.get("/api/video_url")
348
  def api_video_url(url:str=Query(...)):
349
- """Extract video URL. Returns {src, poster, all_parts?}."""
350
- if "24h.com.vn" in url: v=extract_video_url(url)
351
- elif "bongdaplus.vn" in url: v=extract_bdp_video(url)
352
- else: v=None
353
- return JSONResponse(v if v else {"error":"not found"})
354
 
355
  @app.get("/api/article")
356
  def api_article(url:str=Query(...)):
357
- if "vnexpress.net" in url: data=scrape_vne_article(url)
358
- elif "bbc.com" in url: data=scrape_bbc_article(url)
359
- else: data=None
360
- return JSONResponse(data if data else {"error":"not supported"})
361
 
362
  @app.get("/share/{path:path}")
363
  async def share_page(path:str,img:str=Query(default=""),title:str=Query(default="VNEWS")):
364
  og_image=img or "https://s1.vnecdn.net/vnexpress/restruct/i/v9505/logo_default.jpg"
365
- html=f'<!DOCTYPE html><html><head><meta charset="utf-8"><title>{title}</title><meta property="og:title" content="{title}"><meta property="og:image" content="{og_image}"><meta property="og:type" content="article"><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="{og_image}"><meta http-equiv="refresh" content="0;url={SPACE_URL}"></head><body>Redirecting...</body></html>'
366
- return HTMLResponse(content=html)
367
 
368
  @app.get("/")
369
  async def index():
370
- with open("/app/static/index.html","r",encoding="utf-8") as f:
371
- return HTMLResponse(content=f.read())
372
 
373
  app.mount("/static",StaticFiles(directory="/app/static"),name="static")
 
1
+ """VNEWS - FastAPI backend."""
2
  import hashlib, re, time
3
  from concurrent.futures import ThreadPoolExecutor, as_completed
4
  from fastapi import FastAPI, Query
 
8
  from bs4 import BeautifulSoup
9
 
10
  app = FastAPI()
11
+ HEADERS = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36","Accept-Language":"vi-VN,vi;q=0.9,en;q=0.8"}
 
 
 
 
12
  BASE_BDP = "https://bongdaplus.vn"
13
  BASE_24H = "https://www.24h.com.vn"
14
  SPACE_URL = "https://bep40-vnews.hf.space"
 
15
  _cache = {}
16
  _cache_ttl = 300
17
 
18
  def _cached(key, fn):
19
+ now=time.time()
20
+ if key in _cache and now-_cache[key]["t"]<_cache_ttl: return _cache[key]["d"]
21
+ try: data=fn()
22
+ except: data=_cache.get(key,{}).get("d",[])
23
+ _cache[key]={"d":data,"t":now}; return data
 
 
24
 
25
+ def _get(url,headers=None):
26
+ h=headers or HEADERS; r=requests.get(url,headers=h,timeout=15); r.encoding="utf-8"
27
+ return BeautifulSoup(r.text,"lxml")
 
28
 
 
29
  def scrape_vne(cat_url):
30
  try:
31
+ soup=_get(cat_url);arts=[]
32
  for it in soup.select("article.item-news")[:15]:
33
+ a=it.select_one("h2.title-news a") or it.select_one("h3.title-news a")
34
+ if not a:continue
35
+ t=a.get("title","") or a.get_text(strip=True);lk=a.get("href","")
36
+ if not t or not lk:continue
37
+ im=it.find("img");img=(im.get("data-src") or im.get("src","")) if im else ""
 
38
  if img and 'blank' in img:
39
+ src=it.find("source")
40
+ if src:img=src.get("srcset","").split(",")[0].strip().split(" ")[0]
41
+ desc=it.select_one("p.description")
42
  arts.append({"title":t,"link":lk,"img":img,"summary":(desc.get_text(strip=True)[:150] if desc else ""),"source":"vne"})
43
  return arts
44
+ except:return[]
45
 
46
  def scrape_vne_article(url):
47
  try:
48
+ soup=_get(url);h1=soup.select_one("h1.title-detail");desc=soup.select_one("p.description")
49
+ cd=soup.select_one("article.fck_detail");body=[]
 
 
 
 
 
50
  if cd:
51
  for ch in cd.children:
52
+ if not hasattr(ch,'name') or not ch.name:continue
53
+ if ch.name=="p":
54
+ t=ch.get_text(strip=True)
55
+ if t:body.append({"type":"p","text":t})
56
+ elif ch.name=="figure":
57
+ im=ch.find("img")
58
  if im:
59
+ s=im.get("data-src") or im.get("src","")
60
+ cap=ch.find("figcaption")
61
+ if s:body.append({"type":"img","src":s,"alt":cap.get_text(strip=True) if cap else ""})
62
+ elif ch.name in("h2","h3"):body.append({"type":"heading","text":ch.get_text(strip=True)})
63
+ return{"title":h1.get_text(strip=True) if h1 else "","summary":desc.get_text(strip=True) if desc else "","body":body,"source":"vne","url":url}
64
+ except:return None
 
65
 
66
  def scrape_bdp(url):
67
  try:
68
+ soup=_get(url);arts=[];seen=set()
69
+ for sel in["div.news.fst","div.sld-itm.news","li.news"]:
70
  for it in soup.select(sel):
71
+ tag=it.find("a",class_="title") or it.find("a",href=True)
72
+ if not tag:continue
73
+ t=tag.get_text(strip=True);lk=tag.get("href","")
74
+ if not t or len(t)<5:continue
75
+ if lk and not lk.startswith("http"):lk=BASE_BDP+lk
76
+ if lk in seen:continue
77
+ im=it.find("img");img=(im.get("data-src") or im.get("src","")) if im else ""
78
+ seen.add(lk);arts.append({"title":t,"link":lk,"img":img,"source":"bdp","summary":""})
79
  return arts[:15]
80
+ except:return[]
81
 
82
  def scrape_bdp_videos():
83
  try:
84
+ soup=_get(f"{BASE_BDP}/video");arts=[];seen=set()
85
+ for a in soup.find_all("a",href=True):
86
+ href=a.get("href","")
87
+ if"/video/" not in href or href in("/video/","/video/ban-thang-dep","/video/highlight"):continue
88
+ if not href.startswith("http"):href=BASE_BDP+href
89
+ if href in seen:continue
90
+ title=re.sub(r'^\d{2}:\d{2}','',a.get_text(strip=True)).strip()
91
+ if not title or len(title)<5:continue
92
+ img_tag=a.find("img") or(a.parent.find("img") if a.parent else None)
93
+ img=(img_tag.get("data-src") or img_tag.get("src","")) if img_tag else ""
94
+ seen.add(href);arts.append({"title":title,"link":href,"img":img,"source":"bdp"})
95
  return arts[:20]
96
+ except:return[]
97
 
98
  def scrape_24h_highlights():
99
+ """Scrape highlights + fetch og:image for items missing img."""
100
  try:
101
+ soup=_get(f"{BASE_24H}/video-highlight-c953.html");arts=[];seen=set()
102
+ for a in soup.find_all("a",href=True):
103
+ href=a.get("href","");title=a.get("title","") or a.get_text(strip=True)
104
+ if not title or len(title)<15 or "javascript:" in href:continue
105
+ if not href.startswith("http"):href=BASE_24H+href
106
+ if href in seen or "video" not in href.lower() or href.endswith("-c953.html"):continue
 
107
  seen.add(href)
108
+ img=a.find("img")
109
+ img_src=""
110
+ if img:img_src=img.get("data-original") or img.get("data-src") or img.get("src","")
111
+ if img_src and "base64" in img_src:img_src=""
 
112
  arts.append({"title":title,"link":href,"img":img_src,"source":"24h"})
113
+ # Fetch og:image for items without img (parallel, max 10)
114
+ def _fetch_og(art):
115
+ if art["img"]:return art
116
+ try:
117
+ r=requests.get(art["link"],headers=HEADERS,timeout=8);r.encoding="utf-8"
118
+ s=BeautifulSoup(r.text,"lxml")
119
+ og=s.find("meta",property="og:image")
120
+ if og:art["img"]=og.get("content","")
121
+ except:pass
122
+ return art
123
+ need_img=[a for a in arts if not a["img"]][:10]
124
+ if need_img:
125
+ with ThreadPoolExecutor(5) as ex:
126
+ list(ex.map(_fetch_og,need_img))
127
  return arts[:30]
128
+ except:return[]
129
 
130
  def scrape_24h_shorts():
131
+ """Scrape shorts - try <article> first, fallback to <a> with images."""
132
  try:
133
+ r=requests.get(f"{BASE_24H}/video/video-tin-tuc-cvd769.html",headers=HEADERS,timeout=15)
134
+ r.encoding="utf-8";soup=BeautifulSoup(r.text,"lxml")
135
+ arts=[];seen=set()
136
+ # Method 1: <article> tags
137
  for art in soup.find_all("article"):
138
+ a=art.find("a",href=True)
139
+ if not a:continue
140
+ href=a.get("href","")
141
+ if not href.startswith("http"):href=BASE_24H+href
142
+ if href in seen:continue
143
+ img_tag=art.find("img")
144
+ title=(img_tag.get("alt","") if img_tag else "") or a.get("title","") or a.get_text(strip=True)
145
+ if not title or len(title)<10:continue
146
+ img_src=""
147
+ if img_tag:img_src=img_tag.get("data-original") or img_tag.get("data-src") or img_tag.get("src","")
148
+ if img_src and "base64" in img_src:img_src=""
149
+ seen.add(href);arts.append({"title":title,"link":href,"img":img_src,"source":"24h-shorts"})
150
+ # Method 2: fallback - all <a> with <img> inside
151
+ if len(arts)<3:
152
+ for a in soup.find_all("a",href=True):
153
+ href=a.get("href","")
154
+ if not href.endswith(".html") or "javascript:" in href or "-cvd" in href:continue
155
+ if not href.startswith("http"):href=BASE_24H+href
156
+ if href in seen:continue
157
+ img=a.find("img")
158
+ if not img:
159
+ p=a.parent
160
+ if p:img=p.find("img")
161
+ if not img:continue
162
+ img_src=img.get("data-original") or img.get("data-src") or img.get("src","")
163
+ if not img_src or "base64" in img_src:continue
164
+ title=img.get("alt","") or a.get("title","") or a.get_text(strip=True)
165
+ if not title or len(title)<8:continue
166
+ seen.add(href);arts.append({"title":title,"link":href,"img":img_src,"source":"24h-shorts"})
167
  return arts[:20]
168
+ except:return[]
169
 
170
  def scrape_bbc_vietnamese():
 
171
  try:
172
+ bbc_h={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36","Accept-Language":"en-GB,en;q=0.9"}
173
+ r=requests.get("https://www.bbc.com/vietnamese",headers=bbc_h,timeout=15);r.encoding="utf-8"
174
+ soup=BeautifulSoup(r.text,"lxml");arts=[];seen=set()
 
 
 
 
 
 
 
175
  for a in soup.select("a[href*='/vietnamese/']"):
176
+ href=a.get("href","")
177
+ if not href or href=="/vietnamese" or href.count("/")<3:continue
178
+ if not href.startswith("http"):href="https://www.bbc.com"+href
179
+ if href in seen:continue
180
+ title=a.get_text(strip=True)
181
+ if not title or len(title)<15:continue
182
+ if any(x in title.lower() for x in["đăng nhập","trang chủ","bbc news"]):continue
183
+ img="";container=a.parent
 
 
 
 
184
  for _ in range(3):
185
  if container:
186
+ im=container.find("img")
187
+ if im:img=im.get("src","") or im.get("data-src","");break
188
+ container=container.parent
189
+ seen.add(href);arts.append({"title":title,"link":href,"img":img,"source":"bbc","summary":""})
190
+ if len(arts)>=15:break
 
 
 
191
  return arts
192
+ except:return[]
193
 
194
  def scrape_bbc_article(url):
195
  try:
196
+ bbc_h={"User-Agent":"Mozilla/5.0","Accept-Language":"en-GB,en;q=0.9"}
197
+ r=requests.get(url,headers=bbc_h,timeout=15);r.encoding="utf-8"
198
+ soup=BeautifulSoup(r.text,"lxml");h1=soup.find("h1")
199
+ body=[]
 
 
 
200
  for p in soup.select("[data-component='text-block'] p, article p, main p"):
201
+ t=p.get_text(strip=True)
202
+ if t and len(t)>20:body.append({"type":"p","text":t})
 
 
203
  for img in soup.select("main img, article img"):
204
+ src=img.get("src","")
205
+ if src and("ichef" in src or "bbci" in src):body.append({"type":"img","src":src,"alt":img.get("alt","")})
206
+ return{"title":h1.get_text(strip=True) if h1 else "","summary":"","body":body,"source":"bbc","url":url}
207
+ except:return None
 
208
 
209
  def extract_video_url(article_url):
 
210
  try:
211
+ r=requests.get(article_url,headers={**HEADERS,"Referer":"https://www.24h.com.vn/"},timeout=10);r.encoding="utf-8"
212
+ m3u8s=re.findall(r'(https?://cdn\.24h\.com\.vn/[^\s"\'\\<>]+\.m3u8)',r.text)
213
+ esc=[u.replace('\\//','/').replace('\\/','/') for u in re.findall(r'(https?:\\/\\/cdn\.24h\.com\.vn\\/[^\s"\'<>]+\.m3u8)',r.text)]
214
+ all_urls=list(dict.fromkeys(m3u8s+esc))
215
+ full=[u for u in all_urls if'_720p' not in u];p720=[u for u in all_urls if'_720p' in u]
216
+ primary=full or p720
217
+ if not primary:return None
218
+ soup=BeautifulSoup(r.text,"lxml");og=soup.find("meta",property="og:image")
219
+ poster=og.get("content","") if og else ""
220
+ # Multi-part probe
221
+ base_url=primary[0];parts=[base_url]
222
+ is_720p='_720p.m3u8' in base_url
223
+ name=base_url.replace('_720p.m3u8','').replace('.m3u8','')
224
+ m=re.search(r'(\D)(\d{1,2})$',name)
 
 
 
 
 
225
  if m:
226
+ cur=int(m.group(2));w=len(m.group(2));bn=name[:m.start(2)]
227
+ for i in range(cur+1,cur+5):
228
+ nn=f"{bn}{i:02d}" if w==2 else f"{bn}{i}"
229
+ for sfx in[('_720p.m3u8' if is_720p else '.m3u8'),'_720p.m3u8','.m3u8']:
230
+ pu=nn+sfx
231
+ if pu in parts:continue
232
  try:
233
+ if requests.head(pu,headers=HEADERS,timeout=3,allow_redirects=True).status_code==200:parts.append(pu);break
234
+ except:continue
235
+ else:break
 
 
236
  if len(parts)==1:
237
+ m2=re.match(r'^(.+-)(\d{7,})(-.+?)(\d{1,2})((?:_720p)?\.m3u8)$',base_url)
238
  if m2:
239
+ pfx,cid,mid,pn,sfx=m2.group(1),int(m2.group(2)),m2.group(3),int(m2.group(4)),m2.group(5)
240
  for i in range(1,5):
241
+ pu=f"{pfx}{cid+i}{mid}{pn+i}{sfx}"
242
  try:
243
+ if requests.head(pu,headers=HEADERS,timeout=3,allow_redirects=True).status_code==200:parts.append(pu)
244
+ else:break
245
+ except:break
246
+ if len(parts)>1:return{"src":parts[0],"poster":poster,"all_parts":parts}
247
+ return{"src":primary[0],"poster":poster}
248
+ except:return None
 
 
 
249
 
250
  def extract_bdp_video(url):
251
  try:
252
+ m=re.search(r'-(\d{6,})\.html',url)
253
+ if not m:return None
254
+ r=requests.get(f"{BASE_BDP}/video-embed/{m.group(1)}.html",headers=HEADERS,timeout=10);r.encoding="utf-8"
255
+ soup=BeautifulSoup(r.text,"lxml");video=soup.select_one("video#videoPlayer")
256
+ if not video:return None
257
+ source=video.find("source")
258
+ return{"src":source.get("src","") if source else "","poster":video.get("poster","")}
259
+ except:return None
 
 
 
260
 
261
  # ═══ API ═══
262
+ VNE_CATS={"thoi-su":("https://vnexpress.net/thoi-su","Thời Sự"),"the-gioi":("https://vnexpress.net/the-gioi","Thế Giới"),"kinh-doanh":("https://vnexpress.net/kinh-doanh","Kinh Doanh"),"cong-nghe":("https://vnexpress.net/so-hoa","Công Nghệ"),"the-thao":("https://vnexpress.net/the-thao","Thể Thao"),"giai-tri":("https://vnexpress.net/giai-tri","Giải Trí"),"suc-khoe":("https://vnexpress.net/suc-khoe","Sức Khỏe")}
263
+ BDP_CATS={"ngoai-hang-anh":("https://bongdaplus.vn/ngoai-hang-anh","Ngoại Hạng Anh"),"la-liga":("https://bongdaplus.vn/la-liga","La Liga"),"champions-league":("https://bongdaplus.vn/champions-league-cup-c1","Champions League"),"bong-da-vn":("https://bongdaplus.vn/bong-da-viet-nam","Bóng Đá VN")}
 
 
 
 
 
 
 
 
 
 
 
 
 
264
 
265
  @app.get("/api/homepage")
266
  def api_homepage():
267
  def _f():
268
  articles=[]
269
  with ThreadPoolExecutor(6) as ex:
270
+ futs={ex.submit(scrape_vne,VNE_CATS[k][0]):VNE_CATS[k][1] for k in["thoi-su","the-gioi","kinh-doanh","the-thao","giai-tri"]}
271
  futs[ex.submit(scrape_bbc_vietnamese)]="BBC"
272
  for f in as_completed(futs):
273
  try:
274
+ for a in f.result():a["group"]=futs[f];articles.append(a)
275
+ except:pass
276
  try:
277
+ for a in scrape_bdp(f"{BASE_BDP}/tin-moi")[:6]:a["group"]="Bóng Đá";articles.append(a)
278
+ except:pass
279
  return articles
280
  return JSONResponse(_cached("homepage",_f))
281
 
 
283
  def api_category(cat_id:str):
284
  def _f():
285
  if cat_id=="bbc":return scrape_bbc_vietnamese()
286
+ if cat_id in VNE_CATS:arts=scrape_vne(VNE_CATS[cat_id][0]);[a.update({"group":VNE_CATS[cat_id][1]}) for a in arts];return arts
287
+ if cat_id in BDP_CATS:arts=scrape_bdp(BDP_CATS[cat_id][0]);[a.update({"group":BDP_CATS[cat_id][1]}) for a in arts];return arts
 
 
 
 
 
 
288
  return[]
289
  return JSONResponse(_cached(f"cat_{cat_id}",_f))
290
 
 
309
 
310
  @app.get("/api/video_url")
311
  def api_video_url(url:str=Query(...)):
312
+ if "24h.com.vn" in url:v=extract_video_url(url)
313
+ elif "bongdaplus.vn" in url:v=extract_bdp_video(url)
314
+ else:v=None
315
+ return JSONResponse(v if v else{"error":"not found"})
 
316
 
317
  @app.get("/api/article")
318
  def api_article(url:str=Query(...)):
319
+ if "vnexpress.net" in url:data=scrape_vne_article(url)
320
+ elif "bbc.com" in url:data=scrape_bbc_article(url)
321
+ else:data=None
322
+ return JSONResponse(data if data else{"error":"not supported"})
323
 
324
  @app.get("/share/{path:path}")
325
  async def share_page(path:str,img:str=Query(default=""),title:str=Query(default="VNEWS")):
326
  og_image=img or "https://s1.vnecdn.net/vnexpress/restruct/i/v9505/logo_default.jpg"
327
+ return HTMLResponse(f'<!DOCTYPE html><html><head><meta charset="utf-8"><title>{title}</title><meta property="og:title" content="{title}"><meta property="og:image" content="{og_image}"><meta property="og:type" content="article"><meta name="twitter:card" content="summary_large_image"><meta name="twitter:image" content="{og_image}"><meta http-equiv="refresh" content="0;url={SPACE_URL}/#/{path}"></head><body>Redirecting...</body></html>')
 
328
 
329
  @app.get("/")
330
  async def index():
331
+ with open("/app/static/index.html","r",encoding="utf-8") as f:return HTMLResponse(content=f.read())
 
332
 
333
  app.mount("/static",StaticFiles(directory="/app/static"),name="static")