Fix WC2026 scraper: use bongda.com.vn internal AJAX API (same as livescore) with tournament_id=24254

wc2026_scraper.py

@@ -1,17 +1,23 @@
 """
-World Cup 2026 Data Module - WORKING sources
-bongda.com.vn is a JavaScript SPA (empty with requests), so we use:
-- Tin tức: VnExpress, Dân Trí, BongDaPlus RSS/search
-- Lịch thi đấu + BXH: livescore.com scraping (server-rendered)
-- Đường tới WC: VnExpress + news search
-All cached with short TTL for LIVE updates.
+World Cup 2026 Data Module
+Uses bongda.com.vn INTERNAL AJAX API (same method as livescore)
+Tournament ID: 24254
+Also fetches news from VnExpress, DanTri, BongDaPlus, 24h, TT&VH
 """
 import requests, re, time, json, os, threading
 from bs4 import BeautifulSoup
 from urllib.parse import quote
+from datetime import datetime
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
-UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36'}
+BONGDA_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+    "Accept-Language": "vi-VN,vi;q=0.9",
+    "Referer": "https://bongda.com.vn/giai-dau/24254/summary/world-cup",
+    "X-Requested-With": "XMLHttpRequest"
+}
+UA = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
+WC_TOURNAMENT_ID = 24254
 CACHE = {}
 CACHE_LOCK = threading.Lock()
 
@@ -28,6 +34,18 @@ def _set(key, data):
     with CACHE_LOCK:
         CACHE[key] = {'t': time.time(), 'd': data}
 
+def _fetch_bongda_api(endpoint):
+    """Call bongda.com.vn internal AJAX API - returns HTML string."""
+    try:
+        r = requests.get(f"https://bongda.com.vn{endpoint}", headers=BONGDA_HEADERS, timeout=12)
+        if r.status_code == 200:
+            data = r.json()
+            if data.get("status") == "success":
+                return data.get("html", "")
+    except:
+        pass
+    return ""
+
 def _fetch(url, timeout=12):
     try:
         r = requests.get(url, headers=UA, timeout=timeout, allow_redirects=True)
@@ -38,6 +56,76 @@ def _fetch(url, timeout=12):
         pass
     return ''
 
+# ==================== FIXTURES (from bongda.com.vn API) ====================
+def scrape_fixtures():
+    """Get WC2026 fixtures using bongda.com.vn AJAX API."""
+    c = _cached('wc_fixtures', 120)
+    if c is not None: return c
+    
+    # Try multiple API endpoints that bongda.com.vn uses
+    html = _fetch_bongda_api(f"/api/fixtures/get-by-tournament?tournament_id={WC_TOURNAMENT_ID}")
+    if not html:
+        html = _fetch_bongda_api(f"/api/fixtures/get-by-tournament?tournament_id={WC_TOURNAMENT_ID}&status=all")
+    if not html:
+        # Try date-based for today
+        today = datetime.now().strftime("%Y-%m-%d")
+        html = _fetch_bongda_api(f"/api/fixtures/get-by-date?date={today}&tournament_id={WC_TOURNAMENT_ID}")
+    
+    result = {'html': html} if html else {'html': '', 'matches': []}
+    _set('wc_fixtures', result)
+    return result
+
+# ==================== STANDINGS (BXH) ====================
+def scrape_standings():
+    """Get WC2026 standings/league table using bongda.com.vn AJAX API."""
+    c = _cached('wc_standings', 180)
+    if c is not None: return c
+    
+    html = _fetch_bongda_api(f"/api/league-table/home?tournament_id={WC_TOURNAMENT_ID}&is_detail=True")
+    if not html:
+        html = _fetch_bongda_api(f"/api/league-table/home?tournament_id={WC_TOURNAMENT_ID}")
+    
+    result = {'html': html} if html else {'html': ''}
+    _set('wc_standings', result)
+    return result
+
+# ==================== STATS ====================
+def scrape_stats():
+    """Get WC2026 statistics using bongda.com.vn AJAX API."""
+    c = _cached('wc_stats', 300)
+    if c is not None: return c
+    
+    # Player performance / top scorers
+    html = _fetch_bongda_api(f"/api/event-standing/top-scorers?tournament_id={WC_TOURNAMENT_ID}")
+    if not html:
+        html = _fetch_bongda_api(f"/api/event-standing/player-performance?tournament_id={WC_TOURNAMENT_ID}")
+    
+    result = {'html': html} if html else {'html': ''}
+    _set('wc_stats', result)
+    return result
+
+# ==================== SUMMARY (Live) ====================
+def scrape_summary():
+    """Get WC2026 live/recent matches."""
+    c = _cached('wc_summary', 60)
+    if c is not None: return c
+    
+    # Live matches for this tournament
+    html_live = _fetch_bongda_api(f"/api/fixtures/live?tournament_id={WC_TOURNAMENT_ID}")
+    # Today's matches
+    today = datetime.now().strftime("%Y-%m-%d")
+    html_today = _fetch_bongda_api(f"/api/fixtures/get-by-date?date={today}&tournament_id={WC_TOURNAMENT_ID}")
+    # Incoming
+    html_incoming = _fetch_bongda_api(f"/api/fixtures/incoming?tournament_id={WC_TOURNAMENT_ID}")
+    
+    result = {
+        'html_live': html_live,
+        'html_today': html_today,
+        'html_incoming': html_incoming
+    }
+    _set('wc_summary', result)
+    return result
+
 # ==================== NEWS ====================
 def scrape_wc_news():
     """Get World Cup 2026 news from multiple VN sources."""
@@ -46,30 +134,30 @@ def scrape_wc_news():
     
     news = []
     
-    # 1. VnExpress World Cup RSS
+    # 1. TT&VH World Cup RSS
     try:
-        html = _fetch('https://vnexpress.net/rss/the-thao.rss')
+        html = _fetch('https://thethaovanhoa.vn/rss/world-cup-2026.rss')
         if html:
             soup = BeautifulSoup(html, 'xml')
-            for item in soup.find_all('item')[:30]:
+            for item in soup.find_all('item')[:12]:
                 title = _clean(item.find('title').get_text() if item.find('title') else '')
                 link = _clean(item.find('link').get_text() if item.find('link') else '')
-                if title and link and any(kw in title.lower() for kw in ['world cup', 'wc 2026', 'fifa', 'vòng chung kết']):
-                    desc = item.find('description')
-                    img = ''
-                    if desc:
-                        ds = BeautifulSoup(desc.get_text(), 'lxml')
-                        im = ds.find('img')
-                        if im: img = im.get('src', '')
-                    news.append({'title': title, 'link': link, 'img': img, 'source': 'VnExpress'})
+                desc = item.find('description')
+                img = ''
+                if desc:
+                    ds = BeautifulSoup(desc.get_text(), 'lxml')
+                    im = ds.find('img')
+                    if im: img = im.get('src', '')
+                if title and link:
+                    news.append({'title': title, 'link': link, 'img': img, 'source': 'TT&VH'})
     except: pass
     
-    # 2. Search VnExpress for WC2026
+    # 2. VnExpress search World Cup
     try:
         html = _fetch(f'https://timkiem.vnexpress.net/?q={quote("World Cup 2026")}')
         if html:
             soup = BeautifulSoup(html, 'lxml')
-            for art in soup.select('article.item-news')[:12]:
+            for art in soup.select('article.item-news')[:10]:
                 a = art.select_one('h2 a, h3 a')
                 if not a: continue
                 title = _clean(a.get('title', '') or a.get_text())
@@ -80,25 +168,12 @@ def scrape_wc_news():
                     news.append({'title': title, 'link': href, 'img': img, 'source': 'VnExpress'})
     except: pass
     
-    # 3. Dân Trí search
-    try:
-        html = _fetch(f'https://dantri.com.vn/tim-kiem/World+Cup+2026.htm')
-        if html:
-            soup = BeautifulSoup(html, 'lxml')
-            for a in soup.select('h3 a[href], .article-title a[href]')[:10]:
-                title = _clean(a.get_text())
-                href = a.get('href', '')
-                if not href.startswith('http'): href = 'https://dantri.com.vn' + href
-                if title and len(title) > 15 and href not in [n['link'] for n in news]:
-                    news.append({'title': title, 'link': href, 'img': '', 'source': 'Dân Trí'})
-    except: pass
-    
-    # 4. BongDaPlus
+    # 3. BongDaPlus World Cup
     try:
         html = _fetch('https://bongdaplus.vn/world-cup')
         if html:
             soup = BeautifulSoup(html, 'lxml')
-            for a in soup.select('h2 a[href], h3 a[href], .title a[href]')[:10]:
+            for a in soup.select('h2 a[href], h3 a[href], .title a[href]')[:8]:
                 title = _clean(a.get_text())
                 href = a.get('href', '')
                 if not href.startswith('http'): href = 'https://bongdaplus.vn' + href
@@ -106,211 +181,22 @@ def scrape_wc_news():
                     news.append({'title': title, 'link': href, 'img': '', 'source': 'BongDaPlus'})
     except: pass
     
-    # 5. 24h.com.vn World Cup
+    # 4. 24h World Cup
     try:
         html = _fetch('https://www.24h.com.vn/world-cup-2026-c860.html')
         if html:
             soup = BeautifulSoup(html, 'lxml')
-            for a in soup.select('a.news-title, h2 a, h3 a')[:8]:
-                title = _clean(a.get_text())
+            for a in soup.select('a[title]')[:8]:
+                title = _clean(a.get('title', ''))
                 href = a.get('href', '')
                 if not href.startswith('http'): href = 'https://www.24h.com.vn' + href
-                if title and len(title) > 15 and 'world cup' in title.lower() and href not in [n['link'] for n in news]:
+                if title and len(title) > 15 and href not in [n['link'] for n in news]:
                     news.append({'title': title, 'link': href, 'img': '', 'source': '24h'})
     except: pass
     
     _set('wc_news', news[:30])
     return news[:30]
 
-# ==================== FIXTURES (from livescore.com) ====================
-def scrape_fixtures():
-    """Scrape WC2026 fixtures from livescore.com (server-rendered)."""
-    c = _cached('wc_fixtures', 120)
-    if c is not None: return c
-    
-    matches = []
-    # livescore.com has server-rendered pages
-    urls = [
-        'https://www.livescore.com/en/football/world-cup-2026/fixtures/',
-        'https://www.livescore.com/en/football/world-cup-2026/results/',
-    ]
-    
-    for url in urls:
-        html = _fetch(url)
-        if not html: continue
-        soup = BeautifulSoup(html, 'lxml')
-        
-        current_date = ''
-        for el in soup.select('[data-testid], .match, .event, div[class*="match"], div[class*="event"]'):
-            # Try to extract match data
-            teams = el.select('[class*="team"], [class*="Team"]')
-            score_el = el.select_one('[class*="score"], [class*="Score"]')
-            time_el = el.select_one('[class*="time"], [class*="Time"], [class*="status"]')
-            
-            if len(teams) >= 2:
-                home = _clean(teams[0].get_text())
-                away = _clean(teams[1].get_text())
-                score = _clean(score_el.get_text()) if score_el else ''
-                match_time = _clean(time_el.get_text()) if time_el else ''
-                if home and away:
-                    matches.append({
-                        'home': home, 'away': away,
-                        'score': score, 'date': match_time,
-                        'status': 'live' if any(x in match_time.lower() for x in ['live', "'", 'ht']) else '',
-                        'group': ''
-                    })
-    
-    # Fallback: parse from thethao247.vn (server-rendered)
-    if not matches:
-        try:
-            html = _fetch('https://thethao247.vn/world-cup/426-lich-thi-dau-world-cup-2026-d399919.html')
-            if html:
-                soup = BeautifulSoup(html, 'lxml')
-                for tr in soup.select('table tr'):
-                    tds = tr.select('td')
-                    if len(tds) >= 3:
-                        matches.append({
-                            'home': _clean(tds[0].get_text()),
-                            'score': _clean(tds[1].get_text()),
-                            'away': _clean(tds[2].get_text()),
-                            'date': _clean(tds[3].get_text()) if len(tds) > 3 else '',
-                            'status': '', 'group': ''
-                        })
-        except: pass
-    
-    # Fallback 2: 24h.com.vn fixtures
-    if not matches:
-        try:
-            html = _fetch('https://www.24h.com.vn/bong-da/lich-thi-dau-bong-da-world-cup-2026-moi-nhat-c48a1747402.html')
-            if html:
-                soup = BeautifulSoup(html, 'lxml')
-                for row in soup.select('.schedule-item, .match-item, tr'):
-                    teams = row.select('.team, td')
-                    if len(teams) >= 2:
-                        home = _clean(teams[0].get_text())
-                        away = _clean(teams[-1].get_text())
-                        if home and away and len(home) < 40:
-                            matches.append({'home': home, 'away': away, 'score': '', 'date': '', 'status': '', 'group': ''})
-        except: pass
-    
-    _set('wc_fixtures', matches[:60])
-    return matches[:60]
-
-# ==================== STANDINGS ====================
-def scrape_standings():
-    """Scrape WC2026 standings from livescore.com."""
-    c = _cached('wc_standings', 180)
-    if c is not None: return c
-    
-    groups = {}
-    
-    # Try livescore.com group standings
-    group_letters = 'ABCDEFGHIJKL'
-    for letter in group_letters:
-        url = f'https://www.livescore.com/en/football/world-cup-2026/group-{letter.lower()}/standings/'
-        html = _fetch(url, timeout=8)
-        if not html: continue
-        soup = BeautifulSoup(html, 'lxml')
-        
-        teams = []
-        for row in soup.select('tr, [class*="row"], [class*="Row"]'):
-            cells = row.select('td, [class*="cell"], [class*="Cell"]')
-            if len(cells) >= 3:
-                # Find team name (longest text that's not a number)
-                texts = [_clean(c.get_text()) for c in cells]
-                team_name = ''
-                pts = ''
-                played = ''
-                for t in texts:
-                    if len(t) > 2 and not t.isdigit() and len(t) > len(team_name):
-                        team_name = t
-                if texts:
-                    pts = texts[-1] if texts[-1].isdigit() else ''
-                    for t in texts[1:4]:
-                        if t.isdigit() and not played:
-                            played = t
-                if team_name:
-                    teams.append({'team': team_name, 'played': played, 'pts': pts})
-        
-        if teams:
-            groups[f'Bảng {letter}'] = teams
-    
-    # Fallback: get from 24h.com.vn BXH page
-    if not groups:
-        try:
-            html = _fetch('https://www.24h.com.vn/bong-da/bang-xep-hang-bong-da-world-cup-2026-moi-nhat-c48a1747408.html')
-            if html:
-                soup = BeautifulSoup(html, 'lxml')
-                current_group = ''
-                for el in soup.select('h3, h4, table, tr'):
-                    if el.name in ('h3', 'h4'):
-                        text = _clean(el.get_text())
-                        if 'bảng' in text.lower() or 'group' in text.lower():
-                            current_group = text
-                            if current_group not in groups:
-                                groups[current_group] = []
-                    elif el.name == 'tr' and current_group:
-                        tds = el.select('td')
-                        if len(tds) >= 3:
-                            texts = [_clean(td.get_text()) for td in tds]
-                            team = max([t for t in texts if t and not t.isdigit()], key=len, default='')
-                            pts = texts[-1] if texts and texts[-1].isdigit() else ''
-                            if team:
-                                groups[current_group].append({'team': team, 'played': '', 'pts': pts})
-        except: pass
-    
-    _set('wc_standings', groups)
-    return groups
-
-# ==================== STATS ====================
-def scrape_stats():
-    """Get WC2026 stats - top scorers, cards etc."""
-    c = _cached('wc_stats', 600)
-    if c is not None: return c
-    
-    stats = {'top_scorers': [], 'top_assists': []}
-    
-    # World Cup hasn't started yet (June 2026), so stats may be empty
-    # Try to get from livescore
-    html = _fetch('https://www.livescore.com/en/football/world-cup-2026/top-scorers/')
-    if html:
-        soup = BeautifulSoup(html, 'lxml')
-        for row in soup.select('tr, [class*="row"]')[:15]:
-            cells = row.select('td, [class*="cell"]')
-            if len(cells) >= 2:
-                texts = [_clean(c.get_text()) for c in cells]
-                player = max([t for t in texts if t and not t.isdigit() and len(t) > 2], key=len, default='')
-                goals = ''
-                for t in reversed(texts):
-                    if t.isdigit():
-                        goals = t; break
-                if player and goals:
-                    stats['top_scorers'].append({'player': player, 'team': '', 'goals': goals})
-    
-    _set('wc_stats', stats)
-    return stats
-
-# ==================== SUMMARY ====================
-def scrape_summary():
-    """Live/upcoming WC2026 matches."""
-    c = _cached('wc_summary', 60)
-    if c is not None: return c
-    
-    data = {'live_matches': [], 'upcoming': [], 'recent_results': []}
-    fixtures = scrape_fixtures()
-    
-    for m in fixtures:
-        status = (m.get('status', '') or '').lower()
-        if 'live' in status or "'" in m.get('date', ''):
-            data['live_matches'].append(m)
-        elif m.get('score') and m['score'] != 'vs' and '-' in m['score']:
-            data['recent_results'].append(m)
-        else:
-            data['upcoming'].append(m)
-    
-    _set('wc_summary', data)
-    return data
-
 # ==================== ROAD TO WORLD CUP ====================
 def scrape_road_to_wc():
     """Get 'Đường tới World Cup 2026' articles."""
@@ -318,14 +204,14 @@ def scrape_road_to_wc():
     if c is not None: return c
     
     articles = []
-    queries = ['đường tới World Cup 2026', 'vòng loại World Cup 2026', 'tuyển Việt Nam World Cup']
+    queries = ['đường tới World Cup 2026', 'vòng loại World Cup 2026', 'tuyển Việt Nam World Cup 2026']
     
     for q in queries:
         try:
             html = _fetch(f'https://timkiem.vnexpress.net/?q={quote(q)}')
             if not html: continue
             soup = BeautifulSoup(html, 'lxml')
-            for art in soup.select('article.item-news')[:6]:
+            for art in soup.select('article.item-news')[:5]:
                 a = art.select_one('h2 a, h3 a')
                 if not a: continue
                 title = _clean(a.get('title', '') or a.get_text())
@@ -336,9 +222,9 @@ def scrape_road_to_wc():
                     articles.append({'title': title, 'link': href, 'img': img, 'source': 'VnExpress', 'type': 'road'})
         except: continue
     
-    # Dân Trí
+    # Dân Trí search
     try:
-        html = _fetch('https://dantri.com.vn/tim-kiem/v%C3%B2ng+lo%E1%BA%A1i+World+Cup+2026.htm')
+        html = _fetch(f'https://dantri.com.vn/tim-kiem/World+Cup+2026.htm')
         if html:
             soup = BeautifulSoup(html, 'lxml')
             for a in soup.select('h3 a[href]')[:6]:
@@ -359,7 +245,7 @@ def get_wc2026_all():
     if c is not None: return c
     
     data = {}
-    with ThreadPoolExecutor(5) as ex:
+    with ThreadPoolExecutor(6) as ex:
         futs = {
             ex.submit(scrape_summary): 'summary',
             ex.submit(scrape_fixtures): 'fixtures',
@@ -368,12 +254,12 @@ def get_wc2026_all():
             ex.submit(scrape_wc_news): 'news',
             ex.submit(scrape_road_to_wc): 'road',
         }
-        for f in as_completed(futs, timeout=20):
+        for f in as_completed(futs, timeout=18):
             key = futs[f]
             try:
                 data[key] = f.result()
             except:
-                data[key] = [] if key in ('fixtures', 'news', 'road') else {}
+                data[key] = {} if key in ('summary', 'fixtures', 'standings', 'stats') else []
     
     _set('wc_all', data)
     return data