Upload match_detail.py

match_detail.py

@@ -1,21 +1,23 @@
 """
 Match Detail Scraper for bongda.com.vn
-Fetches: preview/commentaries, lineups, head-to-head, stats for any match
+Fetches: pre-match info, lineups, head-to-head, form, stats for any match
+
+API endpoints (discovered from page source):
+- /api/event-standing/pre-match?event_id={id}     → preview/match info
+- /api/fixtures/form?team_id={id}                  → recent form for a team
+- /api/fixtures/h2h-match?event_id={id}            → H2H match list
+- /api/fixtures/h2h-stats?event_id={id}            → H2H stats comparison
+- /api/event-standing/player-performance?event_id  → player performance stats
 """
 import requests, re, json, time, threading
 from bs4 import BeautifulSoup
 
 BONGDA_HEADERS = {
-    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
-    "Accept": "application/json, text/javascript, */*; q=0.01",
-    "Accept-Language": "vi-VN,vi;q=0.9",
-    "Referer": "https://bongda.com.vn/lich-thi-dau",
-    "X-Requested-With": "XMLHttpRequest"
-}
-
-UA = {
     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
-    "Accept-Language": "vi-VN,vi;q=0.9"
+    "Accept": "application/json, text/javascript, */*; q=0.01",
+    "Accept-Language": "vi-VN,vi;q=0.9,en-US;q=0.8,en;q=0.7",
+    "Referer": "https://bongda.com.vn/",
+    "X-Requested-With": "XMLHttpRequest",
 }
 
 _match_cache = {}
@@ -36,19 +38,21 @@ def _set_cache(key, data):
 
 
 def _bongda_api(endpoint, params=None):
-    """Call bongda.com.vn API and return HTML snippet."""
+    """Call bongda.com.vn API and return parsed response."""
     try:
         url = f"https://bongda.com.vn{endpoint}"
         if params:
             url += "?" + "&".join(f"{k}={v}" for k, v in params.items())
-        r = requests.get(url, headers=BONGDA_HEADERS, timeout=10)
+        r = requests.get(url, headers=BONGDA_HEADERS, timeout=15)
         if r.status_code == 200:
-            data = r.json()
-            if data.get("status") == "success":
-                return data.get("html", "")
+            try:
+                data = r.json()
+                return data
+            except:
+                pass
     except:
         pass
-    return ""
+    return None
 
 
 def _clean(s):
@@ -70,10 +74,11 @@ def fetch_match_detail_by_url(url):
 def fetch_match_detail(event_id):
     """
     Fetch complete match detail:
-    - commentaries/preview
+    - pre-match info (teams, league, date, stadium, referee)
     - lineups
-    - head-to-head
-    - match stats
+    - head-to-head matches & stats
+    - team form
+    - player performance stats
     """
     cache_key = f"match_detail_{event_id}"
     cached = _cached(cache_key)
@@ -82,124 +87,258 @@ def fetch_match_detail(event_id):
 
     result = {"event_id": event_id, "found": True}
 
-    # 1. Commentaries / Preview
-    commentaries_html = _bongda_api("/api/fixtures/commentaries", {"event_id": event_id})
-    result["commentaries_html"] = commentaries_html
-
-    # Parse preview text
-    if commentaries_html:
-        try:
-            soup = BeautifulSoup(commentaries_html, 'lxml')
-            preview_text = []
-            for el in soup.find_all(['p', 'div', 'span', 'h2', 'h3', 'li']):
-                txt = el.get_text(strip=True)
-                if txt and len(txt) > 15:
-                    preview_text.append(txt)
-            result["preview_text"] = preview_text
-        except:
-            result["preview_text"] = []
+    # 1. Pre-match info (teams, league, date, stadium, referee)
+    pre_match = _bongda_api("/api/event-standing/pre-match", {"event_id": event_id})
+    result["pre_match"] = pre_match
+    if pre_match and pre_match.get("status") == "success":
+        result["pre_match_html"] = pre_match.get("html", "")
+        result["info"] = _parse_pre_match(pre_match.get("html", ""))
     else:
-        result["preview_text"] = []
-
-    # 2. Lineups
-    lineups_html = _bongda_api("/api/fixtures/lineups", {"event_id": event_id})
-    result["lineups_html"] = lineups_html
-    if lineups_html:
-        result["lineups"] = _parse_lineups(lineups_html)
-    else:
-        result["lineups"] = {}
-
-    # 3. Head-to-Head
-    h2h_html = _bongda_api("/api/fixtures/head-to-head", {"event_id": event_id})
-    result["h2h_html"] = h2h_html
-    if h2h_html:
-        result["h2h"] = _parse_h2h(h2h_html)
+        result["pre_match_html"] = ""
+        result["info"] = {}
+
+    # 2. H2H matches
+    h2h_match = _bongda_api("/api/fixtures/h2h-match", {"event_id": event_id})
+    result["h2h_match"] = h2h_match
+    if h2h_match and h2h_match.get("status") == "success":
+        result["h2h_html"] = h2h_match.get("html", "")
+        result["h2h"] = _parse_h2h_matches(h2h_match.get("html", ""))
     else:
+        result["h2h_html"] = ""
         result["h2h"] = []
 
-    # 4. Match Stats
-    stats_html = _bongda_api("/api/event-standing/player-performance", {"event_id": event_id})
-    result["stats_html"] = stats_html
-
-    # 5. Match basic info
-    result["info"] = _parse_match_info(commentaries_html)
+    # 3. H2H stats comparison
+    h2h_stats = _bongda_api("/api/fixtures/h2h-stats", {"event_id": event_id})
+    result["h2h_stats"] = h2h_stats
+    if h2h_stats and h2h_stats.get("status") == "success":
+        result["h2h_stats_html"] = h2h_stats.get("html", "")
+        result["h2h_stats_parsed"] = _parse_h2h_stats(h2h_stats.get("html", ""))
+    else:
+        result["h2h_stats_html"] = ""
+        result["h2h_stats_parsed"] = {}
+
+    # 4. Player performance stats
+    perf = _bongda_api("/api/event-standing/player-performance", {"event_id": event_id})
+    result["performance"] = perf
+    if perf and perf.get("status") == "success":
+        result["stats_html"] = perf.get("html", "")
+    else:
+        result["stats_html"] = ""
+
+    # 5. Team form (need team IDs from pre-match info)
+    home_team_id = None
+    away_team_id = None
+    if result["info"].get("home_team_id"):
+        home_team_id = result["info"]["home_team_id"]
+    if result["info"].get("away_team_id"):
+        away_team_id = result["info"]["away_team_id"]
+
+    # Try to extract team IDs from pre-match HTML if not in info
+    if not home_team_id or not away_team_id:
+        pm_html = result.get("pre_match_html", "")
+        if pm_html:
+            ids = re.findall(r'/doi-bong/(\d+)/overview/', pm_html)
+            if len(ids) >= 2:
+                home_team_id = home_team_id or ids[0]
+                away_team_id = away_team_id or ids[1]
+
+    result["home_form"] = []
+    result["away_form"] = []
+    if home_team_id:
+        form_data = _bongda_api("/api/fixtures/form", {"team_id": home_team_id})
+        if form_data and form_data.get("status") == "success":
+            result["home_form"] = _parse_form(form_data.get("html", ""))
+            result["home_form_html"] = form_data.get("html", "")
+    if away_team_id:
+        form_data = _bongda_api("/api/fixtures/form", {"team_id": away_team_id})
+        if form_data and form_data.get("status") == "success":
+            result["away_form"] = _parse_form(form_data.get("html", ""))
+            result["away_form_html"] = form_data.get("html", "")
 
     _set_cache(cache_key, result)
     return result
 
 
-def _parse_lineups(html):
-    """Parse lineups HTML into structured data."""
+def _parse_pre_match(html):
+    """Parse pre-match HTML into structured info."""
+    info = {}
+    try:
+        soup = BeautifulSoup(html, 'lxml')
+        text = soup.get_text(' ')
+
+        # Extract date
+        date_m = re.search(r'(\d{1,2}[/-]\d{1,2}[/-]\d{2,4}|\d{4}[/-]\d{1,2}[/-]\d{1,2})', text)
+        if date_m:
+            info["date"] = date_m.group(1)
+
+        # Extract time
+        time_m = re.search(r'(\d{1,2}:\d{2})', text)
+        if time_m:
+            info["time"] = time_m.group(1)
+
+        # Extract stadium
+        stadium_m = re.search(r'(Sân vận động|SVĐ|Stadium|Nhà thi đấu)[:\s]+([^,\n]+)', text, re.I)
+        if stadium_m:
+            info["stadium"] = _clean(stadium_m.group(2))
+
+        # Extract referee
+        referee_m = re.search(r'(Trọng tài|Referee)[:\s]+([^,\n]+)', text, re.I)
+        if referee_m:
+            info["referee"] = _clean(referee_m.group(2))
+
+        # Extract team IDs from links
+        team_links = soup.select('a[href*="/doi-bong/"]')
+        for link in team_links:
+            href = link.get('href', '')
+            m = re.search(r'/doi-bong/(\d+)/', href)
+            if m:
+                team_id = m.group(1)
+                team_name = _clean(link.get_text())
+                if team_name:
+                    if not info.get("home_team_id"):
+                        info["home_team_id"] = team_id
+                        info["home_team"] = team_name
+                    elif not info.get("away_team_id"):
+                        info["away_team_id"] = team_id
+                        info["away_team"] = team_name
+
+        # Extract league/tournament
+        league_link = soup.select_one('a[href*="/giai-dau/"]')
+        if league_link:
+            info["league"] = _clean(league_link.get_text())
+
+        # Extract team logos
+        logos = soup.select('img[src*="team-logo"]')
+        if len(logos) >= 1:
+            info["home_logo"] = logos[0].get('src', '')
+        if len(logos) >= 2:
+            info["away_logo"] = logos[1].get('src', '')
+
+    except:
+        pass
+    return info
+
+
+def _parse_h2h_matches(html):
+    """Parse H2H matches HTML into list of previous matches."""
+    matches = []
     try:
         soup = BeautifulSoup(html, 'lxml')
-        lineups = {"home_team": "", "away_team": "", "home_formation": "", "away_formation": "",
-                   "home_players": [], "away_players": [], "home_subs": [], "away_subs": []}
-        
-        team_headers = soup.select('.team-name, .team-header, [class*=team] h3, [class*=team] h4')
-        if len(team_headers) >= 2:
-            lineups["home_team"] = _clean(team_headers[0].get_text())
-            lineups["away_team"] = _clean(team_headers[1].get_text())
-        
-        formations = soup.select('.formation, [class*=formation]')
-        if len(formations) >= 2:
-            lineups["home_formation"] = _clean(formations[0].get_text())
-            lineups["away_formation"] = _clean(formations[1].get_text())
-
-        home_section = soup.select_one('.home-team, .team-home, [class*=home]')
-        away_section = soup.select_one('.away-team, .team-away, [class*=away]')
-
-        if home_section:
-            for player_el in home_section.select('.player, .player-item, .player-name, tr, li'):
-                name = _clean(player_el.get_text())
-                if name and 2 < len(name) < 60:
-                    lineups["home_players"].append(name)
-
-        if away_section:
-            for player_el in away_section.select('.player, .player-item, .player-name, tr, li'):
-                name = _clean(player_el.get_text())
-                if name and 2 < len(name) < 60:
-                    lineups["away_players"].append(name)
-
-        lineups["home_players"] = list(dict.fromkeys(lineups["home_players"]))
-        lineups["away_players"] = list(dict.fromkeys(lineups["away_players"]))
-        return lineups
+        for item in soup.select('li.match-detail, .match-detail'):
+            match = {}
+            # Date and league
+            dt_el = item.select_one('.datetime')
+            if dt_el:
+                time_el = dt_el.select_one('.match-time, p.match-time')
+                if time_el:
+                    match["date"] = _clean(time_el.get_text())
+                league_el = dt_el.select_one('.league, a.league')
+                if league_el:
+                    match["league"] = _clean(league_el.get_text())
+
+            # Teams
+            home_el = item.select_one('.home-team .name, .team.home-team .name')
+            away_el = item.select_one('.away-team .name, .team.away-team .name')
+            if home_el:
+                match["home"] = _clean(home_el.get_text())
+            if away_el:
+                match["away"] = _clean(away_el.get_text())
+
+            # Score
+            status_el = item.select_one('.status a, .status')
+            if status_el:
+                score_text = _clean(status_el.get_text())
+                match["score"] = score_text
+
+            if match.get("home") and match.get("away"):
+                matches.append(match)
     except:
-        return {"home_team": "", "away_team": "", "home_players": [], "away_players": [], "home_subs": [], "away_subs": []}
+        pass
+    return matches
 
 
-def _parse_h2h(html):
-    """Parse head-to-head HTML into list of previous matches."""
+def _parse_h2h_stats(html):
+    """Parse H2H stats comparison HTML."""
+    stats = {}
     try:
         soup = BeautifulSoup(html, 'lxml')
-        matches = []
-        for row in soup.select('tr, .match-item, [class*=match], li'):
-            cells = row.find_all(['td', 'div', 'span'])
-            if len(cells) >= 3:
-                date = _clean(cells[0].get_text())
-                home = _clean(cells[1].get_text())
-                away = _clean(cells[2].get_text()) if len(cells) > 2 else ""
-                score = _clean(cells[3].get_text()) if len(cells) > 3 else ""
-                if date and home and len(date) > 3:
-                    matches.append({"date": date, "home": home, "away": away, "score": score})
-        return matches
+        for row in soup.select('.stat-row, li.stat-row'):
+            label_el = row.select_one('.stat-label span, .stat-label')
+            if not label_el:
+                continue
+            label = _clean(label_el.get_text())
+
+            home_bar = row.select_one('.team-progress.home .progress-bar, .home .progress-bar')
+            away_bar = row.select_one('.team-progress.away .progress-bar, .away .progress-bar')
+
+            home_val = "0"
+            away_val = "0"
+
+            if home_bar:
+                pct = home_bar.get('style', '')
+                m = re.search(r'width:\s*(\d+)%', pct)
+                if m:
+                    home_val = m.group(1)
+                span = home_bar.select_one('span')
+                if span:
+                    home_val = _clean(span.get_text()) or home_val
+
+            if away_bar:
+                pct = away_bar.get('style', '')
+                m = re.search(r'width:\s*(\d+)%', pct)
+                if m:
+                    away_val = m.group(1)
+                span = away_bar.select_one('span')
+                if span:
+                    away_val = _clean(span.get_text()) or away_val
+
+            stats[label] = {"home": home_val, "away": away_val}
     except:
-        return []
+        pass
+    return stats
 
 
-def _parse_match_info(html):
-    """Parse basic match info from commentaries HTML."""
-    info = {}
+def _parse_form(html):
+    """Parse team form HTML into list of recent matches."""
+    matches = []
     try:
         soup = BeautifulSoup(html, 'lxml')
-        text = soup.get_text(' ')
-        date_m = re.search(r'(\d{1,2}[/-]\d{1,2}[/-]\d{2,4}|\d{4}[/-]\d{1,2}[/-]\d{1,2})', text)
-        if date_m: info["date"] = date_m.group(1)
-        time_m = re.search(r'(\d{1,2}:\d{2})', text)
-        if time_m: info["time"] = time_m.group(1)
-        stadium_m = re.search(r'(Sân vận động|SVĐ|Stadium|Nhà thi đấu)[:\s]+([^,\n]+)', text, re.I)
-        if stadium_m: info["stadium"] = _clean(stadium_m.group(2))
-        referee_m = re.search(r'(Trọng tài|Referee)[:\s]+([^,\n]+)', text, re.I)
-        if referee_m: info["referee"] = _clean(referee_m.group(2))
+        for item in soup.select('li.match-detail, .match-detail'):
+            match = {}
+            dt_el = item.select_one('.datetime')
+            if dt_el:
+                time_el = dt_el.select_one('.match-time, p.match-time')
+                if time_el:
+                    match["date"] = _clean(time_el.get_text())
+                league_el = dt_el.select_one('.league, a.league')
+                if league_el:
+                    match["league"] = _clean(league_el.get_text())
+
+            home_el = item.select_one('.home-team .name, .team.home-team .name')
+            away_el = item.select_one('.away-team .name, .team.away-team .name')
+            if home_el:
+                match["home"] = _clean(home_el.get_text())
+            if away_el:
+                match["away"] = _clean(away_el.get_text())
+
+            status_el = item.select_one('.status a, .status')
+            if status_el:
+                match["score"] = _clean(status_el.get_text())
+
+            # Determine result (T=Thắng, B=Bại, H=Hòa)
+            label_el = item.select_one('.label_form')
+            if label_el:
+                label_text = _clean(label_el.get_text())
+                css_class = ' '.join(label_el.get('class', []))
+                if 'bg-green' in css_class or label_text == 'T':
+                    match["result"] = "T"
+                elif 'bg-red' in css_class or label_text == 'B':
+                    match["result"] = "B"
+                else:
+                    match["result"] = "H"
+
+            if match.get("home") and match.get("away"):
+                matches.append(match)
     except:
         pass
-    return info
+    return matches