Upload match_detail.py

match_detail.py

@@ -0,0 +1,257 @@
+"""
+Match Detail Scraper for bongda.com.vn
+Fetches: preview/commentaries, lineups, head-to-head, stats for any match
+"""
+import requests, re, json, time, threading
+from bs4 import BeautifulSoup
+from urllib.parse import quote
+
+BONGDA_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+    "Accept": "application/json, text/javascript, */*; q=0.01",
+    "Accept-Language": "vi-VN,vi;q=0.9",
+    "Referer": "https://bongda.com.vn/lich-thi-dau",
+    "X-Requested-With": "XMLHttpRequest"
+}
+
+UA = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+    "Accept-Language": "vi-VN,vi;q=0.9"
+}
+
+_match_cache = {}
+_match_cache_lock = threading.Lock()
+_cache_ttl = 120  # 2 minutes for live matches
+
+
+def _cached(key):
+    with _match_cache_lock:
+        if key in _match_cache and time.time() - _match_cache[key]['t'] < _cache_ttl:
+            return _match_cache[key]['d']
+    return None
+
+
+def _set_cache(key, data):
+    with _match_cache_lock:
+        _match_cache[key] = {'t': time.time(), 'd': data}
+
+
+def _bongda_api(endpoint, params=None):
+    """Call bongda.com.vn API and return HTML snippet."""
+    try:
+        url = f"https://bongda.com.vn{endpoint}"
+        if params:
+            url += "?" + "&".join(f"{k}={v}" for k, v in params.items())
+        r = requests.get(url, headers=BONGDA_HEADERS, timeout=10)
+        if r.status_code == 200:
+            data = r.json()
+            if data.get("status") == "success":
+                return data.get("html", "")
+    except:
+        pass
+    return ""
+
+
+def _clean(s):
+    return re.sub(r'\s+', ' ', str(s or '')).strip()
+
+
+def fetch_match_detail_by_url(url):
+    """
+    Extract event_id from a bongda.com.vn match URL like:
+    https://bongda.com.vn/tran-dau/5154126/preview/giao-huu-belgium-tunisia
+    Then fetch all match detail data.
+    """
+    # Extract match_id from URL
+    m = re.search(r'/tran-dau/(\d+)/', url)
+    if m:
+        event_id = int(m.group(1))
+        return fetch_match_detail(event_id)
+    
+    # Try to fetch the page and extract event_id from embedded data
+    try:
+        r = requests.get(url, headers=UA, timeout=15)
+        if r.status_code == 200:
+            # Look for event_id in page source
+            m = re.search(r'event[_-]?id["\s:=]+(\d{5,})', r.text, re.I)
+            if m:
+                event_id = int(m.group(1))
+                return fetch_match_detail(event_id)
+            # Look in script tags
+            soup = BeautifulSoup(r.text, 'lxml')
+            for script in soup.find_all('script'):
+                txt = script.string or ''
+                m2 = re.search(r'event[_-]?id["\s:=]+(\d{5,})', txt, re.I)
+                if m2:
+                    event_id = int(m2.group(1))
+                    return fetch_match_detail(event_id)
+    except:
+        pass
+    
+    return {"error": "Could not extract event_id from URL"}
+
+
+def fetch_match_detail(event_id):
+    """
+    Fetch complete match detail:
+    - commentaries/preview
+    - lineups
+    - head-to-head
+    - match stats
+    Returns dict with HTML snippets + parsed preview text.
+    """
+    cache_key = f"match_detail_{event_id}"
+    cached = _cached(cache_key)
+    if cached:
+        return cached
+
+    result = {"event_id": event_id}
+
+    # 1. Commentaries / Preview
+    commentaries_html = _bongda_api("/api/fixtures/commentaries", {"event_id": event_id})
+    result["commentaries_html"] = commentaries_html
+
+    # Parse preview text from commentaries HTML
+    if commentaries_html:
+        try:
+            soup = BeautifulSoup(commentaries_html, 'lxml')
+            preview_text = []
+            for el in soup.find_all(['p', 'div', 'span']):
+                txt = el.get_text(strip=True)
+                if txt and len(txt) > 20:
+                    preview_text.append(txt)
+            result["preview_text"] = preview_text
+        except:
+            result["preview_text"] = []
+    else:
+        result["preview_text"] = []
+
+    # 2. Lineups
+    lineups_html = _bongda_api("/api/fixtures/lineups", {"event_id": event_id})
+    result["lineups_html"] = lineups_html
+
+    # Parse lineups structured data
+    if lineups_html:
+        result["lineups"] = _parse_lineups(lineups_html)
+    else:
+        result["lineups"] = {}
+
+    # 3. Head-to-Head
+    h2h_html = _bongda_api("/api/fixtures/head-to-head", {"event_id": event_id})
+    result["h2h_html"] = h2h_html
+
+    # Parse H2H structured data
+    if h2h_html:
+        result["h2h"] = _parse_h2h(h2h_html)
+    else:
+        result["h2h"] = []
+
+    # 4. Match Stats
+    stats_html = _bongda_api("/api/event-standing/player-performance", {"event_id": event_id})
+    result["stats_html"] = stats_html
+
+    # 5. Match Info (basic info like referee, venue, date)
+    # This might be embedded in the commentaries or available via another endpoint
+    result["info"] = _parse_match_info(commentaries_html)
+
+    _set_cache(cache_key, result)
+    return result
+
+
+def _parse_lineups(html):
+    """Parse lineups HTML into structured data."""
+    try:
+        soup = BeautifulSoup(html, 'lxml')
+        lineups = {"home_team": "", "away_team": "", "home_formation": "", "away_formation": "",
+                   "home_players": [], "away_players": [], "home_subs": [], "away_subs": []}
+        
+        # Find team names
+        team_headers = soup.select('.team-name, .team-header, [class*=team] h3, [class*=team] h4')
+        if len(team_headers) >= 2:
+            lineups["home_team"] = _clean(team_headers[0].get_text())
+            lineups["away_team"] = _clean(team_headers[1].get_text())
+        
+        # Find formations
+        formations = soup.select('.formation, [class*=formation]')
+        if len(formations) >= 2:
+            lineups["home_formation"] = _clean(formations[0].get_text())
+            lineups["away_formation"] = _clean(formations[1].get_text())
+
+        # Find starting XI players - look for player rows/items
+        home_section = soup.select_one('.home-team, .team-home, [class*=home]')
+        away_section = soup.select_one('.away-team, .team-away, [class*=away]')
+
+        if home_section:
+            for player_el in home_section.select('.player, .player-item, .player-name, tr, li'):
+                name = _clean(player_el.get_text())
+                if name and len(name) > 2 and len(name) < 60:
+                    lineups["home_players"].append(name)
+
+        if away_section:
+            for player_el in away_section.select('.player, .player-item, .player-name, tr, li'):
+                name = _clean(player_el.get_text())
+                if name and len(name) > 2 and len(name) < 60:
+                    lineups["away_players"].append(name)
+
+        # Remove duplicates while preserving order
+        lineups["home_players"] = list(dict.fromkeys(lineups["home_players"]))
+        lineups["away_players"] = list(dict.fromkeys(lineups["away_players"]))
+
+        return lineups
+    except:
+        return {"home_team": "", "away_team": "", "home_players": [], "away_players": [],
+                "home_subs": [], "away_subs": []}
+
+
+def _parse_h2h(html):
+    """Parse head-to-head HTML into list of previous matches."""
+    try:
+        soup = BeautifulSoup(html, 'lxml')
+        matches = []
+        
+        for row in soup.select('tr, .match-item, [class*=match], li'):
+            cells = row.find_all(['td', 'div', 'span'])
+            if len(cells) >= 3:
+                date = _clean(cells[0].get_text())
+                home = _clean(cells[1].get_text())
+                away = _clean(cells[2].get_text()) if len(cells) > 2 else ""
+                score = _clean(cells[3].get_text()) if len(cells) > 3 else ""
+                if date and home and len(date) > 3:
+                    matches.append({
+                        "date": date,
+                        "home": home,
+                        "away": away,
+                        "score": score
+                    })
+        
+        return matches
+    except:
+        return []
+
+
+def _parse_match_info(html):
+    """Parse basic match info from commentaries HTML."""
+    info = {}
+    try:
+        soup = BeautifulSoup(html, 'lxml')
+        text = soup.get_text(' ')
+        
+        # Look for common patterns
+        date_m = re.search(r'(\d{1,2}[/-]\d{1,2}[/-]\d{2,4}|\d{4}[/-]\d{1,2}[/-]\d{1,2})', text)
+        if date_m:
+            info["date"] = date_m.group(1)
+        
+        time_m = re.search(r'(\d{1,2}:\d{2})', text)
+        if time_m:
+            info["time"] = time_m.group(1)
+        
+        stadium_m = re.search(r'(Sân vận động|SVĐ|Stadium|Nhà thi đấu)[:\s]+([^,\n]+)', text, re.I)
+        if stadium_m:
+            info["stadium"] = _clean(stadium_m.group(2))
+        
+        referee_m = re.search(r'(Trọng tài|Referee)[:\s]+([^,\n]+)', text, re.I)
+        if referee_m:
+            info["referee"] = _clean(referee_m.group(2))
+    except:
+        pass
+    return info