Test upload size - first half of ai_ext.py from c93b544

ai_ext.py

@@ -15,6 +15,7 @@ from fastapi.responses import HTMLResponse, JSONResponse, FileResponse
 
 from main import app
 
+# Import wall store from main.py so we read/write the SAME file
 try:
     from main import _load_wall, _save_wall, _web_context  # noqa: F401
 except ImportError:
@@ -74,3 +75,198 @@ HEADERS = {
     "Accept-Language": "vi-VN,vi;q=0.9,en;q=0.8"
 }
 LAST_QWEN_ERROR = ""
+
+# ===== TTS VOICE CONFIG =====
+TTS_VOICES = {
+    "hoaimy":     {"id": "vi-VN-HoaiMyNeural",    "gender": "female", "name": "Hoài My",     "country": "Việt Nam", "lang": "vi", "flag": "🇻🇳"},
+    "namminh":    {"id": "vi-VN-NamMinhNeural",   "gender": "male",   "name": "Nam Minh",    "country": "Việt Nam", "lang": "vi", "flag": "🇻🇳"},
+    "en_us_f":    {"id": "en-US-AvaNeural",        "gender": "female", "name": "Ava",         "country": "Mỹ",       "lang": "en", "flag": "🇺🇸"},
+    "en_us_m":    {"id": "en-US-BrianNeural",      "gender": "male",   "name": "Brian",       "country": "Mỹ",       "lang": "en", "flag": "🇺🇸"},
+    "en_gb_f":    {"id": "en-GB-SoniaNeural",      "gender": "female", "name": "Sonia",       "country": "Anh",      "lang": "en", "flag": "🇬🇧"},
+    "en_gb_m":    {"id": "en-GB-RyanNeural",       "gender": "male",   "name": "Ryan",        "country": "Anh",      "lang": "en", "flag": "🇬🇧"},
+    "zh_cn_f":    {"id": "zh-CN-XiaoxiaoNeural",   "gender": "female", "name": "Xiaoxiao",    "country": "Trung Quốc", "lang": "zh", "flag": "🇨🇳"},
+    "zh_cn_m":    {"id": "zh-CN-YunyangNeural",    "gender": "male",   "name": "Yunyang",     "country": "Trung Quốc", "lang": "zh", "flag": "🇨🇳"},
+    "zh_hk_f":    {"id": "zh-HK-HiuGaaiNeural",    "gender": "female", "name": "HiuGaai",     "country": "Hồng Kông", "lang": "zh", "flag": "🇭🇰"},
+    "ja_jp_f":    {"id": "ja-JP-NanamiNeural",     "gender": "female", "name": "Nanami",      "country": "Nhật Bản", "lang": "ja", "flag": "🇯🇵"},
+    "ja_jp_m":    {"id": "ja-JP-KeitaNeural",      "gender": "male",   "name": "Keita",       "country": "Nhật Bản", "lang": "ja", "flag": "🇯🇵"},
+    "ko_kr_f":    {"id": "ko-KR-SunHiNeural",      "gender": "female", "name": "Sun-Hi",      "country": "Hàn Quốc", "lang": "ko", "flag": "🇰🇷"},
+    "ko_kr_m":    {"id": "ko-KR-InJoonNeural",     "gender": "male",   "name": "InJoon",      "country": "Hàn Quốc", "lang": "ko", "flag": "🇰🇷"},
+    "fr_fr_f":    {"id": "fr-FR-DeniseNeural",      "gender": "female", "name": "Denise",      "country": "Pháp",     "lang": "fr", "flag": "🇫🇷"},
+    "fr_fr_m":    {"id": "fr-FR-HenriNeural",       "gender": "male",   "name": "Henri",       "country": "Pháp",     "lang": "fr", "flag": "🇫🇷"},
+    "de_de_f":    {"id": "de-DE-KatjaNeural",       "gender": "female", "name": "Katja",       "country": "Đức",      "lang": "de", "flag": "🇩🇪"},
+    "de_de_m":    {"id": "de-DE-KillianNeural",     "gender": "male",   "name": "Killian",     "country": "Đức",      "lang": "de", "flag": "🇩🇪"},
+    "es_es_f":    {"id": "es-ES-XimenaNeural",      "gender": "female", "name": "Ximena",      "country": "Tây Ban Nha", "lang": "es", "flag": "🇪🇸"},
+    "es_mx_m":    {"id": "es-MX-JorgeNeural",       "gender": "male",   "name": "Jorge",       "country": "Mexico",   "lang": "es", "flag": "🇲🇽"},
+    "th_th_f":    {"id": "th-TH-PremwadeeNeural",   "gender": "female", "name": "Premwadee",   "country": "Thái Lan", "lang": "th", "flag": "🇹🇭"},
+    "th_th_m":    {"id": "th-TH-NiwatNeural",       "gender": "male",   "name": "Niwat",       "country": "Thái Lan", "lang": "th", "flag": "🇹🇭"},
+    "hi_in_f":    {"id": "hi-IN-SwaraNeural",       "gender": "female", "name": "Swara",       "country": "Ấn Độ",    "lang": "hi", "flag": "🇮🇳"},
+    "hi_in_m":    {"id": "hi-IN-MadhurNeural",      "gender": "male",   "name": "Madhur",      "country": "Ấn Độ",    "lang": "hi", "flag": "🇮🇳"},
+}
+TTS_DEFAULT_VOICE = "hoaimy"
+TTS_DEFAULT_SPEED = 1.2
+
+TOPIC_VOICE_MAP = {
+    "bóng đá": "namminh", "thể thao": "namminh", "world cup": "namminh",
+    "premier league": "namminh", "champions league": "namminh", "la liga": "namminh",
+    "serie a": "namminh", "bundesliga": "namminh", "v-league": "namminh",
+    "tennis": "namminh", "olympic": "namminh", "f1": "namminh", "moto": "namminh",
+    "sức khỏe": "hoaimy", "làm đẹp": "hoaimy", "giải trí": "hoaimy",
+    "âm nhạc": "hoaimy", "phim": "hoaimy", "thời trang": "hoaimy",
+    "ẩm thực": "hoaimy", "du lịch": "hoaimy", "gia đình": "hoaimy",
+    "tình yêu": "hoaimy", "hôn nhân": "hoaimy", "mẹ và bé": "hoaimy",
+    "công nghệ": "namminh", "ai": "namminh", "robot": "namminh",
+    "khoa học": "namminh", "vũ trụ": "namminh", "điện thoại": "namminh",
+    "laptop": "namminh", "game": "namminh",
+    "chính trị": "namminh", "kinh tế": "namminh", "tài chính": "namminh",
+    "chứng khoán": "namminh", "ngân hàng": "namminh", "thị trường": "namminh",
+    "xã hội": "namminh", "pháp luật": "namminh", "giáo dục": "namminh",
+}
+
+def _detect_voice_for_topic(title: str, text: str) -> str:
+    combined = (title + " " + text[:500]).lower()
+    for keyword, voice_id in TOPIC_VOICE_MAP.items():
+        if keyword in combined:
+            return voice_id
+    return TTS_DEFAULT_VOICE
+
+def _clean_text(s: str) -> str:
+    s = html_lib.unescape(s or "")
+    return re.sub(r"\s+", " ", s).strip()
+
+def _domain(u):
+    try:
+        return urlparse(u).netloc.replace("www.", "")
+    except Exception:
+        return ""
+
+def _safe_name(s):
+    return re.sub(r"[^a-zA-Z0-9_-]+", "_", str(s))[:80]
+
+def _clean_ai_output(text: str) -> str:
+    if not text:
+        return ""
+    text = re.sub(r'^#{1,6}\s+', '', text, flags=re.MULTILINE)
+    text = re.sub(r'\*\*([^*]+)\*\*', r'\1', text)
+    text = re.sub(r'\*([^*]+)\*', r'\1', text)
+    text = re.sub(r'^---+\s*$', '', text, flags=re.MULTILINE)
+    text = re.sub(r'^[-*_]{3,}\s*$', '', text, flags=re.MULTILINE)
+    leakage = [
+        r'Dưới đây là', r'Theo yêu cầu', r'Tôi sẽ viết', r'Tôi sẽ tóm tắt',
+        r'Đây là bài', r'Đây là nội dung', r'Bài viết sau đây',
+        r'Nội dung (tóm tắt|chính)', r'Nhiệm vụ', r'Vai trò', r'Tôi là',
+        r'Dựa trên.*tôi sẽ', r'Hãy', r'Bạn cần', r'Đọc bài viết',
+        r'Tôi xin', r'Xin chào', r'Trân trọng', r'Kính thưa',
+        r'Dựa trên.*dưới đây', r'Sau đây là', r'Dưới đây là bài',
+    ]
+    for phrase in leakage:
+        text = re.sub(r'^' + phrase + r'[^\n]*\n?', '', text, flags=re.MULTILINE | re.IGNORECASE)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    def _norm(s):
+        return re.sub(r'\s+', ' ', s.strip().lower())
+    raw_parts = re.split(r'(?<=[.!?])\s+', text.strip())
+    seen_sentences = set()
+    unique_parts = []
+    for part in raw_parts:
+        n = _norm(part)
+        is_dup = False
+        if n:
+            if n in seen_sentences:
+                is_dup = True
+            else:
+                partial = re.sub(r'\W+', '', n)
+                for seen in seen_sentences:
+                    seen_clean = re.sub(r'\W+', '', seen)
+                    if partial and seen_clean and (partial in seen_clean or seen_clean in partial):
+                        shorter = min(len(partial), len(seen_clean))
+                        longer = max(len(partial), len(seen_clean))
+                        if shorter > 20 and shorter / longer > 0.75:
+                            is_dup = True
+                            break
+        if is_dup:
+            continue
+        if n:
+            seen_sentences.add(n)
+        unique_parts.append(part)
+    result = ' '.join(unique_parts).strip()
+    lines = result.split('\n')
+    final_lines = []
+    prev_line = ""
+    for line in lines:
+        stripped = line.strip()
+        if stripped and stripped == prev_line:
+            continue
+        final_lines.append(line)
+        prev_line = stripped
+    result = '\n'.join(final_lines).strip()
+    return result
+
+
+# ===== EXTRACT ALL IMAGES FROM ARTICLE =====
+def _extract_all_images(soup, base_url: str) -> List[Dict]:
+    images = []
+    seen_urls = set()
+    skip_patterns = [
+        "avatar", "icon", "logo", "button", "banner-ad", "tracking",
+        "beacon", "pixel", "1x1", "spacer", "emoji", "sprite", "placeholder",
+        "advertisement", "ads", "widget", "sidebar", "footer-logo",
+    ]
+    def _add_image(src: str, alt: str = "", source_tag: str = "img"):
+        if not src or src.startswith("data:"):
+            return
+        abs_url = urljoin(base_url, src.strip())
+        if abs_url in seen_urls:
+            return
+        if any(p in abs_url.lower() for p in skip_patterns):
+            return
+        try:
+            parsed = urlparse(abs_url)
+            path = parsed.path.lower()
+            if any(path.endswith(ext) for ext in ['.svg', '.ico', '.gif']):
+                return
+        except Exception:
+            pass
+        seen_urls.add(abs_url)
+        images.append({"url": abs_url, "alt": alt, "source": source_tag})
+    for img in soup.find_all("img"):
+        src = (img.get("src") or img.get("data-src") or img.get("data-lazy-src") or
+               img.get("data-original") or img.get("data-srcset", "").split(",")[0].strip().split(" ")[0])
+        _add_image(src, alt=img.get("alt", ""), source_tag="img")
+    for img in soup.find_all("img", srcset=True):
+        for part in img["srcset"].split(","):
+            part = part.strip()
+            if part:
+                _add_image(part.split(" ")[0], alt=img.get("alt", ""), source_tag="srcset")
+    for picture in soup.find_all("picture"):
+        for source in picture.find_all("source"):
+            srcset = source.get("srcset", "")
+            for part in srcset.split(","):
+                part = part.strip()
+                if part:
+                    _add_image(part.split(" ")[0], source_tag="picture/srcset")
+        fallback_img = picture.find("img")
+        if fallback_img:
+            _add_image(fallback_img.get("src") or fallback_img.get("data-src"),
+                       alt=fallback_img.get("alt", ""), source_tag="picture/img")
+    for img in soup.find_all("img", class_=re.compile(r"wp-image|size-large|size-full|aligncenter")):
+        _add_image(img.get("data-src") or img.get("src"), alt=img.get("alt", ""), source_tag="wp-image")
+    for tag in soup.find_all(style=re.compile(r"background-image")):
+        for m in re.findall(r'url\(["\']?(.*?)["\']?\)', tag.get("style", "")):
+            _add_image(m, source_tag="background-style")
+    og_image = soup.find("meta", property="og:image")
+    if og_image and og_image.get("content"):
+        _add_image(og_image["content"], source_tag="og:image")
+    tw_image = soup.find("meta", attrs={"name": "twitter:image"})
+    if tw_image and tw_image.get("content"):
+        _add_image(tw_image["content"], source_tag="twitter:image")
+    for figure in soup.find_all("figure"):
+        img = figure.find("img")
+        if img:
+            src = img.get("data-src") or img.get("src")
+            figcaption = figure.find("figcaption")
+            alt = figcaption.get_text(strip=True) if figcaption else img.get("alt", "")
+            _add_image(src, alt=alt, source_tag="figure")
+    for a in soup.find_all("a", href=True):
+        href = a["href"]
+        if any(href.lower().endswith(ext) for ext in [".jpg", ".jpeg", ".png", ".webp", ".gif"]):
+            _add_image(href, alt=a.get_text(strip=True)[:80], source_tag="link")
+    return images