Improve article image detection without over-filtering

ai_runtime_final2.py

@@ -0,0 +1,242 @@
+"""Final2: improve article-image detection without over-filtering real article images."""
+import re, requests
+from urllib.parse import urlparse
+import ai_runtime_final as f1
+from ai_runtime_final import app, base, rt, HTMLResponse, JSONResponse, Request, Query
+
+
+def _domain(url):
+    try:return urlparse(url or '').netloc.replace('www.','')
+    except Exception:return ''
+
+
+def _abs_url(src, base_url):
+    if not src:return ''
+    src=src.strip()
+    if src.startswith('//'):return 'https:'+src
+    if src.startswith('/'):
+        try:
+            p=urlparse(base_url);return f'{p.scheme}://{p.netloc}{src}'
+        except Exception:return src
+    return src
+
+BAD_RE=re.compile(r'(related|relate|recommend|suggest|sidebar|ads|advert|popular|xem-them|xemthem|tin-lien-quan|tinlienquan|doc-them|docthem|other-news|news-other|article-related|box-tin|box_related|story-related|recommend-news|same-category|cate-list|most-view|banner|qc|quang-cao|sponsor|social|share|comment|author|newsletter)',re.I)
+GOOD_RE=re.compile(r'(article|content|detail|body|post|entry|story|fck|cms|singular|main|news)',re.I)
+IMG_EXT_RE=re.compile(r'\.(jpg|jpeg|png|webp|avif)(\?|$)',re.I)
+ARTICLE_LINK_RE=re.compile(r'\.(html|htm|shtml|tpo|chn)(\?|$)|/\d{4}/|post\d+|article',re.I)
+
+
+def _clean_soup(soup):
+    for tag in soup.find_all(['script','style','nav','footer','aside','form','noscript','iframe']):
+        tag.decompose()
+
+
+def _find_article_block(soup):
+    """Find the article body first; do not delete suspected related blocks before finding it."""
+    selectors=[
+        'article', 'main article',
+        '.article-content','.article__content','.article__body','.article-body','.article-detail','.article__detail',
+        '.detail-content','.content-detail','.singular-content','.news-content','.post-content','.entry-content',
+        '.knc-content','.fck_detail','.cms-body','.story-body','.maincontent','.main-content',
+        '[class*=article-content]','[class*=article__content]','[class*=detail-content]','[class*=singular-content]',
+        '[class*=cms-body]','[class*=story-body]'
+    ]
+    for sel in selectors:
+        el=soup.select_one(sel)
+        if el and (len(el.find_all('p'))>=2 or len(el.find_all(['figure','picture','img']))>=1):
+            return el
+    best=None;best_score=0
+    for el in soup.find_all(['article','main','section','div']):
+        cls=' '.join(el.get('class',[]));eid=el.get('id','')
+        if BAD_RE.search(cls+' '+eid) and not GOOD_RE.search(cls+' '+eid):
+            continue
+        ps=el.find_all('p');imgs=el.find_all('img')
+        text=' '.join(p.get_text(' ',strip=True) for p in ps)
+        long_ps=sum(1 for p in ps if len(p.get_text(' ',strip=True))>40)
+        score=long_ps*180+len(ps)*40+min(len(text),5000)+len(imgs)*25
+        if GOOD_RE.search(cls+' '+eid):score+=800
+        if score>best_score:
+            best=el;best_score=score
+    return best or soup
+
+
+def _ancestor_bad(im, block):
+    node=im
+    while node and node is not block:
+        if getattr(node,'name',None) in ['aside','nav','footer']:
+            return True
+        cls=' '.join(node.get('class',[])) if hasattr(node,'get') else ''
+        eid=node.get('id','') if hasattr(node,'get') else ''
+        if BAD_RE.search(cls+' '+eid):
+            return True
+        node=getattr(node,'parent',None)
+    return False
+
+
+def _image_anchor_penalty(im, page_url):
+    a=im.find_parent('a')
+    if not a:return 0
+    href=_abs_url(a.get('href',''),page_url)
+    if not href:return 0
+    # If anchor opens the image itself, do not penalize.
+    if IMG_EXT_RE.search(href):return 0
+    # If anchor points to another article, it is probably related content.
+    try:
+        p1=urlparse(page_url);p2=urlparse(href)
+        if href!=page_url and ARTICLE_LINK_RE.search(href) and (p2.path!=p1.path):
+            return -100
+    except Exception:pass
+    return -10
+
+
+def _near_article_text_score(im):
+    score=0
+    # caption/figcaption is strong sign of article image
+    fig=im.find_parent('figure')
+    if fig:
+        score+=5
+        cap=fig.find('figcaption')
+        if cap and len(cap.get_text(' ',strip=True))>10:score+=4
+    if im.find_parent('picture'):score+=2
+    # paragraph around image
+    parent=im.parent
+    for node in [parent, getattr(parent,'parent',None) if parent else None, fig]:
+        if not node:continue
+        ps=node.find_all('p') if hasattr(node,'find_all') else []
+        if any(len(p.get_text(' ',strip=True))>40 for p in ps):score+=3;break
+    # sibling paragraph near figure/image
+    holder=fig or parent
+    if holder:
+        for sib in [holder.find_previous_sibling(), holder.find_next_sibling()]:
+            if sib and len(sib.get_text(' ',strip=True))>40:
+                score+=2
+                break
+    return score
+
+
+def _image_score(im, src, block, page_url):
+    low=(src or '').lower()
+    if not src or src.startswith('data:') or 'base64' in low:return -999
+    if any(x in low for x in ['logo','icon','avatar','sprite','tracking','pixel','social','share','author']):return -999
+    if _ancestor_bad(im,block):return -999
+    score=0
+    # Explicit dimensions: only reject truly tiny images; if missing dimensions, allow.
+    try:
+        w=int(re.sub(r'\D','',str(im.get('width') or '0')) or 0);h=int(re.sub(r'\D','',str(im.get('height') or '0')) or 0)
+        if (w and w<120) or (h and h<90):return -999
+        if w>=500 or h>=300:score+=3
+    except Exception:pass
+    alt=(im.get('alt') or im.get('title') or '').lower()
+    if any(x in alt for x in ['logo','avatar','quảng cáo','advertisement','banner']):return -999
+    cls=' '.join(im.get('class',[]));eid=im.get('id','')
+    if BAD_RE.search(cls+' '+eid):return -999
+    if GOOD_RE.search(cls+' '+eid):score+=2
+    score+=_near_article_text_score(im)
+    score+=_image_anchor_penalty(im,page_url)
+    if any(x in low for x in ['cdn','photo','image','media','upload','thumb','avatar']):score+=1
+    # Tienphong and many VN papers use lazy/data src without figure; still accept if inside article block.
+    if im.find_parent(['article','main']) or GOOD_RE.search(' '.join(block.get('class',[]))+' '+block.get('id','')):score+=3
+    return score
+
+
+def _extract_img_src(im, page_url):
+    src=(im.get('data-src') or im.get('data-original') or im.get('data-lazy-src') or im.get('data-srcset') or im.get('srcset') or im.get('src') or '')
+    if ',' in src:src=src.split(',')[0].strip().split(' ')[0]
+    else:src=src.strip().split(' ')[0]
+    return _abs_url(src,page_url)
+
+
+def _article_only_images(url):
+    imgs=[]
+    try:
+        from bs4 import BeautifulSoup
+        r=requests.get(url,headers=getattr(base,'HEADERS',{}),timeout=18);r.encoding='utf-8'
+        soup=BeautifulSoup(r.text,'lxml')
+        _clean_soup(soup)
+        block=_find_article_block(soup)
+        candidates=[]
+        for el in block.find_all(['figure','picture'],recursive=True):
+            im=el.find('img')
+            if im and im not in candidates:candidates.append(im)
+        for im in block.find_all('img',recursive=True):
+            if im not in candidates:candidates.append(im)
+        scored=[];seen=set()
+        for im in candidates:
+            src=_extract_img_src(im,url)
+            if not src or src in seen:continue
+            seen.add(src)
+            sc=_image_score(im,src,block,url)
+            if sc>=2:
+                scored.append((sc,src))
+        # Keep original article order but only for scored images, filtering duplicate URLs.
+        good=set(src for sc,src in sorted(scored,reverse=True) if sc>=2)
+        for im in candidates:
+            src=_extract_img_src(im,url)
+            if src in good and src not in imgs:imgs.append(src)
+            if len(imgs)>=20:break
+        # Fallback: og:image is usually article main image, and better than no image.
+        if not imgs:
+            og=soup.find('meta',property='og:image') or soup.find('meta',attrs={'name':'twitter:image'})
+            if og:
+                src=_abs_url(og.get('content',''),url)
+                if src and not any(x in src.lower() for x in ['logo','icon','avatar','sprite']):imgs.append(src)
+    except Exception:pass
+    return imgs[:20]
+
+
+def _scrape_url_article_only(url):
+    data=base.scrape_any_url(url)
+    imgs=_article_only_images(url)
+    data['images']=imgs
+    data['image']=imgs[0] if imgs else ''
+    return data
+
+# Override the functions used by inherited endpoints.
+f1._article_only_images=_article_only_images
+f1._scrape_url_article_only=_scrape_url_article_only
+
+# Replace URL endpoints to use improved extraction.
+_PATCH={('/api/url_wall','POST'),('/api/rewrite_share','POST'),('/api/ai/url','POST'),('/','GET')}
+app.router.routes=[r for r in app.router.routes if not any(getattr(r,'path',None)==p and m in getattr(r,'methods',set()) for p,m in _PATCH)]
+
+@app.post('/api/url_wall')
+async def final2_url_wall(request:Request):
+    body=await request.json();url=base._clean_text(body.get('url',''))
+    if not url.startswith('http'):return JSONResponse({'error':'missing url'},status_code=400)
+    try:data=_scrape_url_article_only(url)
+    except Exception as e:return JSONResponse({'error':'Không scrape được URL: '+str(e)[:180]},status_code=422)
+    raw=(data.get('summary','')+'\n'+data.get('text','')).strip()
+    if len(raw)<120:return JSONResponse({'error':'URL không có đủ nội dung để tóm tắt'},status_code=422)
+    prompt=f"""Tóm tắt bài viết nguồn dưới đây để đăng lên Tường AI VNEWS.
+
+Yêu cầu:
+- Chỉ tóm tắt nội dung chính, không viết lại toàn bộ bài.
+- Ngắn gọn, cụ thể, dễ hiểu.
+- Không lặp ý, không thêm chi tiết ngoài nguồn.
+- Tối đa 5 ý chính hoặc 2 đoạn ngắn.
+- Hạn chế dùng dấu đầu dòng.
+
+Tiêu đề gốc: {data.get('title','')}
+Nguồn: {_domain(url)}
+Nội dung gốc:
+{raw[:16000]}"""
+    text=await base.qwen_generate(prompt,image_url=(data.get('image') or None),max_tokens=900)
+    if not text:text=rt.old._fallback_summary_from_prompt(prompt,max_units=5) if hasattr(rt.old,'_fallback_summary_from_prompt') else raw[:900]
+    text=rt.postprocess(text) if hasattr(rt,'postprocess') else text
+    src=[{'title':data.get('title'), 'url':url, 'excerpt':raw[:500], 'via':_domain(url)}]
+    if 'Nguồn tham khảo:' not in text:text+='\n\n'+rt.source_line(src)
+    imgs=data.get('images') or []
+    post=base.make_post(data.get('title') or 'Bài viết',text,imgs[0] if imgs else '',url,'url',sources=src)
+    post['images']=imgs
+    posts=base._load_ai_wall();posts.insert(0,post);base._save_ai_wall(posts)
+    return JSONResponse({'post':post})
+
+@app.post('/api/rewrite_share')
+async def final2_rewrite_share(request:Request):return await final2_url_wall(request)
+@app.post('/api/ai/url')
+async def final2_ai_url(request:Request):return await final2_url_wall(request)
+
+@app.get('/')
+async def index_final2():
+    html=f1._load_index_html();body=getattr(rt.old,'PATCH_INJECT','') + f1.FINAL_INJECT
+    return HTMLResponse(html.replace('</body>',body+'\n</body>') if '</body>' in html else html+body)