import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import spaces # MUST come before any torch/CUDA-touching import import torch import torchvision import torchvision.transforms.functional as _F import sys import types # Compatibility shim: torchvision >= 0.21 removed functional_tensor, which pytorchvideo still imports if not hasattr(torchvision.transforms, 'functional_tensor'): _mod = types.ModuleType('torchvision.transforms.functional_tensor') for _name in dir(_F): setattr(_mod, _name, getattr(_F, _name)) sys.modules['torchvision.transforms.functional_tensor'] = _mod torchvision.transforms.functional_tensor = _mod import gradio as gr import numpy as np import tempfile import time from pathlib import Path from huggingface_hub import snapshot_download, hf_hub_download from omegaconf import OmegaConf import copy import einops # Patch config module to use HF Hub paths before any model imports import config as _config_module # Download pretrained models from HF Hub and patch config paths _qwen3_06b_path = snapshot_download("Qwen/Qwen3-0.6B") _clip_path = snapshot_download("openai/clip-vit-base-patch16") _hubert_path = snapshot_download("TencentGameMate/chinese-hubert-base") _config_module.PATH_TO_LLM['Qwen3_0.6B'] = _qwen3_06b_path _config_module.PATH_TO_VISUAL['CLIP_VIT_BASE'] = _clip_path _config_module.PATH_TO_AUDIO['HUBERT_BASE'] = _hubert_path # Now import the model code (after config is patched) from my_affectgpt.common.registry import registry from my_affectgpt.models.affectgpt import AffectGPT from my_affectgpt.models.encoder import * # registers encoders from my_affectgpt.conversation.conversation_video import Chat from my_affectgpt.processors.video_processor import ( AlproVideoEvalProcessor, load_video, ) from my_affectgpt.processors.blip_processors import Blip2ImageEvalProcessor from my_affectgpt.models.ImageBind.data import load_audio, transform_audio # ---- Model config matching stage2-mgrpo-qwen3-0.6b/config.yaml ---- MODEL_CFG = OmegaConf.create({ "arch": "affectgpt", "model_type": "pretrain_vicuna", "frozen_video_proj": False, "frozen_video_Qformer": False, "frozen_audio_Qformer": False, "frozen_audio_proj": False, "frozen_multi_Qformer": False, "frozen_multi_llama_proj": False, "frozen_llm": False, "multi_fusion_type": "attention", "video_fusion_type": "attention", "audio_fusion_type": "attention", "image_fusion_type": "mean", "ckpt": "", "ckpt_2": "", "llama_model": "Qwen3_0.6B", "acoustic_encoder": "HUBERT_BASE", "visual_encoder": "CLIP_VIT_BASE", "num_audio_query_token": 1, "num_video_query_token": 1, "num_multi_query_token": 1, "num_image_query_token": 1, "max_length": 1024, "lora_r": 16, "vis_processor": { "train": { "name": "alpro_video_eval", "n_frms": 8, "image_size": 224, } }, }) # Download the Light-MER Stage 2 checkpoint ckpt_path = hf_hub_download( "kevin233333/Light-MER", "stage2-mgrpo-qwen3-0.6b/checkpoint_001500_grpo.pth", ) MODEL_CFG.ckpt_3 = ckpt_path print("Loading Light-MER model...") model_cls = registry.get_model_class(MODEL_CFG.arch) model = model_cls.from_config(MODEL_CFG) model = model.to("cuda").eval() chat = Chat(model, MODEL_CFG, device="cuda") # Video processor for inference vis_processor = AlproVideoEvalProcessor( image_size=224, n_frms=8, mean=(0.48145466, 0.4578275, 0.40821073), std=(0.26862954, 0.26130258, 0.27577711), ) # Default prompt (matching the dataset's description question) DEFAULT_QUESTION = "Please infer the person's emotional state and provide your reasoning process." FACE_OR_FRAME = "multiface_audio_face_text" def video_file_has_audio(path): """Best-effort MP4/MOV audio-track check used before video-audio extraction.""" if path is None: return False suffix = Path(path).suffix.lower() if suffix not in {".mp4", ".m4v", ".mov"}: return True try: return b"soun" in Path(path).read_bytes() except Exception: return True @spaces.GPU(duration=30) def recognize_emotion(video_path, audio_path=None, subtitle="", question=DEFAULT_QUESTION, max_new_tokens=512, temperature=0.9, top_p=0.9, do_sample=True, progress=gr.Progress(track_tqdm=True)): """Recognize emotions from video/audio/text using Light-MER.""" subtitle = "" if subtitle is None else subtitle.strip() if question is None or question.strip() == "": question = DEFAULT_QUESTION has_audio_upload = audio_path is not None and audio_path != "" if video_path is None and not has_audio_upload and subtitle == "": return "Please provide a video, audio file, or subtitle/context text." start_time = time.time() warnings = [] audio_source = None sample_data = { 'frame': None, 'raw_frame': None, 'face': None, 'raw_face': None, 'audio': None, 'raw_audio': None, 'image': None, 'raw_image': None, } # The public Space accepts ordinary videos rather than precomputed OpenFace # crops. Use the raw sampled frames honestly as ; do not label # them as face crops. if video_path is not None: try: raw_frame, _ = load_video( video_path=video_path, n_frms=8, height=224, width=224, sampling="uniform", return_msg=True, ) frame = vis_processor.transform(raw_frame) sample_data['frame'] = frame sample_data['raw_frame'] = raw_frame except Exception as e: warnings.append(f"Video loading failed: {e}") print(f"Video loading error: {e}") if has_audio_upload: try: raw_audio = load_audio([audio_path], "cpu", clips_per_video=8)[0] audio = transform_audio(raw_audio, "cpu") sample_data['audio'] = audio sample_data['raw_audio'] = raw_audio audio_source = "uploaded audio" except Exception as e: warnings.append(f"Uploaded audio loading failed: {e}") print(f"Audio loading error: {e}") # If the uploaded video has an audio stream, use it automatically. if video_path is not None and sample_data['audio'] is None: if not video_file_has_audio(video_path): warnings.append("The video has no audio track; running with video frames only.") else: try: raw_audio = load_audio([video_path], "cpu", clips_per_video=8)[0] audio = transform_audio(raw_audio, "cpu") sample_data['audio'] = audio sample_data['raw_audio'] = raw_audio audio_source = "audio extracted from video" except Exception as e: warnings.append("No usable audio stream was found in the video.") print(f"Audio extraction from video failed: {e}") audio_hiddens, audio_llms = chat.postprocess_audio(sample_data) frame_hiddens, frame_llms = chat.postprocess_frame(sample_data) face_hiddens, face_llms = None, None _, image_llms = chat.postprocess_image(sample_data) multi_llms = None video_hiddens_for_fusion = face_hiddens if face_hiddens is not None else frame_hiddens if video_hiddens_for_fusion is not None and audio_hiddens is not None: _, multi_llms = chat.postprocess_multi(video_hiddens_for_fusion, audio_hiddens) img_list = { 'audio': audio_llms, 'frame': frame_llms, 'face': face_llms, 'image': image_llms, 'multi': multi_llms, } prompt_parts = ["###Human: "] if multi_llms is not None: prompt_parts.append("The audio and video merged info is: . ") if audio_llms is not None: prompt_parts.append("The audio content is as follows: . ") if face_llms is not None: prompt_parts.append("Meanwhile, we uniformly sample raw frames from the video and extract faces from these frames: . ") elif frame_llms is not None: if audio_llms is None and subtitle == "": prompt_parts.append("We uniformly sample raw frames from the video: . ") else: prompt_parts.append("Meanwhile, we uniformly sample raw frames from the video: . ") # Keep the trained subtitle framing when text is provided, and for # audio-video runs where the benchmark prompt includes the subtitle field. if subtitle or (frame_llms is not None and audio_llms is not None): prompt_parts.append(f"The subtitle of this video is: {subtitle}. ") prompt_parts.append(f"Now, please answer my question based on all the provided information. {question} ###Assistant: ") prompt = "".join(prompt_parts) response = chat.answer_sample( prompt=prompt, img_list=img_list, num_beams=1, temperature=temperature, do_sample=do_sample, top_p=top_p, max_new_tokens=max_new_tokens, max_length=2000, ) modalities = [] if frame_llms is not None: modalities.append("video frames") if face_llms is not None: modalities.append("face crops") if audio_llms is not None: modalities.append(audio_source or "audio") if subtitle: modalities.append("subtitle/text") if multi_llms is not None: modalities.append("audio-video fusion") if not modalities: modalities.append("text only") elapsed = time.time() - start_time status = [f"Modalities used: {', '.join(modalities)}"] status.extend(warnings) result = f"{response}\n\n---\n" + "\n".join(status) + f"\nInference time: {elapsed:.2f}s" return result # ---- Gradio UI ---- CSS = """ #col-container { max-width: 1100px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo: gr.Markdown("# Light-MER: Lightweight Multimodal Emotion Recognition") gr.Markdown( "A sub-1B parameter multimodal emotion recognition model that processes " "video, audio, and text to recognize emotions. Upload a video, optional audio, " "and optional subtitle/context text." ) gr.Markdown( "Paper: [Do We Really Need Multimodal Emotion Language Models Larger Than 1B Parameters?](https://arxiv.org/abs/2607.12787) | " "[GitHub](https://github.com/GAIR-Lab/Light-MER) | " "[Model](https://huggingface.co/kevin233333/Light-MER)" ) gr.Markdown( "Examples use full audio-visual speech clips from " "[RAVDESS](https://zenodo.org/records/1188976) with transcript text." ) with gr.Row(): with gr.Column(scale=2): video_input = gr.Video(label="Video Input (sampled frames)") audio_input = gr.Audio(label="Audio Input (optional)", type="filepath") subtitle_input = gr.Textbox( label="Subtitle / Context (optional)", placeholder="Enter subtitle text if available...", lines=2, ) question_input = gr.Textbox( label="Question", value=DEFAULT_QUESTION, lines=2, ) run_btn = gr.Button("Recognize Emotion", variant="primary") with gr.Column(scale=3): output_text = gr.Textbox( label="Emotion Recognition Result", lines=15, show_copy_button=True, ) with gr.Accordion("Advanced Settings", open=False): max_tokens = gr.Slider( label="Max New Tokens", minimum=64, maximum=1024, value=512, step=64 ) temp = gr.Slider( label="Temperature", minimum=0.1, maximum=2.0, value=0.9, step=0.1 ) top_p_val = gr.Slider( label="Top P", minimum=0.1, maximum=1.0, value=0.9, step=0.05 ) do_sample_chk = gr.Checkbox(label="Do Sample", value=True) gr.Examples( examples=[ ["examples/ravdess_happy_strong_kids_actor01.mp4", None, "Kids are talking by the door.", DEFAULT_QUESTION], ["examples/ravdess_sad_strong_kids_actor02.mp4", None, "Kids are talking by the door.", DEFAULT_QUESTION], ["examples/ravdess_angry_strong_kids_actor03.mp4", None, "Kids are talking by the door.", DEFAULT_QUESTION], ], inputs=[video_input, audio_input, subtitle_input, question_input], outputs=output_text, fn=recognize_emotion, cache_examples=False, run_on_click=True, ) run_btn.click( fn=recognize_emotion, inputs=[video_input, audio_input, subtitle_input, question_input, max_tokens, temp, top_p_val, do_sample_chk], outputs=output_text, api_name="recognize_emotion", ) demo.launch()