Spaces:
Sleeping
Sleeping
Download app.py from kevin233333/light-mer: direct link, hf CLI and curl.
- Browser
- Download file 13.2 kB
-
https://huggingface.co/spaces/kevin233333/light-mer/resolve/main/app.py
- Command line
-
hf download hf://spaces/kevin233333/light-mer/app.py
-
curl -L -o app.py https://huggingface.co/spaces/kevin233333/light-mer/resolve/main/app.py
13.2 kB
| import os | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| import spaces # MUST come before any torch/CUDA-touching import | |
| import torch | |
| import torchvision | |
| import torchvision.transforms.functional as _F | |
| import sys | |
| import types | |
| # Compatibility shim: torchvision >= 0.21 removed functional_tensor, which pytorchvideo still imports | |
| if not hasattr(torchvision.transforms, 'functional_tensor'): | |
| _mod = types.ModuleType('torchvision.transforms.functional_tensor') | |
| for _name in dir(_F): | |
| setattr(_mod, _name, getattr(_F, _name)) | |
| sys.modules['torchvision.transforms.functional_tensor'] = _mod | |
| torchvision.transforms.functional_tensor = _mod | |
| import gradio as gr | |
| import numpy as np | |
| import tempfile | |
| import time | |
| from pathlib import Path | |
| from huggingface_hub import snapshot_download, hf_hub_download | |
| from omegaconf import OmegaConf | |
| import copy | |
| import einops | |
| # Patch config module to use HF Hub paths before any model imports | |
| import config as _config_module | |
| # Download pretrained models from HF Hub and patch config paths | |
| _qwen3_06b_path = snapshot_download("Qwen/Qwen3-0.6B") | |
| _clip_path = snapshot_download("openai/clip-vit-base-patch16") | |
| _hubert_path = snapshot_download("TencentGameMate/chinese-hubert-base") | |
| _config_module.PATH_TO_LLM['Qwen3_0.6B'] = _qwen3_06b_path | |
| _config_module.PATH_TO_VISUAL['CLIP_VIT_BASE'] = _clip_path | |
| _config_module.PATH_TO_AUDIO['HUBERT_BASE'] = _hubert_path | |
| # Now import the model code (after config is patched) | |
| from my_affectgpt.common.registry import registry | |
| from my_affectgpt.models.affectgpt import AffectGPT | |
| from my_affectgpt.models.encoder import * # registers encoders | |
| from my_affectgpt.conversation.conversation_video import Chat | |
| from my_affectgpt.processors.video_processor import ( | |
| AlproVideoEvalProcessor, | |
| load_video, | |
| ) | |
| from my_affectgpt.processors.blip_processors import Blip2ImageEvalProcessor | |
| from my_affectgpt.models.ImageBind.data import load_audio, transform_audio | |
| # ---- Model config matching stage2-mgrpo-qwen3-0.6b/config.yaml ---- | |
| MODEL_CFG = OmegaConf.create({ | |
| "arch": "affectgpt", | |
| "model_type": "pretrain_vicuna", | |
| "frozen_video_proj": False, | |
| "frozen_video_Qformer": False, | |
| "frozen_audio_Qformer": False, | |
| "frozen_audio_proj": False, | |
| "frozen_multi_Qformer": False, | |
| "frozen_multi_llama_proj": False, | |
| "frozen_llm": False, | |
| "multi_fusion_type": "attention", | |
| "video_fusion_type": "attention", | |
| "audio_fusion_type": "attention", | |
| "image_fusion_type": "mean", | |
| "ckpt": "", | |
| "ckpt_2": "", | |
| "llama_model": "Qwen3_0.6B", | |
| "acoustic_encoder": "HUBERT_BASE", | |
| "visual_encoder": "CLIP_VIT_BASE", | |
| "num_audio_query_token": 1, | |
| "num_video_query_token": 1, | |
| "num_multi_query_token": 1, | |
| "num_image_query_token": 1, | |
| "max_length": 1024, | |
| "lora_r": 16, | |
| "vis_processor": { | |
| "train": { | |
| "name": "alpro_video_eval", | |
| "n_frms": 8, | |
| "image_size": 224, | |
| } | |
| }, | |
| }) | |
| # Download the Light-MER Stage 2 checkpoint | |
| ckpt_path = hf_hub_download( | |
| "kevin233333/Light-MER", | |
| "stage2-mgrpo-qwen3-0.6b/checkpoint_001500_grpo.pth", | |
| ) | |
| MODEL_CFG.ckpt_3 = ckpt_path | |
| print("Loading Light-MER model...") | |
| model_cls = registry.get_model_class(MODEL_CFG.arch) | |
| model = model_cls.from_config(MODEL_CFG) | |
| model = model.to("cuda").eval() | |
| chat = Chat(model, MODEL_CFG, device="cuda") | |
| # Video processor for inference | |
| vis_processor = AlproVideoEvalProcessor( | |
| image_size=224, n_frms=8, | |
| mean=(0.48145466, 0.4578275, 0.40821073), | |
| std=(0.26862954, 0.26130258, 0.27577711), | |
| ) | |
| # Default prompt (matching the dataset's description question) | |
| DEFAULT_QUESTION = "Please infer the person's emotional state and provide your reasoning process." | |
| FACE_OR_FRAME = "multiface_audio_face_text" | |
| def video_file_has_audio(path): | |
| """Best-effort MP4/MOV audio-track check used before video-audio extraction.""" | |
| if path is None: | |
| return False | |
| suffix = Path(path).suffix.lower() | |
| if suffix not in {".mp4", ".m4v", ".mov"}: | |
| return True | |
| try: | |
| return b"soun" in Path(path).read_bytes() | |
| except Exception: | |
| return True | |
| def recognize_emotion(video_path, audio_path=None, subtitle="", question=DEFAULT_QUESTION, | |
| max_new_tokens=512, temperature=0.9, top_p=0.9, | |
| do_sample=True, progress=gr.Progress(track_tqdm=True)): | |
| """Recognize emotions from video/audio/text using Light-MER.""" | |
| subtitle = "" if subtitle is None else subtitle.strip() | |
| if question is None or question.strip() == "": | |
| question = DEFAULT_QUESTION | |
| has_audio_upload = audio_path is not None and audio_path != "" | |
| if video_path is None and not has_audio_upload and subtitle == "": | |
| return "Please provide a video, audio file, or subtitle/context text." | |
| start_time = time.time() | |
| warnings = [] | |
| audio_source = None | |
| sample_data = { | |
| 'frame': None, 'raw_frame': None, | |
| 'face': None, 'raw_face': None, | |
| 'audio': None, 'raw_audio': None, | |
| 'image': None, 'raw_image': None, | |
| } | |
| # The public Space accepts ordinary videos rather than precomputed OpenFace | |
| # crops. Use the raw sampled frames honestly as <FrameHere>; do not label | |
| # them as face crops. | |
| if video_path is not None: | |
| try: | |
| raw_frame, _ = load_video( | |
| video_path=video_path, | |
| n_frms=8, | |
| height=224, | |
| width=224, | |
| sampling="uniform", | |
| return_msg=True, | |
| ) | |
| frame = vis_processor.transform(raw_frame) | |
| sample_data['frame'] = frame | |
| sample_data['raw_frame'] = raw_frame | |
| except Exception as e: | |
| warnings.append(f"Video loading failed: {e}") | |
| print(f"Video loading error: {e}") | |
| if has_audio_upload: | |
| try: | |
| raw_audio = load_audio([audio_path], "cpu", clips_per_video=8)[0] | |
| audio = transform_audio(raw_audio, "cpu") | |
| sample_data['audio'] = audio | |
| sample_data['raw_audio'] = raw_audio | |
| audio_source = "uploaded audio" | |
| except Exception as e: | |
| warnings.append(f"Uploaded audio loading failed: {e}") | |
| print(f"Audio loading error: {e}") | |
| # If the uploaded video has an audio stream, use it automatically. | |
| if video_path is not None and sample_data['audio'] is None: | |
| if not video_file_has_audio(video_path): | |
| warnings.append("The video has no audio track; running with video frames only.") | |
| else: | |
| try: | |
| raw_audio = load_audio([video_path], "cpu", clips_per_video=8)[0] | |
| audio = transform_audio(raw_audio, "cpu") | |
| sample_data['audio'] = audio | |
| sample_data['raw_audio'] = raw_audio | |
| audio_source = "audio extracted from video" | |
| except Exception as e: | |
| warnings.append("No usable audio stream was found in the video.") | |
| print(f"Audio extraction from video failed: {e}") | |
| audio_hiddens, audio_llms = chat.postprocess_audio(sample_data) | |
| frame_hiddens, frame_llms = chat.postprocess_frame(sample_data) | |
| face_hiddens, face_llms = None, None | |
| _, image_llms = chat.postprocess_image(sample_data) | |
| multi_llms = None | |
| video_hiddens_for_fusion = face_hiddens if face_hiddens is not None else frame_hiddens | |
| if video_hiddens_for_fusion is not None and audio_hiddens is not None: | |
| _, multi_llms = chat.postprocess_multi(video_hiddens_for_fusion, audio_hiddens) | |
| img_list = { | |
| 'audio': audio_llms, | |
| 'frame': frame_llms, | |
| 'face': face_llms, | |
| 'image': image_llms, | |
| 'multi': multi_llms, | |
| } | |
| prompt_parts = ["###Human: "] | |
| if multi_llms is not None: | |
| prompt_parts.append("The audio and video merged info is: <Multi><MultiHere></Multi>. ") | |
| if audio_llms is not None: | |
| prompt_parts.append("The audio content is as follows: <Audio><AudioHere></Audio>. ") | |
| if face_llms is not None: | |
| prompt_parts.append("Meanwhile, we uniformly sample raw frames from the video and extract faces from these frames: <Video><FaceHere></Video>. ") | |
| elif frame_llms is not None: | |
| if audio_llms is None and subtitle == "": | |
| prompt_parts.append("We uniformly sample raw frames from the video: <Video><FrameHere></Video>. ") | |
| else: | |
| prompt_parts.append("Meanwhile, we uniformly sample raw frames from the video: <Video><FrameHere></Video>. ") | |
| # Keep the trained subtitle framing when text is provided, and for | |
| # audio-video runs where the benchmark prompt includes the subtitle field. | |
| if subtitle or (frame_llms is not None and audio_llms is not None): | |
| prompt_parts.append(f"The subtitle of this video is: <Subtitle>{subtitle}</Subtitle>. ") | |
| prompt_parts.append(f"Now, please answer my question based on all the provided information. {question} ###Assistant: ") | |
| prompt = "".join(prompt_parts) | |
| response = chat.answer_sample( | |
| prompt=prompt, | |
| img_list=img_list, | |
| num_beams=1, | |
| temperature=temperature, | |
| do_sample=do_sample, | |
| top_p=top_p, | |
| max_new_tokens=max_new_tokens, | |
| max_length=2000, | |
| ) | |
| modalities = [] | |
| if frame_llms is not None: | |
| modalities.append("video frames") | |
| if face_llms is not None: | |
| modalities.append("face crops") | |
| if audio_llms is not None: | |
| modalities.append(audio_source or "audio") | |
| if subtitle: | |
| modalities.append("subtitle/text") | |
| if multi_llms is not None: | |
| modalities.append("audio-video fusion") | |
| if not modalities: | |
| modalities.append("text only") | |
| elapsed = time.time() - start_time | |
| status = [f"Modalities used: {', '.join(modalities)}"] | |
| status.extend(warnings) | |
| result = f"{response}\n\n---\n" + "\n".join(status) + f"\nInference time: {elapsed:.2f}s" | |
| return result | |
| # ---- Gradio UI ---- | |
| CSS = """ | |
| #col-container { max-width: 1100px; margin: 0 auto; } | |
| .dark .gradio-container { color: var(--body-text-color); } | |
| """ | |
| with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo: | |
| gr.Markdown("# Light-MER: Lightweight Multimodal Emotion Recognition") | |
| gr.Markdown( | |
| "A sub-1B parameter multimodal emotion recognition model that processes " | |
| "video, audio, and text to recognize emotions. Upload a video, optional audio, " | |
| "and optional subtitle/context text." | |
| ) | |
| gr.Markdown( | |
| "Paper: [Do We Really Need Multimodal Emotion Language Models Larger Than 1B Parameters?](https://arxiv.org/abs/2607.12787) | " | |
| "[GitHub](https://github.com/GAIR-Lab/Light-MER) | " | |
| "[Model](https://huggingface.co/kevin233333/Light-MER)" | |
| ) | |
| gr.Markdown( | |
| "Examples use full audio-visual speech clips from " | |
| "[RAVDESS](https://zenodo.org/records/1188976) with transcript text." | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=2): | |
| video_input = gr.Video(label="Video Input (sampled frames)") | |
| audio_input = gr.Audio(label="Audio Input (optional)", type="filepath") | |
| subtitle_input = gr.Textbox( | |
| label="Subtitle / Context (optional)", | |
| placeholder="Enter subtitle text if available...", | |
| lines=2, | |
| ) | |
| question_input = gr.Textbox( | |
| label="Question", | |
| value=DEFAULT_QUESTION, | |
| lines=2, | |
| ) | |
| run_btn = gr.Button("Recognize Emotion", variant="primary") | |
| with gr.Column(scale=3): | |
| output_text = gr.Textbox( | |
| label="Emotion Recognition Result", | |
| lines=15, | |
| show_copy_button=True, | |
| ) | |
| with gr.Accordion("Advanced Settings", open=False): | |
| max_tokens = gr.Slider( | |
| label="Max New Tokens", minimum=64, maximum=1024, value=512, step=64 | |
| ) | |
| temp = gr.Slider( | |
| label="Temperature", minimum=0.1, maximum=2.0, value=0.9, step=0.1 | |
| ) | |
| top_p_val = gr.Slider( | |
| label="Top P", minimum=0.1, maximum=1.0, value=0.9, step=0.05 | |
| ) | |
| do_sample_chk = gr.Checkbox(label="Do Sample", value=True) | |
| gr.Examples( | |
| examples=[ | |
| ["examples/ravdess_happy_strong_kids_actor01.mp4", None, "Kids are talking by the door.", DEFAULT_QUESTION], | |
| ["examples/ravdess_sad_strong_kids_actor02.mp4", None, "Kids are talking by the door.", DEFAULT_QUESTION], | |
| ["examples/ravdess_angry_strong_kids_actor03.mp4", None, "Kids are talking by the door.", DEFAULT_QUESTION], | |
| ], | |
| inputs=[video_input, audio_input, subtitle_input, question_input], | |
| outputs=output_text, | |
| fn=recognize_emotion, | |
| cache_examples=False, | |
| run_on_click=True, | |
| ) | |
| run_btn.click( | |
| fn=recognize_emotion, | |
| inputs=[video_input, audio_input, subtitle_input, question_input, | |
| max_tokens, temp, top_p_val, do_sample_chk], | |
| outputs=output_text, | |
| api_name="recognize_emotion", | |
| ) | |
| demo.launch() |