light-mer / app.py
asdfasdfqrqwer's picture
Use different actors for demo examples
5f62ce6
Raw History Blame Contribute Delete
13.2 kB
import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import spaces # MUST come before any torch/CUDA-touching import
import torch
import torchvision
import torchvision.transforms.functional as _F
import sys
import types
# Compatibility shim: torchvision >= 0.21 removed functional_tensor, which pytorchvideo still imports
if not hasattr(torchvision.transforms, 'functional_tensor'):
_mod = types.ModuleType('torchvision.transforms.functional_tensor')
for _name in dir(_F):
setattr(_mod, _name, getattr(_F, _name))
sys.modules['torchvision.transforms.functional_tensor'] = _mod
torchvision.transforms.functional_tensor = _mod
import gradio as gr
import numpy as np
import tempfile
import time
from pathlib import Path
from huggingface_hub import snapshot_download, hf_hub_download
from omegaconf import OmegaConf
import copy
import einops
# Patch config module to use HF Hub paths before any model imports
import config as _config_module
# Download pretrained models from HF Hub and patch config paths
_qwen3_06b_path = snapshot_download("Qwen/Qwen3-0.6B")
_clip_path = snapshot_download("openai/clip-vit-base-patch16")
_hubert_path = snapshot_download("TencentGameMate/chinese-hubert-base")
_config_module.PATH_TO_LLM['Qwen3_0.6B'] = _qwen3_06b_path
_config_module.PATH_TO_VISUAL['CLIP_VIT_BASE'] = _clip_path
_config_module.PATH_TO_AUDIO['HUBERT_BASE'] = _hubert_path
# Now import the model code (after config is patched)
from my_affectgpt.common.registry import registry
from my_affectgpt.models.affectgpt import AffectGPT
from my_affectgpt.models.encoder import * # registers encoders
from my_affectgpt.conversation.conversation_video import Chat
from my_affectgpt.processors.video_processor import (
AlproVideoEvalProcessor,
load_video,
)
from my_affectgpt.processors.blip_processors import Blip2ImageEvalProcessor
from my_affectgpt.models.ImageBind.data import load_audio, transform_audio
# ---- Model config matching stage2-mgrpo-qwen3-0.6b/config.yaml ----
MODEL_CFG = OmegaConf.create({
"arch": "affectgpt",
"model_type": "pretrain_vicuna",
"frozen_video_proj": False,
"frozen_video_Qformer": False,
"frozen_audio_Qformer": False,
"frozen_audio_proj": False,
"frozen_multi_Qformer": False,
"frozen_multi_llama_proj": False,
"frozen_llm": False,
"multi_fusion_type": "attention",
"video_fusion_type": "attention",
"audio_fusion_type": "attention",
"image_fusion_type": "mean",
"ckpt": "",
"ckpt_2": "",
"llama_model": "Qwen3_0.6B",
"acoustic_encoder": "HUBERT_BASE",
"visual_encoder": "CLIP_VIT_BASE",
"num_audio_query_token": 1,
"num_video_query_token": 1,
"num_multi_query_token": 1,
"num_image_query_token": 1,
"max_length": 1024,
"lora_r": 16,
"vis_processor": {
"train": {
"name": "alpro_video_eval",
"n_frms": 8,
"image_size": 224,
}
},
})
# Download the Light-MER Stage 2 checkpoint
ckpt_path = hf_hub_download(
"kevin233333/Light-MER",
"stage2-mgrpo-qwen3-0.6b/checkpoint_001500_grpo.pth",
)
MODEL_CFG.ckpt_3 = ckpt_path
print("Loading Light-MER model...")
model_cls = registry.get_model_class(MODEL_CFG.arch)
model = model_cls.from_config(MODEL_CFG)
model = model.to("cuda").eval()
chat = Chat(model, MODEL_CFG, device="cuda")
# Video processor for inference
vis_processor = AlproVideoEvalProcessor(
image_size=224, n_frms=8,
mean=(0.48145466, 0.4578275, 0.40821073),
std=(0.26862954, 0.26130258, 0.27577711),
)
# Default prompt (matching the dataset's description question)
DEFAULT_QUESTION = "Please infer the person's emotional state and provide your reasoning process."
FACE_OR_FRAME = "multiface_audio_face_text"
def video_file_has_audio(path):
"""Best-effort MP4/MOV audio-track check used before video-audio extraction."""
if path is None:
return False
suffix = Path(path).suffix.lower()
if suffix not in {".mp4", ".m4v", ".mov"}:
return True
try:
return b"soun" in Path(path).read_bytes()
except Exception:
return True
@spaces.GPU(duration=30)
def recognize_emotion(video_path, audio_path=None, subtitle="", question=DEFAULT_QUESTION,
max_new_tokens=512, temperature=0.9, top_p=0.9,
do_sample=True, progress=gr.Progress(track_tqdm=True)):
"""Recognize emotions from video/audio/text using Light-MER."""
subtitle = "" if subtitle is None else subtitle.strip()
if question is None or question.strip() == "":
question = DEFAULT_QUESTION
has_audio_upload = audio_path is not None and audio_path != ""
if video_path is None and not has_audio_upload and subtitle == "":
return "Please provide a video, audio file, or subtitle/context text."
start_time = time.time()
warnings = []
audio_source = None
sample_data = {
'frame': None, 'raw_frame': None,
'face': None, 'raw_face': None,
'audio': None, 'raw_audio': None,
'image': None, 'raw_image': None,
}
# The public Space accepts ordinary videos rather than precomputed OpenFace
# crops. Use the raw sampled frames honestly as <FrameHere>; do not label
# them as face crops.
if video_path is not None:
try:
raw_frame, _ = load_video(
video_path=video_path,
n_frms=8,
height=224,
width=224,
sampling="uniform",
return_msg=True,
)
frame = vis_processor.transform(raw_frame)
sample_data['frame'] = frame
sample_data['raw_frame'] = raw_frame
except Exception as e:
warnings.append(f"Video loading failed: {e}")
print(f"Video loading error: {e}")
if has_audio_upload:
try:
raw_audio = load_audio([audio_path], "cpu", clips_per_video=8)[0]
audio = transform_audio(raw_audio, "cpu")
sample_data['audio'] = audio
sample_data['raw_audio'] = raw_audio
audio_source = "uploaded audio"
except Exception as e:
warnings.append(f"Uploaded audio loading failed: {e}")
print(f"Audio loading error: {e}")
# If the uploaded video has an audio stream, use it automatically.
if video_path is not None and sample_data['audio'] is None:
if not video_file_has_audio(video_path):
warnings.append("The video has no audio track; running with video frames only.")
else:
try:
raw_audio = load_audio([video_path], "cpu", clips_per_video=8)[0]
audio = transform_audio(raw_audio, "cpu")
sample_data['audio'] = audio
sample_data['raw_audio'] = raw_audio
audio_source = "audio extracted from video"
except Exception as e:
warnings.append("No usable audio stream was found in the video.")
print(f"Audio extraction from video failed: {e}")
audio_hiddens, audio_llms = chat.postprocess_audio(sample_data)
frame_hiddens, frame_llms = chat.postprocess_frame(sample_data)
face_hiddens, face_llms = None, None
_, image_llms = chat.postprocess_image(sample_data)
multi_llms = None
video_hiddens_for_fusion = face_hiddens if face_hiddens is not None else frame_hiddens
if video_hiddens_for_fusion is not None and audio_hiddens is not None:
_, multi_llms = chat.postprocess_multi(video_hiddens_for_fusion, audio_hiddens)
img_list = {
'audio': audio_llms,
'frame': frame_llms,
'face': face_llms,
'image': image_llms,
'multi': multi_llms,
}
prompt_parts = ["###Human: "]
if multi_llms is not None:
prompt_parts.append("The audio and video merged info is: <Multi><MultiHere></Multi>. ")
if audio_llms is not None:
prompt_parts.append("The audio content is as follows: <Audio><AudioHere></Audio>. ")
if face_llms is not None:
prompt_parts.append("Meanwhile, we uniformly sample raw frames from the video and extract faces from these frames: <Video><FaceHere></Video>. ")
elif frame_llms is not None:
if audio_llms is None and subtitle == "":
prompt_parts.append("We uniformly sample raw frames from the video: <Video><FrameHere></Video>. ")
else:
prompt_parts.append("Meanwhile, we uniformly sample raw frames from the video: <Video><FrameHere></Video>. ")
# Keep the trained subtitle framing when text is provided, and for
# audio-video runs where the benchmark prompt includes the subtitle field.
if subtitle or (frame_llms is not None and audio_llms is not None):
prompt_parts.append(f"The subtitle of this video is: <Subtitle>{subtitle}</Subtitle>. ")
prompt_parts.append(f"Now, please answer my question based on all the provided information. {question} ###Assistant: ")
prompt = "".join(prompt_parts)
response = chat.answer_sample(
prompt=prompt,
img_list=img_list,
num_beams=1,
temperature=temperature,
do_sample=do_sample,
top_p=top_p,
max_new_tokens=max_new_tokens,
max_length=2000,
)
modalities = []
if frame_llms is not None:
modalities.append("video frames")
if face_llms is not None:
modalities.append("face crops")
if audio_llms is not None:
modalities.append(audio_source or "audio")
if subtitle:
modalities.append("subtitle/text")
if multi_llms is not None:
modalities.append("audio-video fusion")
if not modalities:
modalities.append("text only")
elapsed = time.time() - start_time
status = [f"Modalities used: {', '.join(modalities)}"]
status.extend(warnings)
result = f"{response}\n\n---\n" + "\n".join(status) + f"\nInference time: {elapsed:.2f}s"
return result
# ---- Gradio UI ----
CSS = """
#col-container { max-width: 1100px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo:
gr.Markdown("# Light-MER: Lightweight Multimodal Emotion Recognition")
gr.Markdown(
"A sub-1B parameter multimodal emotion recognition model that processes "
"video, audio, and text to recognize emotions. Upload a video, optional audio, "
"and optional subtitle/context text."
)
gr.Markdown(
"Paper: [Do We Really Need Multimodal Emotion Language Models Larger Than 1B Parameters?](https://arxiv.org/abs/2607.12787) | "
"[GitHub](https://github.com/GAIR-Lab/Light-MER) | "
"[Model](https://huggingface.co/kevin233333/Light-MER)"
)
gr.Markdown(
"Examples use full audio-visual speech clips from "
"[RAVDESS](https://zenodo.org/records/1188976) with transcript text."
)
with gr.Row():
with gr.Column(scale=2):
video_input = gr.Video(label="Video Input (sampled frames)")
audio_input = gr.Audio(label="Audio Input (optional)", type="filepath")
subtitle_input = gr.Textbox(
label="Subtitle / Context (optional)",
placeholder="Enter subtitle text if available...",
lines=2,
)
question_input = gr.Textbox(
label="Question",
value=DEFAULT_QUESTION,
lines=2,
)
run_btn = gr.Button("Recognize Emotion", variant="primary")
with gr.Column(scale=3):
output_text = gr.Textbox(
label="Emotion Recognition Result",
lines=15,
show_copy_button=True,
)
with gr.Accordion("Advanced Settings", open=False):
max_tokens = gr.Slider(
label="Max New Tokens", minimum=64, maximum=1024, value=512, step=64
)
temp = gr.Slider(
label="Temperature", minimum=0.1, maximum=2.0, value=0.9, step=0.1
)
top_p_val = gr.Slider(
label="Top P", minimum=0.1, maximum=1.0, value=0.9, step=0.05
)
do_sample_chk = gr.Checkbox(label="Do Sample", value=True)
gr.Examples(
examples=[
["examples/ravdess_happy_strong_kids_actor01.mp4", None, "Kids are talking by the door.", DEFAULT_QUESTION],
["examples/ravdess_sad_strong_kids_actor02.mp4", None, "Kids are talking by the door.", DEFAULT_QUESTION],
["examples/ravdess_angry_strong_kids_actor03.mp4", None, "Kids are talking by the door.", DEFAULT_QUESTION],
],
inputs=[video_input, audio_input, subtitle_input, question_input],
outputs=output_text,
fn=recognize_emotion,
cache_examples=False,
run_on_click=True,
)
run_btn.click(
fn=recognize_emotion,
inputs=[video_input, audio_input, subtitle_input, question_input,
max_tokens, temp, top_p_val, do_sample_chk],
outputs=output_text,
api_name="recognize_emotion",
)
demo.launch()