kqwyf commited on 12 days ago

Commit

2fa27bf

verified ·

1 Parent(s): ca2d737

Upload folder using huggingface_hub

Browse files

This view is limited to 50 files because it contains too many changes. See raw diff

Files changed (50) hide show

.gitattributes +10 -0
chat_template.jinja +55 -67
config.json +9 -11
configuration_bailing_moe_v3_audio.py +2 -10
generation_config.json +7 -0
model-00001-of-00002.safetensors +2 -2
model-00002-of-00002.safetensors +1 -1
model.safetensors.index.json +62 -399
modeling_bailing_moe_v3.py +34 -8
modeling_bailing_moe_v3_audio.py +34 -15
processing_bailing_moe_v3_audio.py +36 -8
processor_config.json +6 -0
talker/AudioVAE/__init__.py +0 -0
talker/AudioVAE/__pycache__/__init__.cpython-312.pyc +0 -0
talker/AudioVAE/__pycache__/configuration_audio_vae.cpython-312.pyc +0 -0
talker/AudioVAE/__pycache__/istft.cpython-312.pyc +0 -0
talker/AudioVAE/__pycache__/modeling_audio_vae.cpython-312.pyc +0 -0
talker/AudioVAE/__pycache__/vae_modules.cpython-312.pyc +0 -0
talker/AudioVAE/configuration_audio_vae.py +19 -0
talker/AudioVAE/istft.py +181 -0
talker/AudioVAE/modeling_audio_vae.py +80 -0
talker/AudioVAE/vae_modules.py +204 -0
talker/__init__.py +5 -0
talker/__pycache__/__init__.cpython-312.pyc +0 -0
talker/__pycache__/configuration_bailing_talker.cpython-312.pyc +0 -0
talker/__pycache__/inference.cpython-312.pyc +0 -0
talker/__pycache__/tts_prompt.cpython-312.pyc +0 -0
talker/campplus.onnx +3 -0
talker/config.json +35 -0
talker/configuration_bailing_talker.py +20 -0
talker/data/lid.176.LICENSE.md +10 -0
talker/data/lid.176.ftz +3 -0
talker/front/__init__.py +0 -0
talker/front/__pycache__/__init__.cpython-312.pyc +0 -0
talker/front/__pycache__/number_en.cpython-312.pyc +0 -0
talker/front/__pycache__/text_segment_cut.cpython-312.pyc +0 -0
talker/front/__pycache__/toolkit.cpython-312.pyc +0 -0
talker/front/number_en.py +343 -0
talker/front/text_segment_cut.py +388 -0
talker/front/toolkit.py +240 -0
talker/inference.py +443 -0
talker/llm/added_tokens.json +75 -0
talker/llm/config.json +27 -0
talker/llm/generation_config.json +14 -0
talker/llm/merges.txt +0 -0
talker/llm/special_tokens_map.json +137 -0
talker/llm/tokenizer.json +3 -0
talker/llm/tokenizer_config.json +619 -0
talker/llm/vocab.json +0 -0
talker/model.safetensors +3 -0

.gitattributes ADDED Viewed

	@@ -0,0 +1,10 @@

+model-00001-of-00002.safetensors filter=lfs diff=lfs merge=lfs -text
+model-00002-of-00002.safetensors filter=lfs diff=lfs merge=lfs -text
+talker/campplus.onnx filter=lfs diff=lfs merge=lfs -text
+talker/data/lid.176.ftz filter=lfs diff=lfs merge=lfs -text
+talker/llm/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+talker/model.safetensors filter=lfs diff=lfs merge=lfs -text
+talker/talker_tn/en_tn_tagger.fst filter=lfs diff=lfs merge=lfs -text
+talker/talker_tn/en_tn_verbalizer.fst filter=lfs diff=lfs merge=lfs -text
+talker/talker_tn/zh_tn_tagger.fst filter=lfs diff=lfs merge=lfs -text
+talker/vae/model.safetensors filter=lfs diff=lfs merge=lfs -text

chat_template.jinja CHANGED Viewed

@@ -1,49 +1,34 @@
-{#- Bailing V3 config_vl chat template -#}
-{#- Supports: multimodal content (image/audio/video), thinking, tool calling, add_vision_id -#}
-{#- ==================== thinking option normalization ==================== -#}
-{%- if enable_thinking is defined %}
-    {%- if enable_thinking is false %}
-        {%- set thinking_option = 'off' %}
-    {%- else %}
-        {%- set thinking_option = 'on' %}
-    {%- endif %}
-{%- elif thinking_option is not defined %}
-    {%- set thinking_option = 'off' %}
 {%- endif %}
-{#- ==================== vision / video counters ==================== -#}
-{%- set image_count = namespace(value=0) %}
-{%- set video_count = namespace(value=0) %}
 {#- ==================== render_content macro ==================== -#}
-{%- macro render_content(content, do_vision_count, is_system_content=false) %}
     {%- if content is string %}
         {{- content }}
     {%- elif content is iterable and content is not mapping %}
         {%- for item in content %}
-            {%- if item.type == 'image' or 'image' in item or 'image_url' in item %}
-                {%- if do_vision_count %}
-                    {%- set image_count.value = image_count.value + 1 %}
-                {%- endif %}
-                {%- if add_vision_id is defined and add_vision_id %}
-                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
-                {%- endif %}
-                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
-            {%- elif item.type == 'audio' or 'audio' in item or 'audio_url' in item %}
                 {{- '<|audio_start|><|audio_pad|><|audio_end|>' }}
-            {%- elif item.type == 'video' or 'video' in item %}
-                {%- if do_vision_count %}
-                    {%- set video_count.value = video_count.value + 1 %}
-                {%- endif %}
-                {%- if add_vision_id is defined and add_vision_id %}
-                    {{- 'Video ' ~ video_count.value ~ ': ' }}
-                {%- endif %}
-                {{- '<|video_start|><|video_pad|><|video_end|>' }}
-            {%- elif item.type == 'text' or 'text' in item %}
                 {{- item.text }}
             {%- else %}
-                {{- raise_exception('Unexpected item type in content list: each item must have type image/audio/video/text.') }}
             {%- endif %}
         {%- endfor %}
     {%- elif content is none or content is undefined %}
@@ -60,14 +45,17 @@
 {#- ==================== system message ==================== -#}
 {{- '<role>SYSTEM</role>' }}
 {%- if messages[0].role == 'system' %}
-    {%- set system_content = render_content(messages[0].content, false, true) %}
     {{- system_content }}
-    {%- if system_content and system_content[-1] != '\n' %}
         {{- '\n' }}
     {%- endif %}
 {%- endif %}
-{#- tools -#}
 {%- if tools %}
     {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
     {%- for tool in tools %}
@@ -76,19 +64,17 @@
     {%- endfor %}
     {{- "\n</tools>\n\nIf none of the functions can be used, point it out. If the given question lacks the parameters required by the function, also point it out.\nIf you need to use a function, for each function call, output the function name and arguments within the following XML format:\n<tool_call>{function-name}\n<arg_key>{arg-key-1}</arg_key>\n<arg_value>{arg-value-1}</arg_value>\n<arg_key>{arg-key-2}</arg_key>\n<arg_value>{arg-value-2}</arg_value>\n...\n</tool_call>\n" }}
 {%- endif %}
-{#- thinking instruction (skip if system prompt already contains it) -#}
-{%- if messages[0].role == 'system' and messages[0].content is string and ('detailed thinking on' in messages[0].content or 'detailed thinking off' in messages[0].content) %}
-    {{- '<|role_end|>' }}
-{%- else %}
-    {{- 'detailed thinking ' + thinking_option + '<|role_end|>' }}
 {%- endif %}
 {#- ==================== multi_step_tool: find last real user query ==================== -#}
 {%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
 {%- for message in messages[::-1] %}
     {%- set index = (messages|length - 1) - loop.index0 %}
     {%- if ns.multi_step_tool and message.role == "user" %}
-        {%- set user_content = render_content(message.content, false) %}
         {%- if not(user_content.startswith('<tool_response>') and user_content.endswith('</tool_response>')) %}
             {%- set ns.multi_step_tool = false %}
             {%- set ns.last_query_index = index %}
@@ -98,52 +84,52 @@
 {#- ==================== message loop ==================== -#}
 {%- for message in messages %}
-    {%- set content = render_content(message.content, true) %}
     {%- if message.role == "user" %}
         {{- '<role>HUMAN</role>' + content + '<|role_end|>' }}
     {%- elif message.role == "system" %}
         {%- if not loop.first %}
-            {{- raise_exception('System message must be at the beginning (index 0).') }}
         {%- endif %}
-        {#- system already rendered above; skip -#}
     {%- elif message.role == "assistant" %}
         {%- set reasoning_content = '' %}
-        {%- if thinking_option == 'on' and loop.index0 > ns.last_query_index %}
-            {%- if message.reasoning_content is string %}
-                {%- set reasoning_content = message.reasoning_content %}
-            {%- else %}
-                {%- if '</think>' in content %}
-                    {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
-                    {%- set content = content.split('</think>')[-1].lstrip('\n') %}
-                {%- endif %}
-            {%- endif %}
         {%- else %}
             {%- if '</think>' in content %}
                 {%- set content = content.split('</think>')[-1].lstrip('\n') %}
             {%- endif %}
         {%- endif %}
-        {%- if reasoning_content %}
-            {{- '<role>ASSISTANT</role>' + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
         {%- else %}
-            {{- '<role>ASSISTANT</role>' + content }}
         {%- endif %}
         {%- if message.tool_calls %}
             {%- for tool_call in message.tool_calls %}
                 {%- if (loop.first and content) or (not loop.first) %}
                     {{- '\n' }}
                 {%- endif %}
                 {%- if tool_call.function %}
-                    {%- set tool_call = tool_call.function %}
                 {%- endif %}
-                {{- '<tool_call>' + tool_call.name }}
-                {%- set _args = tool_call.arguments %}
                 {%- for k, v in _args.items() %}
                     {{- '\n<arg_key>' + k + '</arg_key>' }}
                     {{- '\n<arg_value>' }}
                     {%- if v is string %}
                         {{- v }}
                     {%- else %}
-                        {{- v | tojson }}
                     {%- endif %}
                     {{- '</arg_value>' }}
                 {%- endfor %}
@@ -152,6 +138,7 @@
         {%- endif %}
         {{- '<|role_end|>' }}
     {%- elif message.role == "tool" %}
         {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
             {{- '<role>OBSERVATION</role>' }}
         {%- endif %}
@@ -162,16 +149,17 @@
             {{- '<|role_end|>' }}
         {%- endif %}
     {%- else %}
-        {{- raise_exception('Unexpected message role: ' + message.role + '. Expected system/user/assistant/tool.') }}
     {%- endif %}
 {%- endfor %}
 {#- ==================== generation prompt ==================== -#}
 {%- if add_generation_prompt %}
     {{- '<role>ASSISTANT</role>' }}
-    {%- if thinking_option == 'on' %}
-        {{- '\n<think>\n' }}
-    {%- elif thinking_option == 'off' %}
-        {{- '<think>\n\n</think>\n\n' }}
     {%- endif %}
 {%- endif %}

+{#- Bailing V3 audio and text chat template -#}
+{#- Supports: audio/text content, enable_thinking, preserved thinking, tool calling -#}
+{#- ==================== thinking normalization ==================== -#}
+{%- if enable_thinking is not defined %}
+    {%- set enable_thinking = false %}
+{%- endif %}
+{%- if enable_thinking %}
+    {%- set thinking_instruction = 'detailed thinking on' %}
+{%- else %}
+    {%- set thinking_instruction = 'detailed thinking off' %}
 {%- endif %}
+{#- ==================== preserved thinking normalization ==================== -#}
+{%- if preserve_thinking is not defined %}
+    {%- set preserve_thinking = true %}
+{%- endif %}
 {#- ==================== render_content macro ==================== -#}
+{%- macro render_content(content) %}
     {%- if content is string %}
         {{- content }}
     {%- elif content is iterable and content is not mapping %}
         {%- for item in content %}
+            {%- set item_type = item.type if item.type is defined else '' %}
+            {%- if item_type == 'audio' or (not item_type and ('audio' in item or 'audio_url' in item)) %}
                 {{- '<|audio_start|><|audio_pad|><|audio_end|>' }}
+            {%- elif item_type == 'text' or (not item_type and 'text' in item) %}
                 {{- item.text }}
             {%- else %}
+                {{- raise_exception('Unexpected item type in content list: each item must have type audio/text.') }}
             {%- endif %}
         {%- endfor %}
     {%- elif content is none or content is undefined %}
 {#- ==================== system message ==================== -#}
 {{- '<role>SYSTEM</role>' }}
+{%- set has_thinking_instruction = false %}
 {%- if messages[0].role == 'system' %}
+    {%- set system_content = render_content(messages[0].content) %}
+    {%- if 'detailed thinking on' in system_content or 'detailed thinking off' in system_content %}
+        {%- set has_thinking_instruction = true %}
+    {%- endif %}
     {{- system_content }}
+    {%- if (tools or not has_thinking_instruction) and system_content and system_content[-1] != '\n' %}
         {{- '\n' }}
     {%- endif %}
 {%- endif %}
 {%- if tools %}
     {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
     {%- for tool in tools %}
     {%- endfor %}
     {{- "\n</tools>\n\nIf none of the functions can be used, point it out. If the given question lacks the parameters required by the function, also point it out.\nIf you need to use a function, for each function call, output the function name and arguments within the following XML format:\n<tool_call>{function-name}\n<arg_key>{arg-key-1}</arg_key>\n<arg_value>{arg-value-1}</arg_value>\n<arg_key>{arg-key-2}</arg_key>\n<arg_value>{arg-value-2}</arg_value>\n...\n</tool_call>\n" }}
 {%- endif %}
+{%- if not has_thinking_instruction %}
+    {{- thinking_instruction }}
 {%- endif %}
+{{- '<|role_end|>' }}
 {#- ==================== multi_step_tool: find last real user query ==================== -#}
 {%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
 {%- for message in messages[::-1] %}
     {%- set index = (messages|length - 1) - loop.index0 %}
     {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set user_content = render_content(message.content) %}
         {%- if not(user_content.startswith('<tool_response>') and user_content.endswith('</tool_response>')) %}
             {%- set ns.multi_step_tool = false %}
             {%- set ns.last_query_index = index %}
 {#- ==================== message loop ==================== -#}
 {%- for message in messages %}
     {%- if message.role == "user" %}
+        {%- set content = render_content(message.content) %}
         {{- '<role>HUMAN</role>' + content + '<|role_end|>' }}
     {%- elif message.role == "system" %}
         {%- if not loop.first %}
+            {%- set content = render_content(message.content) %}
+            {{- '<role>SYSTEM</role>' + content + '<|role_end|>' }}
         {%- endif %}
     {%- elif message.role == "assistant" %}
+        {%- set content = render_content(message.content) %}
         {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string and message.reasoning_content != '' %}
+            {%- set reasoning_content = message.reasoning_content %}
         {%- else %}
             {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
                 {%- set content = content.split('</think>')[-1].lstrip('\n') %}
             {%- endif %}
         {%- endif %}
+        {%- if preserve_thinking or loop.index0 > ns.last_query_index %}
+            {%- if reasoning_content != '' %}
+                {{- '<role>ASSISTANT</role>' + '\n<think>' + reasoning_content.strip('\n') + '</think>' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<role>ASSISTANT</role>\n<think></think>' + content }}
+            {%- endif %}
         {%- else %}
+            {{- '<role>ASSISTANT</role>\n<think></think>' + content }}
         {%- endif %}
         {%- if message.tool_calls %}
             {%- for tool_call in message.tool_calls %}
                 {%- if (loop.first and content) or (not loop.first) %}
                     {{- '\n' }}
                 {%- endif %}
+                {%- set tc = tool_call %}
                 {%- if tool_call.function %}
+                    {%- set tc = tool_call.function %}
                 {%- endif %}
+                {{- '<tool_call>' + tc.name }}
+                {%- set _args = tc.arguments %}
                 {%- for k, v in _args.items() %}
                     {{- '\n<arg_key>' + k + '</arg_key>' }}
                     {{- '\n<arg_value>' }}
                     {%- if v is string %}
                         {{- v }}
                     {%- else %}
+                        {{- v | tojson(ensure_ascii=False) }}
                     {%- endif %}
                     {{- '</arg_value>' }}
                 {%- endfor %}
         {%- endif %}
         {{- '<|role_end|>' }}
     {%- elif message.role == "tool" %}
+        {%- set content = render_content(message.content) %}
         {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
             {{- '<role>OBSERVATION</role>' }}
         {%- endif %}
             {{- '<|role_end|>' }}
         {%- endif %}
     {%- else %}
+        {%- set content = render_content(message.content) %}
+        {{- '<role>' + message.role + '</role>' + content + '<|role_end|>' }}
     {%- endif %}
 {%- endfor %}
 {#- ==================== generation prompt ==================== -#}
 {%- if add_generation_prompt %}
     {{- '<role>ASSISTANT</role>' }}
+    {%- if enable_thinking %}
+        {{- '\n<think>' }}
+    {%- else %}
+        {{- '\n<think></think>' }}
     {%- endif %}
 {%- endif %}

config.json CHANGED Viewed

@@ -13,7 +13,7 @@
       2,
       2
     ],
-    "model_type": "bailing_moe_v3_vl_audio",
     "n_audio_ctx": 3000,
     "n_audio_head": 20,
     "n_audio_layer": 32,
@@ -23,14 +23,16 @@
   "audio_token_id": 157168,
   "auto_map": {
     "AutoConfig": "configuration_bailing_moe_v3_audio.BailingMoeV3AudioConfig",
-    "AutoModelForCausalLM": "modeling_bailing_moe_v3_audio.BailingMoeV3AudioForConditionalGeneration"
   },
   "dtype": "bfloat16",
   "model_type": "bailing_moe_v3_audio",
   "norm_query_embeds": true,
   "text_config": {
-    "attention_dropout": 0,
-    "embedding_dropout": 0,
     "expert_swiglu_limit_list": null,
     "first_k_dense_replace": 1,
     "gated_attention_proj_granularity_type": "head_wise",
@@ -38,8 +40,6 @@
     "head_dim": 128,
     "hidden_act": "silu",
     "hidden_size": 1536,
-    "image_patch_token": null,
-    "image_start_token": null,
     "initializer_range": 0.02,
     "intermediate_size": 4608,
     "kda_lower_bound": -5,
@@ -47,7 +47,7 @@
     "kv_lora_rank": 512,
     "layer_group_size": 4,
     "linear_silu": true,
-    "max_position_embeddings": 8192,
     "max_window_layers": 20,
     "model_type": "",
     "moe_intermediate_size": 512,
@@ -66,7 +66,7 @@
     "num_kv_heads_for_linear_attn": 0,
     "num_nextn_predict_layers": 1,
     "num_shared_experts": 1,
-    "output_dropout": 0,
     "output_router_logits": false,
     "partial_rotary_factor": 0.5,
     "q_lora_rank": 256,
@@ -98,11 +98,9 @@
     "use_qkv_bias": false,
     "v_head_dim": 128,
     "value_norm": false,
-    "video_patch_token": 157175,
-    "video_start_token": null,
     "vocab_size": 157184
   },
   "tie_word_embeddings": false,
-  "transformers_version": "4.56.2",
   "torch_dtype": "bfloat16"
 }

       2,
       2
     ],
+    "model_type": "bailing_moe_v3_audio",
     "n_audio_ctx": 3000,
     "n_audio_head": 20,
     "n_audio_layer": 32,
   "audio_token_id": 157168,
   "auto_map": {
     "AutoConfig": "configuration_bailing_moe_v3_audio.BailingMoeV3AudioConfig",
+    "AutoModel": "modeling_bailing_moe_v3_audio.BailingMoeV3AudioForConditionalGeneration",
+    "AutoModelForCausalLM": "modeling_bailing_moe_v3_audio.BailingMoeV3AudioForConditionalGeneration",
+    "AutoProcessor": "processing_bailing_moe_v3_audio.BailingMoeV3AudioProcessor"
   },
   "dtype": "bfloat16",
   "model_type": "bailing_moe_v3_audio",
   "norm_query_embeds": true,
   "text_config": {
+    "attention_dropout": 0.0,
+    "embedding_dropout": 0.0,
     "expert_swiglu_limit_list": null,
     "first_k_dense_replace": 1,
     "gated_attention_proj_granularity_type": "head_wise",
     "head_dim": 128,
     "hidden_act": "silu",
     "hidden_size": 1536,
     "initializer_range": 0.02,
     "intermediate_size": 4608,
     "kda_lower_bound": -5,
     "kv_lora_rank": 512,
     "layer_group_size": 4,
     "linear_silu": true,
+    "max_position_embeddings": 131072,
     "max_window_layers": 20,
     "model_type": "",
     "moe_intermediate_size": 512,
     "num_kv_heads_for_linear_attn": 0,
     "num_nextn_predict_layers": 1,
     "num_shared_experts": 1,
+    "output_dropout": 0.0,
     "output_router_logits": false,
     "partial_rotary_factor": 0.5,
     "q_lora_rank": 256,
     "use_qkv_bias": false,
     "v_head_dim": 128,
     "value_norm": false,
     "vocab_size": 157184
   },
   "tie_word_embeddings": false,
+  "transformers_version": "5.14.1",
   "torch_dtype": "bfloat16"
 }

configuration_bailing_moe_v3_audio.py CHANGED Viewed

@@ -180,7 +180,6 @@ class BailingMoeV3AudioEncoderConfig(PretrainedConfig):
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
-        cls._set_token_in_kwargs(kwargs)
         config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
         if "audio_config" in config_dict:
             config_dict = config_dict["audio_config"]
@@ -195,6 +194,7 @@ class BailingMoeV3AudioConfig(PretrainedConfig):
         text_config=None,
         audio_config=None,
         audio_token_id=157168,
         tie_word_embeddings=False,
         **kwargs,
     ):
@@ -211,6 +211,7 @@ class BailingMoeV3AudioConfig(PretrainedConfig):
         self.text_config = text_config
         self.audio_config = audio_config
         self.audio_token_id = audio_token_id
         super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
@@ -218,17 +219,8 @@ class BailingMoeV3AudioConfig(PretrainedConfig):
     def from_pretrained(
         cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs
     ) -> "BailingMoeV3AudioConfig":
-        cls._set_token_in_kwargs(kwargs)
         config_dict, kwargs = cls.get_config_dict(
             pretrained_model_name_or_path, **kwargs
         )
-        config_dict = {
-            key: value
-            for key, value in config_dict.items()
-            if not (
-                key.startswith(("vision", "image_", "video_"))
-                or key == "mrope_section"
-            )
-        }
         config_dict["model_type"] = cls.model_type
         return cls.from_dict(config_dict, **kwargs)

     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
         config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
         if "audio_config" in config_dict:
             config_dict = config_dict["audio_config"]
         text_config=None,
         audio_config=None,
         audio_token_id=157168,
+        norm_query_embeds=False,
         tie_word_embeddings=False,
         **kwargs,
     ):
         self.text_config = text_config
         self.audio_config = audio_config
         self.audio_token_id = audio_token_id
+        self.norm_query_embeds = norm_query_embeds
         super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
     def from_pretrained(
         cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs
     ) -> "BailingMoeV3AudioConfig":
         config_dict, kwargs = cls.get_config_dict(
             pretrained_model_name_or_path, **kwargs
         )
         config_dict["model_type"] = cls.model_type
         return cls.from_dict(config_dict, **kwargs)

generation_config.json ADDED Viewed

	@@ -0,0 +1,7 @@

+{
+  "eos_token_id": [
+    156892,
+    156895
+  ],
+  "pad_token_id": 156892
+}

model-00001-of-00002.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3b06d2d6953ae89d05180fee7fdb32de28c0149175713d378e443cc1a941d1f3
-size 10481188035

 version https://git-lfs.github.com/spec/v1
+oid sha256:53b7663c7c52aa36e487977d6e9bb2f5666b7aaa543bd5d5dd4e96ec88cfd957
+size 9551820960

model-00002-of-00002.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8d07b7b66a6e6a70a967afb89037775bb99ac7bbfffba55a2afbbbdf5fc3dac
 size 8192122649

 version https://git-lfs.github.com/spec/v1
+oid sha256:08d9e7bd9d885a0d6608953c6bbe40dbfb67f673a0cce9382414b55a9380053a
 size 8192122649

model.safetensors.index.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 18672021088
   },
   "weight_map": {
     "model.audio.blocks.28.attn.query.weight": "model-00001-of-00002.safetensors",
@@ -8,12 +8,6 @@
     "model.layers.23.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "linear_proj_audio.2.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.20.attn.out.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.23.mlp.2.weight": "model-00001-of-00002.safetensors",
-    "model.layers.2.attention.f_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.7.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.20.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.24.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.attention.g_proj.weight": "model-00001-of-00002.safetensors",
@@ -41,12 +35,12 @@
     "model.layers.9.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.28.attn.value.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.attention.f_proj.weight": "model-00001-of-00002.safetensors",
-    "linear_proj_audio.4.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.21.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.26.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.attention.f_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.22.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.27.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
@@ -72,17 +66,17 @@
     "model.layers.16.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.attention.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attention.q_b_proj.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.29.attn.key.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.attention.g_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.23.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.conv1.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.23.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.29.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.attention.f_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.23.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.attention.q_a_proj.weight": "model-00001-of-00002.safetensors",
@@ -107,28 +101,19 @@
     "model.layers.3.attention.q_a_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.attention.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.29.attn.out.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.attention.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.23.attention.q_b_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.24.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.31.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.attention.q_a_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.f_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.25.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attention.q_a_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.patch_embed.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.0.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.5.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.10.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.12.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.14.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.17.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.20.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.25.norm1.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn.out.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn_ln.bias": "model-00001-of-00002.safetensors",
@@ -148,14 +133,6 @@
     "model.layers.22.attention.o_norm.weight": "model-00001-of-00002.safetensors",
     "model.norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.final_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.0.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.6.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.11.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.13.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.15.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.17.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.21.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.25.norm1.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn.query.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn_ln.weight": "model-00001-of-00002.safetensors",
@@ -177,14 +154,6 @@
     "model.layers.22.attention.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.hnorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attention.kv_a_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.0.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.6.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.12.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.13.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.16.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.17.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.21.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.25.norm2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.attn.out.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn.value.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.mlp.2.weight": "model-00001-of-00002.safetensors",
@@ -208,14 +177,6 @@
     "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.0.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.6.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.13.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.13.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.17.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.17.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.21.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.25.norm2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.attn.query.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.mlp.0.weight": "model-00001-of-00002.safetensors",
@@ -237,19 +198,6 @@
     "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.29.attn.query.weight": "model-00001-of-00002.safetensors",
-    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.5.attention.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.2.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.4.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.8.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.11.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.14.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.15.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.18.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.23.norm2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn.out.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn.query.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.mlp.2.weight": "model-00001-of-00002.safetensors",
@@ -270,14 +218,11 @@
     "model.layers.13.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attention.dense.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.2.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.4.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.9.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.11.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.15.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.16.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.19.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.24.attn.proj.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn.query.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.attn.out.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn.value.bias": "model-00001-of-00002.safetensors",
@@ -299,14 +244,6 @@
     "model.layers.20.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.2.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.2.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.4.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.9.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.11.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.16.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.16.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.24.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn.value.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.attn.query.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn_ln.bias": "model-00001-of-00002.safetensors",
@@ -326,14 +263,6 @@
     "model.layers.15.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.attention.dt_bias": "model-00001-of-00002.safetensors",
     "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.2.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.4.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.9.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.11.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.16.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.17.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.20.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.24.norm1.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.attn.value.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn_ln.weight": "model-00001-of-00002.safetensors",
@@ -354,14 +283,6 @@
     "model.layers.17.attention.dt_bias": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.5.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.9.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.12.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.16.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.18.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.20.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.21.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.24.norm1.weight": "model-00001-of-00002.safetensors",
     "model.audio.conv1.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn_ln.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.attn_ln.bias": "model-00001-of-00002.safetensors",
@@ -383,14 +304,6 @@
     "model.layers.21.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.5.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.9.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.12.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.16.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.19.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.20.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.22.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.24.norm2.bias": "model-00001-of-00002.safetensors",
     "model.audio.conv2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.mlp.2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.attn_ln.weight": "model-00001-of-00002.safetensors",
@@ -410,18 +323,6 @@
     "model.layers.18.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.22.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.0.mlp.0.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.13.attn.key.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.attention.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.2.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.5.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.9.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.12.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.16.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.20.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.23.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.24.norm2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.mlp_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.mlp.2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.mlp_ln.weight": "model-00001-of-00002.safetensors",
@@ -442,18 +343,12 @@
     "model.layers.20.attention.dt_bias": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attention.q_a_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.12.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.attention.v_proj.weight": "model-00001-of-00002.safetensors",
-    "linear_proj.0.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.5.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.12.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.17.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.20.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.20.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.24.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.25.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.merger.norm.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.mlp_ln.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.mlp_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.attn.key.weight": "model-00001-of-00002.safetensors",
@@ -474,14 +369,6 @@
     "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.attention.dt_bias": "model-00001-of-00002.safetensors",
     "linear_proj_audio.0.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.0.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.1.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.6.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.9.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.13.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.18.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.21.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.26.attn.proj.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.attn.value.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn_ln.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.mlp_ln.weight": "model-00001-of-00002.safetensors",
@@ -496,15 +383,6 @@
     "model.layers.22.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attention.kv_a_layernorm.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.0.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.1.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.7.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.14.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.18.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.21.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.22.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.25.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.26.norm2.bias": "model-00001-of-00002.safetensors",
     "model.audio.conv2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.mlp_ln.bias": "model-00001-of-00002.safetensors",
@@ -529,157 +407,87 @@
     "model.layers.22.attention.dt_bias": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.1.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.21.attn.proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.12.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.eh_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.23.attn.proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.13.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.attention.f_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.0.attn.qkv.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.1.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.2.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "linear_proj.2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.1.attn.qkv.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.10.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.20.attn.proj.weight": "model-00001-of-00002.safetensors",
     "linear_proj_audio.7.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.10.attn.qkv.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.10.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.attention.kv_b_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.11.attn.qkv.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.11.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.attention.kv_b_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.11.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.12.attn.qkv.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.attention.kv_b_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.12.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.13.attn.qkv.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attention.kv_b_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.24.attn.proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.13.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.attention.g_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.12.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.14.attn.qkv.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.6.attn.query.weight": "model-00001-of-00002.safetensors",
-    "model.layers.3.attention.kv_b_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.13.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.15.attn.qkv.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.attention.kv_b_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.13.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.16.attn.qkv.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attention.kv_b_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.patch_embed.proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.0.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.5.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.12.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.14.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.17.attn.qkv.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.17.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.20.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.25.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.merger.norm.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn.value.bias": "model-00001-of-00002.safetensors",
     "model.layers.12.attention.A_log": "model-00001-of-00002.safetensors",
     "model.layers.16.attention.o_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.attention.A_log": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.14.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.18.attn.qkv.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.29.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.gate.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.15.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.19.attn.qkv.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.20.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.23.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.attention.dense.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.22.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.attention.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.2.attn.qkv.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.15.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.0.attn.query.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.3.attn.out.weight": "model-00001-of-00002.safetensors",
-    "model.layers.21.mlp.gate.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.16.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.20.attn.qkv.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.gate.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.16.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.21.attn.qkv.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp.gate.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.17.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.22.attn.qkv.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.30.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.gate.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.17.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.23.attn.qkv.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.30.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.gate.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.18.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.24.attn.qkv.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.30.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.gate.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.25.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.1.mlp.2.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.13.attn.value.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.attention.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.18.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.25.attn.qkv.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.10.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.30.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.gate.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.19.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.26.attn.qkv.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.10.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.31.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.gate.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.3.attn.qkv.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.19.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.10.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.31.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.gate.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.2.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.4.attn.qkv.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.10.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.31.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.gate.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.2.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.5.attn.qkv.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.11.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.31.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.mlp.gate.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.1.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.6.attn.qkv.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.7.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.14.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.18.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.20.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.22.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.22.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.26.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.26.attn.qkv.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn_ln.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.mlp_ln.weight": "model-00001-of-00002.safetensors",
@@ -706,20 +514,6 @@
     "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.attention.q_conv1d.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attention.g_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.26.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.10.mlp.0.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.14.attn.key.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.attention.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.3.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.3.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.7.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.7.attn.qkv.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.10.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.14.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.19.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.20.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.22.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.23.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.mlp.2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.attn.value.bias": "model-00001-of-00002.safetensors",
@@ -748,16 +542,9 @@
     "model.layers.16.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
     "model.layers.20.attention.b_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.attention.v_conv1d.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.3.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.4.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.7.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.8.attn.qkv.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.10.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.14.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.19.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.21.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.22.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.24.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.mlp_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn.query.weight": "model-00001-of-00002.safetensors",
@@ -786,16 +573,6 @@
     "model.layers.17.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
     "model.layers.21.attention.b_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.attention.k_conv1d.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.3.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.5.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.7.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.9.attn.qkv.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.10.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.14.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.19.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.21.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.22.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.25.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.mlp_ln.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.mlp.0.bias": "model-00001-of-00002.safetensors",
@@ -827,17 +604,6 @@
     "model.layers.20.attention.A_log": "model-00001-of-00002.safetensors",
     "model.layers.21.attention.q_conv1d.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.b_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.pos_embed.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.1.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.1.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.6.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.9.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.13.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.13.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.18.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.21.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.22.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.26.norm1.bias": "model-00001-of-00002.safetensors",
     "model.audio.positional_embedding": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn.query.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.mlp.0.bias": "model-00001-of-00002.safetensors",
@@ -862,131 +628,67 @@
     "model.layers.20.attention.k_conv1d.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attention.q_a_layernorm.weight": "model-00001-of-00002.safetensors",
     "linear_proj_audio.2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.22.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.attention.f_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.attention.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.23.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.attention.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.3.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.10.mlp.2.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.14.attn.out.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.attention.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.23.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.8.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.attention.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attention.f_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.24.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.8.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.attention.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attention.g_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.24.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.8.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.attention.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attention.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.25.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.8.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.attention.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attention.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.25.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attention.f_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attention.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.26.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attention.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.4.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.11.mlp.0.weight": "model-00001-of-00002.safetensors",
-    "model.audio.blocks.14.attn.query.weight": "model-00001-of-00002.safetensors",
-    "model.layers.13.attention.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.26.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attention.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.attention.dense.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.3.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attention.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.attention.f_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.0.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.3.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attention.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.attention.g_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.1.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.4.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attention.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.attention.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.4.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.10.attn.proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.attention.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.attention.f_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.5.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.11.attn.proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.attention.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.attention.g_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.5.attn.proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.11.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.14.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.attention.f_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.5.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.12.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.8.attention.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.attention.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.6.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.13.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.attention.f_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.attention.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.6.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.14.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.attention.g_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.attention.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.7.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.15.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.attention.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.10.attention.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.7.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.16.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.attention.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.11.attention.dense.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.8.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.17.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.attention.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.attention.f_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.6.attn.proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.12.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.15.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.attention.g_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.8.mlp.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.18.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.9.attention.o_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.attention.g_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.9.mlp.linear_fc1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.19.attn.proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.12.attention.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.layers.24.attention.dense.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.7.attn.proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.12.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.15.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.attention.k_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.8.attn.proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.13.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.15.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.attention.q_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.9.attn.proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.13.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.15.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.attention.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.0.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.2.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.3.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.4.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.8.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.11.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.15.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.23.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.merger.linear_fc2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.mlp_ln.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn.query.bias": "model-00001-of-00002.safetensors",
@@ -1013,18 +715,7 @@
     "model.layers.21.attention.v_conv1d.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.enorm.weight": "model-00001-of-00002.safetensors",
-    "linear_proj.2.bias": "model-00001-of-00002.safetensors",
     "linear_proj_audio.7.bias": "model-00001-of-00002.safetensors",
-    "lm_head.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.0.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.3.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.4.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.6.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.8.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.10.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.15.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.19.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.23.attn.proj.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.mlp.2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn.out.bias": "model-00001-of-00002.safetensors",
@@ -1053,15 +744,9 @@
     "model.layers.19.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.A_log": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.k_conv1d.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.1.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.3.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.5.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.7.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.8.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.10.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.15.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.19.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.23.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.mlp_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn.query.bias": "model-00001-of-00002.safetensors",
@@ -1090,15 +775,6 @@
     "model.layers.19.attention.q_a_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.attention.kv_a_proj_with_mqa.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.q_conv1d.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.3.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.6.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.8.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.8.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.10.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.10.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.15.norm1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.19.norm2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.23.norm1.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.mlp_ln.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.8.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn.value.bias": "model-00001-of-00002.safetensors",
@@ -1127,15 +803,6 @@
     "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.v_conv1d.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attention.kv_a_proj_with_mqa.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.2.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.4.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.7.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.8.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.9.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.11.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.11.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.15.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.23.norm1.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn.out.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.mlp.0.bias": "model-00001-of-00002.safetensors",
@@ -1164,14 +831,8 @@
     "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.attention.o_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.1.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.7.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.14.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.18.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.22.attn.proj.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.26.norm1.weight": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.26.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.merger.linear_fc1.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn.value.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.mlp.2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.mlp_ln.weight": "model-00001-of-00002.safetensors",
@@ -1201,17 +862,8 @@
     "model.layers.17.attention.b_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.attention.o_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
-    "linear_proj.0.bias": "model-00001-of-00002.safetensors",
     "linear_proj_audio.4.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.0.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.1.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.6.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.8.mlp.linear_fc1.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.12.attn.qkv.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.13.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.18.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.21.norm2.bias": "model-00001-of-00002.safetensors",
-    "model.visual.blocks.26.mlp.linear_fc2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn.out.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.attn_ln.bias": "model-00001-of-00002.safetensors",
@@ -1243,7 +895,8 @@
     "model.layers.23.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
     "model.layers.24.attention.kv_a_proj_with_mqa.weight": "model-00001-of-00002.safetensors",
     "linear_proj_audio.0.bias": "model-00001-of-00002.safetensors",
-    "model.word_embeddings.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.17.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.17.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
@@ -1258,6 +911,8 @@
     "model.layers.1.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.attention.q_b_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.18.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.19.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
@@ -1268,13 +923,13 @@
     "model.audio.blocks.18.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.19.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.attention.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.19.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.attention.v_proj.weight": "model-00001-of-00002.safetensors",
-    "model.visual.merger.linear_fc2.weight": "model-00001-of-00002.safetensors",
-    "model.visual.merger.linear_fc1.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.19.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
@@ -1285,11 +940,17 @@
     "model.audio.blocks.19.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.20.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.attention.f_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.19.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.20.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.21.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
@@ -1305,6 +966,8 @@
     "model.audio.blocks.22.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.attention.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
@@ -10522,4 +10185,4 @@
     "model.layers.24.mlp.experts.127.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.24.mlp.experts.127.down_proj.weight": "model-00002-of-00002.safetensors"
   }
-}

 {
   "metadata": {
+    "total_size": 17742692480
   },
   "weight_map": {
     "model.audio.blocks.28.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "linear_proj_audio.2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.20.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.24.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.21.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.26.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.attention.f_proj.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.28.attn.value.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.attention.f_proj.weight": "model-00001-of-00002.safetensors",
+    "linear_proj_audio.4.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.22.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.27.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.attention.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attention.q_b_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.conv1.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.23.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.29.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.attention.f_proj.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.29.attn.key.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.attention.g_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.23.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.attention.q_a_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.attention.q_a_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.attention.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.24.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.31.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.attention.q_a_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.f_proj.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.29.attn.out.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.23.attention.q_b_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.25.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attention.q_a_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn.out.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn_ln.bias": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.o_norm.weight": "model-00001-of-00002.safetensors",
     "model.norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.final_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn.query.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn_ln.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.hnorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attention.kv_a_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.attn.out.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn.value.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.attn.query.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn.out.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn.query.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.29.attn.query.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.attention.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn.query.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.attn.out.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn.value.bias": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn.value.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.attn.query.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn_ln.bias": "model-00001-of-00002.safetensors",
     "model.layers.15.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.16.attention.dt_bias": "model-00001-of-00002.safetensors",
     "model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.attn.value.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn_ln.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.attention.dt_bias": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.audio.conv1.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn_ln.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.attn_ln.bias": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.audio.conv2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.mlp.2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.attn_ln.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.mlp_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.mlp.2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.mlp_ln.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.attention.dt_bias": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attention.q_a_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.0.mlp.0.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.13.attn.key.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.attention.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.12.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.attention.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.mlp_ln.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.mlp_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.attention.dt_bias": "model-00001-of-00002.safetensors",
     "linear_proj_audio.0.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.attn.value.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn_ln.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.mlp_ln.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attention.kv_a_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.audio.conv2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.mlp_ln.bias": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.dt_bias": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.12.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.eh_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.13.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.attention.f_proj.weight": "model-00001-of-00002.safetensors",
     "linear_proj_audio.7.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.attention.kv_b_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.attention.kv_b_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.attention.kv_b_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attention.kv_b_proj.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.6.attn.query.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.attention.kv_b_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.13.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.13.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.attention.kv_b_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attention.kv_b_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn.value.bias": "model-00001-of-00002.safetensors",
     "model.layers.12.attention.A_log": "model-00001-of-00002.safetensors",
     "model.layers.16.attention.o_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.attention.A_log": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.29.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.2.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.gate.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.0.attn.query.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.3.attn.out.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.20.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.23.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.shared_experts.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.attention.dense.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.22.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.12.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.attention.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.30.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.30.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.30.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.10.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.30.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.gate.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.1.mlp.2.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.13.attn.value.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.attention.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.10.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.31.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.10.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.31.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.10.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.31.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.9.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.11.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.31.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.mlp.gate.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn_ln.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.mlp_ln.weight": "model-00001-of-00002.safetensors",
     "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.attention.q_conv1d.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.mlp.2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.attn.value.bias": "model-00001-of-00002.safetensors",
     "model.layers.16.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
     "model.layers.20.attention.b_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.attention.v_conv1d.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.10.mlp.0.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.14.attn.key.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.attention.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.mlp_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
     "model.layers.21.attention.b_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.attention.k_conv1d.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.mlp_ln.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.layers.20.attention.A_log": "model-00001-of-00002.safetensors",
     "model.layers.21.attention.q_conv1d.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.b_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.positional_embedding": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn.query.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.layers.20.attention.k_conv1d.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attention.q_a_layernorm.weight": "model-00001-of-00002.safetensors",
     "linear_proj_audio.2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.attention.f_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.attention.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.attention.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.8.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.attention.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attention.f_proj.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.10.mlp.2.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.14.attn.out.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.attention.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.8.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.attention.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.8.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.attention.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attention.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.8.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.attention.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attention.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attention.f_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attention.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.attention.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attention.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.7.attention.dense.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.11.mlp.0.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.14.attn.query.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.attention.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attention.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.attention.f_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attention.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.attention.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.attention.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.attention.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.attention.f_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.8.attention.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.11.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.14.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.attention.f_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.12.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.15.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.12.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.15.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.attention.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.13.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.15.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.attention.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.13.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.15.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.layers.14.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.attention.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.attention.k_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.mlp_ln.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.attn.query.bias": "model-00001-of-00002.safetensors",
     "model.layers.21.attention.v_conv1d.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.enorm.weight": "model-00001-of-00002.safetensors",
     "linear_proj_audio.7.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.mlp.2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn.out.bias": "model-00001-of-00002.safetensors",
     "model.layers.19.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.A_log": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.k_conv1d.weight": "model-00001-of-00002.safetensors",
+    "model.word_embeddings.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.attention.f_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.attention.q_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.mlp_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.7.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn.query.bias": "model-00001-of-00002.safetensors",
     "model.layers.19.attention.q_a_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.19.attention.kv_a_proj_with_mqa.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.q_conv1d.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.5.mlp_ln.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.8.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn.value.bias": "model-00001-of-00002.safetensors",
     "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.22.attention.v_conv1d.weight": "model-00001-of-00002.safetensors",
     "model.layers.23.attention.kv_a_proj_with_mqa.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.6.attn.out.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.attn_ln.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.9.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.attention.o_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.20.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.attention.g_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.attention.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn.value.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.mlp.2.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.4.mlp_ln.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.attention.b_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.attention.o_norm.weight": "model-00001-of-00002.safetensors",
     "model.layers.21.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
     "linear_proj_audio.4.bias": "model-00001-of-00002.safetensors",
+    "lm_head.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.0.mlp.0.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.1.attn.out.bias": "model-00001-of-00002.safetensors",
     "model.audio.blocks.3.attn_ln.bias": "model-00001-of-00002.safetensors",
     "model.layers.23.mlp.gate.expert_bias": "model-00001-of-00002.safetensors",
     "model.layers.24.attention.kv_a_proj_with_mqa.weight": "model-00001-of-00002.safetensors",
     "linear_proj_audio.0.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.attention.o_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.17.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.17.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.3.attention.q_b_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.attention.g_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.attention.dense.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.18.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.19.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.18.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.19.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.attention.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.attention.f_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.19.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.layers.4.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.10.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.17.attention.v_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.19.attn.out.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.19.attn.query.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.20.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.attention.f_proj.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.20.attn.out.weight": "model-00001-of-00002.safetensors",
+    "model.audio.blocks.23.mlp.2.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.attention.f_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.19.attn.value.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.20.mlp.2.weight": "model-00001-of-00002.safetensors",
     "model.layers.5.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.11.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.attention.g_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.attention.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.attention.g_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.2.attn.key.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.21.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.shared_experts.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.audio.blocks.22.mlp.0.weight": "model-00001-of-00002.safetensors",
     "model.layers.6.mlp.shared_experts.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.18.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.attention.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.24.attention.dense.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.1.mlp.experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
     "model.layers.24.mlp.experts.127.up_proj.weight": "model-00002-of-00002.safetensors",
     "model.layers.24.mlp.experts.127.down_proj.weight": "model-00002-of-00002.safetensors"
   }
+}

modeling_bailing_moe_v3.py CHANGED Viewed

@@ -38,7 +38,7 @@ from transformers.modeling_attn_mask_utils import (
 )
 from transformers.modeling_outputs import MoeModelOutputWithPast
 from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
-from transformers.modeling_utils import PreTrainedModel
 from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS, is_torch_greater_or_equal_than_1_13
 from transformers.utils import (
     add_start_docstrings,
@@ -46,8 +46,12 @@ from transformers.utils import (
     logging,
     replace_return_docstrings,
 )
-from transformers.utils.import_utils import is_torch_fx_available
-from configuration_bailing_moe_v3 import BailingMoeV3Config
 from transformers.generation.utils import GenerationMixin
 from dataclasses import dataclass
 from transformers.utils import ModelOutput
@@ -246,6 +250,15 @@ class BailingMoeV3GroupRMSNorm(nn.Module):
 ALL_LAYERNORM_LAYERS.append(BailingMoeV3RMSNorm)
 class BailingMoeV3RotaryEmbedding(nn.Module):
     def __init__(self, config: BailingMoeV3Config, device=None):
         super().__init__()
@@ -260,7 +273,11 @@ class BailingMoeV3RotaryEmbedding(nn.Module):
         self.config = deepcopy(config)
         self.config.head_dim = config.qk_rope_head_dim
         self.config.partial_rotary_factor = 1.0
-        self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
         inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
         self.register_buffer("inv_freq", inv_freq, persistent=False)
@@ -672,8 +689,8 @@ class BailingMoeV3MultiLatentAttention(nn.Module):
         self.scaling = self.qk_head_dim ** (-0.5)
         if self.config.rope_scaling is not None:
             mscale_all_dim = self.config.rope_scaling.get("mscale_all_dim", 0)
-            scaling_factor = self.config.rope_scaling["factor"]
             if mscale_all_dim:
                 mscale = yarn_get_mscale(scaling_factor, mscale_all_dim)
                 self.scaling = self.scaling * mscale * mscale
@@ -727,6 +744,10 @@ class BailingMoeV3MultiLatentAttention(nn.Module):
             value_states = F.pad(value_states, [0, self.qk_head_dim - self.v_head_dim])
         attention_interface: Callable = eager_attention_forward
         attn_output, attn_weights = attention_interface(
             self,
@@ -1345,8 +1366,14 @@ class BailingMoeV3Model(BailingMoeV3PreTrainedModel):
         if inputs_embeds is None:
             inputs_embeds = self.word_embeddings(input_ids)
         if cache_position is None:
-            past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
             cache_position: torch.Tensor = torch.arange(
                 past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
             )
@@ -1354,7 +1381,6 @@ class BailingMoeV3Model(BailingMoeV3PreTrainedModel):
         if position_ids is None:
             position_ids = cache_position.unsqueeze(0)
-        softmax_attention_layer_id = self.config.layer_group_size - 1
         past_seen_tokens = (
             past_key_values.get_seq_length(layer_idx=softmax_attention_layer_id) if past_key_values is not None else 0
         )
@@ -1515,7 +1541,7 @@ class BailingMoeV3Model(BailingMoeV3PreTrainedModel):
 class BailingMoeV3ForCausalLM(BailingMoeV3PreTrainedModel, GenerationMixin):
-    _tied_weights_keys = ["lm_head.weight"]
     def __init__(self, config: BailingMoeV3Config):
         super().__init__(config)

 )
 from transformers.modeling_outputs import MoeModelOutputWithPast
 from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
 from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS, is_torch_greater_or_equal_than_1_13
 from transformers.utils import (
     add_start_docstrings,
     logging,
     replace_return_docstrings,
 )
+try:
+    from transformers.utils.import_utils import is_torch_fx_available
+except ImportError:
+    def is_torch_fx_available():
+        return False
+from .configuration_bailing_moe_v3 import BailingMoeV3Config
 from transformers.generation.utils import GenerationMixin
 from dataclasses import dataclass
 from transformers.utils import ModelOutput
 ALL_LAYERNORM_LAYERS.append(BailingMoeV3RMSNorm)
+def _compute_default_rope_parameters(config, device=None, seq_len=None):
+    # Transformers 5 no longer includes the unscaled initializer in its registry.
+    dim = int(config.head_dim * getattr(config, "partial_rotary_factor", 1.0))
+    inv_freq = 1.0 / (
+        config.rope_theta ** (torch.arange(0, dim, 2, device=device).float() / dim)
+    )
+    return inv_freq, 1.0
 class BailingMoeV3RotaryEmbedding(nn.Module):
     def __init__(self, config: BailingMoeV3Config, device=None):
         super().__init__()
         self.config = deepcopy(config)
         self.config.head_dim = config.qk_rope_head_dim
         self.config.partial_rotary_factor = 1.0
+        self.rope_init_fn = (
+            _compute_default_rope_parameters
+            if self.rope_type == "default"
+            else ROPE_INIT_FUNCTIONS[self.rope_type]
+        )
         inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device)
         self.register_buffer("inv_freq", inv_freq, persistent=False)
         self.scaling = self.qk_head_dim ** (-0.5)
         if self.config.rope_scaling is not None:
             mscale_all_dim = self.config.rope_scaling.get("mscale_all_dim", 0)
             if mscale_all_dim:
+                scaling_factor = self.config.rope_scaling["factor"]
                 mscale = yarn_get_mscale(scaling_factor, mscale_all_dim)
                 self.scaling = self.scaling * mscale * mscale
             value_states = F.pad(value_states, [0, self.qk_head_dim - self.v_head_dim])
         attention_interface: Callable = eager_attention_forward
+        if self.config._attn_implementation != "eager":
+            attention_interface = ALL_ATTENTION_FUNCTIONS.get(
+                self.config._attn_implementation, eager_attention_forward
+            )
         attn_output, attn_weights = attention_interface(
             self,
         if inputs_embeds is None:
             inputs_embeds = self.word_embeddings(input_ids)
+        # Recurrent KDA states do not encode sequence length. Use the first
+        # softmax attention cache for positions as well as the causal mask.
+        softmax_attention_layer_id = min(self.config.layer_group_size - 1, self.config.num_hidden_layers - 1)
         if cache_position is None:
+            past_seen_tokens = (
+                past_key_values.get_seq_length(layer_idx=softmax_attention_layer_id)
+                if past_key_values is not None else 0
+            )
             cache_position: torch.Tensor = torch.arange(
                 past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
             )
         if position_ids is None:
             position_ids = cache_position.unsqueeze(0)
         past_seen_tokens = (
             past_key_values.get_seq_length(layer_idx=softmax_attention_layer_id) if past_key_values is not None else 0
         )
 class BailingMoeV3ForCausalLM(BailingMoeV3PreTrainedModel, GenerationMixin):
+    _tied_weights_keys = {"lm_head.weight": "model.word_embeddings.weight"}
     def __init__(self, config: BailingMoeV3Config):
         super().__init__(config)

modeling_bailing_moe_v3_audio.py CHANGED Viewed

@@ -19,6 +19,7 @@
 # limitations under the License.
 """PyTorch BailingMoE V3 Audio model."""
 from typing import Optional, Union
 import torch
@@ -32,7 +33,7 @@ try:
 except ImportError:
     def is_torch_fx_available():
         return False
-from configuration_bailing_moe_v3_audio import (
     BailingMoeV3AudioConfig,
     BailingMoeV3AudioEncoderConfig,
 )
@@ -118,7 +119,7 @@ class WhisperEncoderForBailing(nn.Module):
         )
-from modeling_bailing_moe_v3 import BailingMoeV3Model
 @dataclass
@@ -158,7 +159,7 @@ class BailingMoeV3AudioPreTrainedModel(PreTrainedModel):
 class BailingMoeV3AudioForConditionalGeneration(BailingMoeV3AudioPreTrainedModel, GenerationMixin):
-    _tied_weights_keys = ["lm_head.weight"]
     def __init__(self, config: BailingMoeV3AudioConfig, **kwargs):
         super().__init__(config, **kwargs)
@@ -222,18 +223,29 @@ class BailingMoeV3AudioForConditionalGeneration(BailingMoeV3AudioPreTrainedModel
         input_lengths = torch.tensor([chunk.shape[0] for chunk in chunks], dtype=torch.long, device=audio_feats.device)
         chunk_batch = torch.nn.utils.rnn.pad_sequence(chunks, batch_first=True)
-        encoded, enc_lengths = self.model.audio(chunk_batch, input_lengths)[:2]
-        projected = self.linear_proj_audio(encoded.transpose(-1, -2))
-        proj_lens = enc_lengths
-        for module in self.linear_proj_audio:
-            if isinstance(module, nn.Conv1d):
-                proj_lens = (proj_lens + 2 * module.padding[0] - module.kernel_size[0]) // module.stride[0] + 1
-        audio_embeds = torch.cat(
-            [projected[i, :proj_lens[i]] for i in range(projected.shape[0])], dim=0
-        ).unsqueeze(0)
-        audio_embeds = F.normalize(audio_embeds, dim=2)
         return audio_embeds
     def prepare_inputs_for_generation(
@@ -255,6 +267,13 @@ class BailingMoeV3AudioForConditionalGeneration(BailingMoeV3AudioPreTrainedModel
             and cache_position.numel() > 0
             and cache_position[0].item() > 0
         )
         if is_decode_step:
             model_inputs.pop("audio_feats", None)
             model_inputs.pop("audio_feats_lengths", None)

 # limitations under the License.
 """PyTorch BailingMoE V3 Audio model."""
+import os
 from typing import Optional, Union
 import torch
 except ImportError:
     def is_torch_fx_available():
         return False
+from .configuration_bailing_moe_v3_audio import (
     BailingMoeV3AudioConfig,
     BailingMoeV3AudioEncoderConfig,
 )
         )
+from .modeling_bailing_moe_v3 import BailingMoeV3Model
 @dataclass
 class BailingMoeV3AudioForConditionalGeneration(BailingMoeV3AudioPreTrainedModel, GenerationMixin):
+    _tied_weights_keys = {"lm_head.weight": "model.word_embeddings.weight"}
     def __init__(self, config: BailingMoeV3AudioConfig, **kwargs):
         super().__init__(config, **kwargs)
         input_lengths = torch.tensor([chunk.shape[0] for chunk in chunks], dtype=torch.long, device=audio_feats.device)
         chunk_batch = torch.nn.utils.rnn.pad_sequence(chunks, batch_first=True)
+        # Limit Whisper activations for long recordings. An unset value keeps
+        # the original single-batch path; each chunk has independent context.
+        micro_batch = int(os.environ.get("HF_SFT_WHISPER_MICRO_BATCH", "0"))
+        if micro_batch < 0:
+            raise ValueError("HF_SFT_WHISPER_MICRO_BATCH must be nonnegative")
+        micro_batch = micro_batch or len(chunks)
+        projected_chunks = []
+        for start in range(0, len(chunks), micro_batch):
+            end = start + micro_batch
+            encoded, enc_lengths = self.model.audio(
+                chunk_batch[start:end], input_lengths[start:end]
+            )[:2]
+            projected = self.linear_proj_audio(encoded.transpose(-1, -2))
+            proj_lens = enc_lengths
+            for module in self.linear_proj_audio:
+                if isinstance(module, nn.Conv1d):
+                    proj_lens = (proj_lens + 2 * module.padding[0] - module.kernel_size[0]) // module.stride[0] + 1
+            projected_chunks.extend(
+                projected[i, :proj_lens[i]] for i in range(projected.shape[0])
+            )
+        audio_embeds = torch.cat(projected_chunks, dim=0).unsqueeze(0)
+        if self.config.norm_query_embeds:
+            audio_embeds = F.normalize(audio_embeds, dim=2)
         return audio_embeds
     def prepare_inputs_for_generation(
             and cache_position.numel() > 0
             and cache_position[0].item() > 0
         )
+        # Transformers 5 slices cached decoding with next_sequence_length
+        # instead of passing cache_position to remote models.
+        is_decode_step = is_decode_step or (
+            past_key_values is not None
+            and kwargs.get("next_sequence_length") is not None
+            and not kwargs.get("is_first_iteration", False)
+        )
         if is_decode_step:
             model_inputs.pop("audio_feats", None)
             model_inputs.pop("audio_feats_lengths", None)

processing_bailing_moe_v3_audio.py CHANGED Viewed

@@ -4,11 +4,12 @@ from pathlib import Path
 import torch
 import torchaudio
 import whisper
 from transformers import AutoTokenizer, BatchFeature
 from transformers.processing_utils import ProcessorMixin
-from configuration_bailing_moe_v3_audio import BailingMoeV3AudioConfig
 class BailingMoeV3AudioProcessor(ProcessorMixin):
@@ -34,7 +35,8 @@ class BailingMoeV3AudioProcessor(ProcessorMixin):
     def _load_audio(self, audio):
         if isinstance(audio, (str, Path)):
-            waveform, sample_rate = torchaudio.load(str(audio), backend="soundfile")
         elif isinstance(audio, tuple) and len(audio) == 2:
             waveform, sample_rate = audio
             waveform = torch.as_tensor(waveform)
@@ -67,6 +69,19 @@ class BailingMoeV3AudioProcessor(ProcessorMixin):
         return total
     def __call__(self, audio, text=None, return_tensors="pt"):
         waveform = self._load_audio(audio)
         n_mels = getattr(self.audio_config, "n_mels", 128)
         mel = whisper.log_mel_spectrogram(waveform, n_mels=n_mels)
@@ -76,12 +91,25 @@ class BailingMoeV3AudioProcessor(ProcessorMixin):
         audio_start = "<|audio_start|>"
         audio_token = "<|audio_pad|>"
         audio_end = "<|audio_end|>"
-        prompt = (
-            audio_start
-            + audio_token * self._num_audio_tokens(mel_length)
-            + audio_end
-            + (text or self.ASR_PROMPT)
-        )
         inputs = self.tokenizer(prompt, return_tensors=return_tensors)
         inputs.pop("token_type_ids", None)
         inputs["audio_feats"] = mel

 import torch
 import torchaudio
+import soundfile as sf
 import whisper
 from transformers import AutoTokenizer, BatchFeature
 from transformers.processing_utils import ProcessorMixin
+from .configuration_bailing_moe_v3_audio import BailingMoeV3AudioConfig
 class BailingMoeV3AudioProcessor(ProcessorMixin):
     def _load_audio(self, audio):
         if isinstance(audio, (str, Path)):
+            samples, sample_rate = sf.read(str(audio), dtype="float32", always_2d=True)
+            waveform = torch.from_numpy(samples.T.copy())
         elif isinstance(audio, tuple) and len(audio) == 2:
             waveform, sample_rate = audio
             waveform = torch.as_tensor(waveform)
         return total
     def __call__(self, audio, text=None, return_tensors="pt"):
+        messages = None
+        if isinstance(audio, list) and audio and isinstance(audio[0], dict):
+            messages = audio
+            audio_items = [
+                item["audio"]
+                for message in messages
+                if isinstance(message.get("content"), list)
+                for item in message["content"]
+                if item.get("type") == "audio"
+            ]
+            if len(audio_items) != 1:
+                raise ValueError("exactly one audio item is required")
+            audio = audio_items[0]
         waveform = self._load_audio(audio)
         n_mels = getattr(self.audio_config, "n_mels", 128)
         mel = whisper.log_mel_spectrogram(waveform, n_mels=n_mels)
         audio_start = "<|audio_start|>"
         audio_token = "<|audio_pad|>"
         audio_end = "<|audio_end|>"
+        if messages is None:
+            prompt = (
+                audio_start
+                + audio_token * self._num_audio_tokens(mel_length)
+                + audio_end
+                + (text or self.ASR_PROMPT)
+            )
+        else:
+            prompt = self.tokenizer.apply_chat_template(
+                messages, tokenize=False, add_generation_prompt=True,
+                enable_thinking=False,
+            )
+            placeholder = audio_start + audio_token + audio_end
+            if prompt.count(placeholder) != 1:
+                raise ValueError("chat prompt must contain exactly one audio placeholder")
+            prompt = prompt.replace(
+                placeholder,
+                audio_start + audio_token * self._num_audio_tokens(mel_length) + audio_end,
+            )
         inputs = self.tokenizer(prompt, return_tensors=return_tensors)
         inputs.pop("token_type_ids", None)
         inputs["audio_feats"] = mel

processor_config.json ADDED Viewed

	@@ -0,0 +1,6 @@

+{
+  "processor_class": "BailingMoeV3AudioProcessor",
+  "auto_map": {
+    "AutoProcessor": "processing_bailing_moe_v3_audio.BailingMoeV3AudioProcessor"
+  }
+}

talker/AudioVAE/__init__.py ADDED Viewed

File without changes

talker/AudioVAE/__pycache__/__init__.cpython-312.pyc ADDED Viewed

Binary file (183 Bytes). View file

talker/AudioVAE/__pycache__/configuration_audio_vae.cpython-312.pyc ADDED Viewed

Binary file (998 Bytes). View file

talker/AudioVAE/__pycache__/istft.cpython-312.pyc ADDED Viewed

Binary file (9.54 kB). View file

talker/AudioVAE/__pycache__/modeling_audio_vae.cpython-312.pyc ADDED Viewed

Binary file (5.05 kB). View file

talker/AudioVAE/__pycache__/vae_modules.cpython-312.pyc ADDED Viewed

Binary file (11.2 kB). View file

talker/AudioVAE/configuration_audio_vae.py ADDED Viewed

	@@ -0,0 +1,19 @@

+from transformers import PretrainedConfig
+class AudioVAEconfig(PretrainedConfig):
+    def __init__(
+        self,
+        sample_rate: int=16000,
+        enc_kwargs: dict = None,
+        dec_kwargs: dict = None,
+        init_method='normal',
+        patch_size=-1,
+        **kwargs
+    ):
+        self.sample_rate = sample_rate
+        self.enc_kwargs = enc_kwargs
+        self.dec_kwargs = dec_kwargs
+        self.init_method = init_method
+        self.patch_size = patch_size
+        super().__init__(**kwargs)

talker/AudioVAE/istft.py ADDED Viewed

	@@ -0,0 +1,181 @@

+import torch.nn as nn
+import torch
+class ISTFT(nn.Module):
+    """
+    Custom implementation of ISTFT since torch.istft doesn't allow custom padding (other than `center=True`) with
+    windowing. This is because the NOLA (Nonzero Overlap Add) check fails at the edges.
+    See issue: https://github.com/pytorch/pytorch/issues/62323
+    Specifically, in the context of neural vocoding we are interested in "same" padding analogous to CNNs.
+    The NOLA constraint is met as we trim padded samples anyway.
+    Args:
+        n_fft (int): Size of Fourier transform.
+        hop_length (int): The distance between neighboring sliding window frames.
+        win_length (int): The size of window frame and STFT filter.
+        padding (str, optional): Type of padding. Options are "center" or "same". Defaults to "same".
+    """
+    def __init__(self, n_fft: int, hop_length: int, win_length: int, padding: str = "same"):
+        super().__init__()
+        if padding not in ["center", "same"]:
+            raise ValueError("Padding must be 'center' or 'same'.")
+        self.padding = padding
+        self.n_fft = n_fft
+        self.hop_length = hop_length
+        self.win_length = win_length
+        window = torch.hann_window(win_length)
+        self.register_buffer("window", window)
+        self.audio_buffer = None
+        self.window_buffer = None
+        self.buffer_len = self.win_length - self.hop_length
+    def __buffer_process(self, x, buffer, pad, last_chunk=False, streaming=False):
+        if streaming:
+            if buffer is None:
+                # first chunk
+                x = x[:, pad:]
+            if buffer is not None:
+                # next chunk
+                x[:, :self.buffer_len] += buffer
+            buffer = x[:, -self.buffer_len:]
+            if not last_chunk:
+                x = x[:, :-self.buffer_len]
+            else:
+                x = x[:, :-pad]
+        else:
+            x = x[:, pad:-pad]
+        return x, buffer
+    def forward(
+        self,
+        spec: torch.Tensor,
+        audio_buffer=None,
+        window_buffer=None,
+        streaming=False,
+        last_chunk=False
+    ):
+        """
+        Compute the Inverse Short Time Fourier Transform (ISTFT) of a complex spectrogram.
+        Args:
+            spec (Tensor): Input complex spectrogram of shape (B, N, T), where B is the batch size,
+                            N is the number of frequency bins, and T is the number of time frames.
+            audio_buffer (Tensor): [Streaming Input/State] The audio overlap buffer from the previous chunk.
+                            Shape: (B, win_length - hop_length)
+            window_buffer (Tensor): [Streaming Input/State] The window overlap buffer from the previous chunk.
+            streaming: If `True`, the function operates in streaming mode, processing `spec` as a single chunk.
+            last_chunk: When `streaming=True` and `last_chunk=True`, the function can perform final "flush" operations
+        Returns:
+            Tensor: Reconstructed time-domain signal of shape (B, L), where L is the length of the output signal.
+        """
+        if self.padding == "center":
+            # Fallback to pytorch native implementation
+            return torch.istft(spec, self.n_fft, self.hop_length, self.win_length, self.window, center=True)
+        elif self.padding == "same":
+            pad = (self.win_length - self.hop_length) // 2
+        else:
+            raise ValueError("Padding must be 'center' or 'same'.")
+        assert spec.dim() == 3, "Expected a 3D tensor as input"
+        B, N, T = spec.shape
+        # Inverse FFT
+        ifft = torch.fft.irfft(spec, self.n_fft, dim=1, norm="backward")
+        ifft = ifft * self.window[None, :, None]
+        # Overlap and Add
+        output_size = (T - 1) * self.hop_length + self.win_length
+        y = torch.nn.functional.fold(
+            ifft, output_size=(1, output_size), kernel_size=(1, self.win_length), stride=(1, self.hop_length),
+        )[:, 0, 0, :]
+        y, audio_buffer = self.__buffer_process(y, audio_buffer, pad, last_chunk=last_chunk, streaming=streaming)
+        # Window envelope
+        window_sq = self.window.square().expand(1, T, -1).transpose(1, 2)
+        window_envelope = torch.nn.functional.fold(
+            window_sq, output_size=(1, output_size), kernel_size=(1, self.win_length), stride=(1, self.hop_length),
+        ).squeeze(0).squeeze(0)
+        window_envelope, window_buffer = self.__buffer_process(window_envelope, window_buffer, pad, last_chunk=last_chunk, streaming=streaming)
+        window_envelope = window_envelope.squeeze()
+        # Normalize
+        assert (window_envelope > 1e-11).all()
+        y = y / window_envelope
+        return y, audio_buffer, window_buffer
+class FourierHead(nn.Module):
+    """Base class for inverse fourier modules."""
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        """
+        Args:
+            x (Tensor): Input tensor of shape (B, L, H), where B is the batch size,
+                        L is the sequence length, and H denotes the model dimension.
+        Returns:
+            Tensor: Reconstructed time-domain audio signal of shape (B, T), where T is the length of the output signal.
+        """
+        raise NotImplementedError("Subclasses must implement the forward method.")
+class ISTFTHead(FourierHead):
+    """
+    ISTFT Head module for predicting STFT complex coefficients.
+    Args:
+        dim (int): Hidden dimension of the model.
+        n_fft (int): Size of Fourier transform.
+        hop_length (int): The distance between neighboring sliding window frames, which should align with
+                          the resolution of the input features.
+        padding (str, optional): Type of padding. Options are "center" or "same". Defaults to "same".
+    """
+    def __init__(self, dim: int, n_fft: int, hop_length: int, padding: str = "same"):
+        super().__init__()
+        out_dim = n_fft + 2
+        self.out = torch.nn.Linear(dim, out_dim)
+        self.istft = ISTFT(n_fft=n_fft, hop_length=hop_length, win_length=n_fft, padding=padding)
+    def forward(
+        self,
+        x: torch.Tensor,
+        audio_buffer=None,
+        window_buffer=None,
+        streaming=False,
+        last_chunk=False
+    ):
+        """
+        Forward pass of the ISTFTHead module.
+        Args:
+            x (Tensor): Input tensor of shape (B, L, H), where B is the batch size,
+                        L is the sequence length, and H denotes the model dimension.
+        Returns:
+            Tensor: Reconstructed time-domain audio signal of shape (B, T), where T is the length of the output signal.
+        """
+        x_pred = self.out(x )
+        # x_pred = x
+        x_pred = x_pred.transpose(1, 2)
+        mag, p = x_pred.chunk(2, dim=1)
+        mag = torch.exp(mag)
+        mag = torch.clip(mag, max=1e2)  # safeguard to prevent excessively large magnitudes
+        # wrapping happens here. These two lines produce real and imaginary value
+        x = torch.cos(p)
+        y = torch.sin(p)
+        # recalculating phase here does not produce anything new
+        # only costs time
+        # phase = torch.atan2(y, x)
+        # S = mag * torch.exp(phase * 1j)
+        # better directly produce the complex value
+        S = mag * (x + 1j * y)
+        audio, audio_buffer, window_buffer = self.istft(S, audio_buffer=audio_buffer, window_buffer=window_buffer, streaming=streaming, last_chunk=last_chunk)
+        return audio.unsqueeze(1),x_pred, audio_buffer, window_buffer

talker/AudioVAE/modeling_audio_vae.py ADDED Viewed

	@@ -0,0 +1,80 @@

+from diffusers.models.autoencoders.autoencoder_oobleck import OobleckDiagonalGaussianDistribution
+from transformers import PreTrainedModel
+import torch
+import torch.nn as nn
+from .configuration_audio_vae import AudioVAEconfig
+from .vae_modules import Encoder, Decoder
+class AudioVAE(PreTrainedModel):
+    config_class = AudioVAEconfig
+    def __init__(self, config: AudioVAEconfig):
+        super().__init__(config)
+        self.encoder = Encoder(
+            encoder_args=config.enc_kwargs['backbone'],
+            input_dim=config.enc_kwargs['input_dim'],
+            hop_size=config.enc_kwargs.get('hop_size', 320),
+            latent_dim=config.enc_kwargs['latent_dim'],
+            patch_size=config.patch_size
+        )
+        self.decoder = Decoder(
+            decoder_args=config.dec_kwargs['backbone'],
+            output_dim=config.dec_kwargs['output_dim'],
+            latent_dim=config.dec_kwargs['latent_dim'],
+            patch_size=config.patch_size
+        )
+        self.post_init()
+    def _init_weights(self, module):
+        std = 0.02
+        if isinstance(module, nn.Linear):
+            if self.config.init_method == 'kaiming':
+                nn.init.kaiming_normal_(module.weight, mode='fan_in', nonlinearity='relu')
+            else:
+                module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+    def encode_latent(self, waveform, waveform_length):
+        """
+        Encodes a raw waveform to obtain its acoustic latent representation.
+        Args:
+            waveform: The input audio waveform, shape (B, T_wav).
+            waveform_length: The length of each waveform, shape (B,).
+        Returns:
+            tuple[torch.Tensor, torch.Tensor]: A tuple containing:
+                - z (torch.Tensor): The sampled acoustic latent vectors, shape (B, T_frame, D_latent).
+                - frame_num (torch.Tensor): The number of frames for each audio, shape (B,).
+        """
+        frame_num = torch.ceil(waveform_length/self.config.enc_kwargs['input_dim']).to(torch.int32)
+        if self.config.patch_size != -1:
+            frame_num = torch.ceil(frame_num/self.config.patch_size)
+        h, y = self.encoder(waveform)
+        h = h.transpose(1, 2)  # [B, d, T]
+        posterior = OobleckDiagonalGaussianDistribution(h)
+        latent = posterior.sample()  # [B, d/2, T]
+        latent = latent.transpose(1, 2)
+        return latent, frame_num
+    def decode(self, latent, past_key_values=None, use_cache=False, stream_state=(None, None, None), last_chunk=False):
+        """
+        Reconstructs the raw audio waveform from its acoustic latent representation.
+        Args:
+            latent: The acoustic latent representation, shape: (B, T_frame, D_latent).
+        Returns:
+            The reconstructed raw audio waveform, shape: (B, T_wav)
+        """
+        waveform, stream_state, past_key_values = self.decoder.low_level_reconstruct(latent, past_key_values=past_key_values, use_cache=use_cache, stream_state=stream_state, last_chunk=last_chunk)
+        return waveform, stream_state, past_key_values

talker/AudioVAE/vae_modules.py ADDED Viewed

	@@ -0,0 +1,204 @@

+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import Qwen2Model, Qwen2Config
+import torch
+from .istft import ISTFTHead
+class StreamingLinearUpsample(nn.Module):
+    def __init__(self, scale_factor=4):
+        super().__init__()
+        self.scale_factor = scale_factor
+        self.upsampler = nn.Upsample(scale_factor=scale_factor, mode='linear', align_corners=False)
+    def forward(self, x, state=None, is_last=False):
+        # 初始化状态
+        if state is None:
+            state = {
+                'prev_chunk': None,
+                'history_last': None,
+                'is_first': True
+            }
+        if x is None and not is_last:
+            return None, state
+        if state['is_first'] and is_last:
+            out = self.upsampler(x.transpose(1, 2)).transpose(1, 2)
+            return out, None # 结束后清除状态
+        output_chunks = []
+        if state['is_first']:
+            state['prev_chunk'] = x
+            state['is_first'] = False
+            if not is_last:
+                return None, state
+        if state['prev_chunk'] is not None:
+            p = state['prev_chunk'].transpose(1, 2)
+            if state['history_last'] is None:
+                lookahead = x[:, :1, :].transpose(1, 2)
+                inp = torch.cat([p, lookahead], dim=2)
+                up = self.upsampler(inp)
+                out_prev = up[:, :, :p.size(2) * self.scale_factor]
+            else:
+                lookahead = x[:, :1, :].transpose(1, 2)
+                inp = torch.cat([state['history_last'], p, lookahead], dim=2)
+                up = self.upsampler(inp)
+                start = self.scale_factor
+                end = start + p.size(2) * self.scale_factor
+                out_prev = up[:, :, start:end]
+            output_chunks.append(out_prev.transpose(1, 2))
+            state['history_last'] = p[:, :, -1:]
+            state['prev_chunk'] = x
+        if is_last:
+            p = state['prev_chunk'].transpose(1, 2)
+            inp = torch.cat([state['history_last'], p], dim=2)
+            up = self.upsampler(inp)
+            out_last = up[:, :, self.scale_factor:]
+            output_chunks.append(out_last.transpose(1, 2))
+            state = None # 结束
+        final_out = torch.cat(output_chunks, dim=1) if output_chunks else None
+        return final_out, state
+class Encoder(nn.Module):
+    def __init__(self, encoder_args, input_dim=320, hop_size=320, latent_dim=64, patch_size=-1):
+        super().__init__()
+        config = Qwen2Config.from_dict(config_dict=encoder_args)
+        self.encoder = Qwen2Model(config)
+        self.input_dim = input_dim
+        self.hop_size = hop_size
+        self.latent_dim = latent_dim
+        self.fc1 = nn.Linear(input_dim, config.hidden_size, bias=False)
+        self.fc2 = nn.Linear(config.hidden_size, config.hidden_size)
+        self.fc3 = nn.Linear(config.hidden_size, latent_dim*2)
+        self.norm = nn.LayerNorm(config.hidden_size)
+        self.patch_size = patch_size
+        if patch_size != -1:
+            config.num_hidden_layers = 4
+            self.aggregator = Qwen2Model(config)
+            self.cls_embed = nn.Parameter(torch.rand(1, 1, config.hidden_size))
+            self.cls_embed.data.normal_(0, 0.02)
+    def get_frames(self, x):
+        num_frames_total = (x.size(-1) + self.hop_size - 1) // self.hop_size  # 向上取整的帧数
+        expected_len = (num_frames_total - 1) * self.hop_size + self.input_dim
+        padding_needed = expected_len - x.size(-1)
+        waveform = F.pad(x, (0, padding_needed), value=0.0)
+        frames = waveform.unfold(dimension=-1, size=self.input_dim, step=self.hop_size)   # [B, T, d]
+        return frames
+    def pad_patch_insert_cls(self, x):
+        bsz, _, dim = x.size()
+        num_frame = x.size(1)
+        r = num_frame % self.patch_size
+        pad_num = self.patch_size-r if r else 0
+        x = F.pad(x, (0, 0, 0, pad_num), value=0.0)  # 帧数对齐到patch_size倍数
+        x = x.reshape(-1, self.patch_size, dim)
+        x = torch.cat((x, self.cls_embed.expand(x.size(0), -1, -1)), dim=1)  # 每个patch后插入一个cls
+        x = x.reshape(bsz, -1, dim)
+        return x
+    def forward(self, waveform):
+        x = self.get_frames(waveform)
+        x = self.fc1(x)
+        x = self.fc2(x)
+        x = self.encoder(inputs_embeds=x)
+        x = x.last_hidden_state
+        # downsample
+        if self.patch_size != -1:
+            x = self.pad_patch_insert_cls(x)
+            x = self.aggregator(inputs_embeds=x)
+            x = x.last_hidden_state
+            bsz, _, dim = x.size()
+            x = x.reshape(-1, self.patch_size+1, dim)
+            x = x[:, -1:, :].reshape(bsz, -1, dim)
+        x = self.fc3(x)
+        return x, waveform.unsqueeze(1)
+class Decoder(nn.Module):
+    def __init__(self, decoder_args, output_dim=320, latent_dim=64, patch_size=-1):
+        super().__init__()
+        config = Qwen2Config.from_dict(config_dict=decoder_args)
+        self.decoder = Qwen2Model(config)
+        self.output_dim = output_dim
+        self.latent_dim = latent_dim
+        self.fc1 = nn.Linear(latent_dim, config.hidden_size)
+        self.hop_length = output_dim
+        self.head = ISTFTHead(dim=config.hidden_size, n_fft=self.hop_length * 4, hop_length=self.hop_length, padding="same")
+        self.patch_size = patch_size
+        if self.patch_size != -1:
+            self.upsampling = StreamingLinearUpsample(scale_factor=patch_size)
+    def low_level_reconstruct(self, x, past_key_values=None, use_cache=False, stream_state=None, last_chunk=False):
+        upsample_state, audio_buffer, window_buffer = stream_state
+        bsz, device, dtype = x.size(0), x.device, x.dtype
+        x = self.fc1(x)
+        if self.patch_size != -1:
+            if use_cache:
+                # streaming
+                x, upsample_state = self.upsampling(x, state=upsample_state, is_last=last_chunk)
+                if x is None:
+                    stream_state = (upsample_state, audio_buffer, window_buffer)
+                    return torch.empty(bsz, 1, 0, device=device, dtype=dtype), stream_state, past_key_values
+            else:
+                x = self.upsampling.upsampler(x.transpose(1, 2)).transpose(1, 2)
+        hidden_states_list = []
+        if use_cache and getattr(self.decoder.config, "sliding_window", None) is not None:
+            sw_size = self.decoder.config.sliding_window
+            target_len = sw_size - 1
+            if past_key_values is None:
+                past_len = 0
+            elif hasattr(past_key_values, "get_seq_length"):
+                past_len = past_key_values.get_seq_length()
+            elif isinstance(past_key_values, tuple) and len(past_key_values) > 0:
+                past_len = past_key_values[0][0].shape[-2]
+            else:
+                past_len = 0
+            curr_len = x.shape[1]
+            if past_len < target_len and (past_len + curr_len) >= sw_size:
+                fill_len = target_len - past_len
+                x_fill = x[:, :fill_len, :]
+                outputs = self.decoder(inputs_embeds=x_fill, past_key_values=past_key_values, use_cache=use_cache)
+                hidden_states_list.append(outputs.last_hidden_state)
+                past_key_values = outputs.past_key_values
+                x = x[:, fill_len:, :]
+        outputs = self.decoder(inputs_embeds=x, past_key_values=past_key_values, use_cache=use_cache)
+        hidden_states_list.append(outputs.last_hidden_state)
+        past_key_values = outputs.past_key_values
+        if len(hidden_states_list) > 1:
+            full_hidden_state = torch.cat(hidden_states_list, dim=1)
+        else:
+            full_hidden_state = hidden_states_list[0]
+        x_out, _, audio_buffer, window_buffer = self.head(
+            full_hidden_state,
+            streaming=use_cache,
+            audio_buffer=audio_buffer,
+            window_buffer=window_buffer,
+            last_chunk=last_chunk
+        )
+        stream_state = (upsample_state, audio_buffer, window_buffer)
+        return x_out, stream_state, past_key_values

talker/__init__.py ADDED Viewed

	@@ -0,0 +1,5 @@

+"""Native Ming Talker inference and checkpoint assets."""
+from .inference import MingTalker
+__all__ = ["MingTalker"]

talker/__pycache__/__init__.cpython-312.pyc ADDED Viewed

Binary file (317 Bytes). View file

talker/__pycache__/configuration_bailing_talker.cpython-312.pyc ADDED Viewed

Binary file (509 Bytes). View file

talker/__pycache__/inference.cpython-312.pyc ADDED Viewed

Binary file (22.6 kB). View file

talker/__pycache__/tts_prompt.cpython-312.pyc ADDED Viewed

Binary file (17.3 kB). View file

talker/campplus.onnx ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:a6ac6a63997761ae2997373e2ee1c47040854b4b759ea41ec48e4e42df0f4d73
+size 28303423

talker/config.json ADDED Viewed

	@@ -0,0 +1,35 @@

+{
+  "aggregator": {
+    "attn_backend": "torch",
+    "attn_mask_enabled": false,
+    "depth": 8,
+    "dropout": 0.0,
+    "hidden_size": 1024,
+    "in_channels": 64,
+    "mlp_ratio": 4,
+    "num_heads": 16,
+    "pe_attn_head": null,
+    "qk_norm": null
+  },
+  "architectures": [
+    "BailingTalker2"
+  ],
+  "cfg_strength": 2.0,
+  "flowmodel": {
+    "attn_backend": "torch",
+    "attn_mask_enabled": false,
+    "depth": 8,
+    "dropout": 0.0,
+    "hidden_size": 1024,
+    "in_channels": 64,
+    "mlp_ratio": 4,
+    "num_heads": 16,
+    "pe_attn_head": null,
+    "qk_norm": null
+  },
+  "history_patch_size": 1,
+  "patch_size": 1,
+  "steps": 10,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3"
+}

talker/configuration_bailing_talker.py ADDED Viewed

	@@ -0,0 +1,20 @@

+# coding=utf-8
+# Copyright 2022 shunxing1234 and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" GLMAudio model configuration """
+from transformers.configuration_utils import PretrainedConfig
+class BailingTalkerConfig(PretrainedConfig):
+    pass

talker/data/lid.176.LICENSE.md ADDED Viewed

	@@ -0,0 +1,10 @@

+# FastText language identification model
+`lid.176.ftz` is the compressed language identification model published by the
+fastText project: https://fasttext.cc/docs/en/language-identification
+The model is distributed under the Creative Commons Attribution-ShareAlike 3.0
+license: https://creativecommons.org/licenses/by-sa/3.0/
+The fastText project states that the model was trained on Wikipedia, Tatoeba,
+and SETimes data.

talker/data/lid.176.ftz ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:8f3472cfe8738a7b6099e8e999c3cbfae0dcd15696aac7d7738a8039db603e83
+size 938013

talker/front/__init__.py ADDED Viewed

File without changes

talker/front/__pycache__/__init__.cpython-312.pyc ADDED Viewed

Binary file (180 Bytes). View file

talker/front/__pycache__/number_en.cpython-312.pyc ADDED Viewed

Binary file (14.7 kB). View file

talker/front/__pycache__/text_segment_cut.cpython-312.pyc ADDED Viewed

Binary file (12.7 kB). View file

talker/front/__pycache__/toolkit.cpython-312.pyc ADDED Viewed

Binary file (9.31 kB). View file

talker/front/number_en.py ADDED Viewed

	@@ -0,0 +1,343 @@

+import re
+import inflect
+'''
+number normalization for English
+'''
+_inflect = inflect.engine()
+# 移除逗号：1,000 → 1000
+_comma_number_re = re.compile(r"([0-9][0-9,]+[0-9])")
+# 百分比：支持负数（-25.5%）
+_percent_number_re = re.compile(r"(-?[0-9.,]*[0-9]+)%")
+# 英镑：£-150 → minus one hundred fifty pounds
+_pounds_re = re.compile(r"£(-?[0-9,]*[0-9]+(?:\.[0-9]+)?)")
+# 美元：$-12.99
+_dollars_re = re.compile(r"\$(-?[0-9.,]*[0-9]+(?:\.[0-9]+)?)")
+# 分数：3/4
+_fraction_re = re.compile(r"([0-9]+)\/([0-9]+)")
+# 序数：1st, 2nd
+_ordinal_re = re.compile(r"\b[0-9]+(st|nd|rd|th)\b")
+# 普通数字（最后处理）
+_number_re = re.compile(r"\b-?[0-9]+(?:\.[0-9]+)?\b")
+# 单位：10ms, 5.5 kg 等
+_unit_re = re.compile(
+    r'\b(-?\d+(?:\.\d+)?)\s*(ms|s|Hz|kHz|MHz|GHz|kb|mb|gb|tb|KB|MB|GB|TB|bps|kbps|Mbps|Gbps|cm|km|kg|V|A|W|°C|°F)\b',
+    re.IGNORECASE
+)
+# 正则：支持 v2, v2.1, gpt-4, gpt-4.5
+_version_re = re.compile(r'\b([a-zA-Z]+)([-]?)([0-9]+(?:\.[0-9]+)?)\b')
+# 空白字符
+_whitespace_re = re.compile(r"\s+")
+# 标点符号（可选）
+_colon_re = re.compile(r":")
+_unit_mapping = {
+    'ms': 'milliseconds',
+    's': 'seconds',
+    'hz': 'hertz',
+    'khz': 'kilohertz',
+    'mhz': 'megahertz',
+    'ghz': 'gigahertz',
+    'kb': 'kilobytes',
+    'mb': 'megabytes',
+    'gb': 'gigabytes',
+    'tb': 'terabytes',
+    'kbps': 'kilobits per second',
+    'mbps': 'megabits per second',
+    'gbps': 'gigabits per second',
+    'bps': 'bits per second',
+    'cm': 'centimeters',
+    'km': 'kilometers',
+    'kg': 'kilograms',
+    'v': 'volts',
+    'a': 'amperes',
+    'w': 'watts',
+    '°c': 'degrees celsius',
+    '°f': 'degrees fahrenheit',
+}
+def _remove_commas(m):
+    return m.group(1).replace(",", "")
+def _expand_colon(m):
+    return " colon "
+def _expand_unit(m):
+    num_str = m.group(1)
+    unit = m.group(2).lower()
+    unit_word = _unit_mapping.get(unit, unit)
+    try:
+        is_negative = num_str.startswith('-')
+        clean_num = num_str.lstrip('-')
+        if not clean_num:
+            clean_num = '0'
+        if '.' in clean_num:
+            parts = clean_num.split('.', 1)
+            integer_part = parts[0] or '0'
+            decimal_part = parts[1]
+            # 检查是否全为数字
+            if not integer_part.isdigit() or not decimal_part.isdigit():
+                return f" {num_str} {unit} "
+            integer_word = _inflect.number_to_words(int(integer_part), andword="") if integer_part != '0' else 'zero'
+            decimal_words = " ".join(
+                _inflect.number_to_words(int(d), andword="") for d in decimal_part if d.isdigit()
+            )
+            num_word = f"{integer_word} point {decimal_words}"
+        else:
+            if not clean_num.isdigit():
+                return f" {num_str} {unit} "
+            num_word = _inflect.number_to_words(int(clean_num), andword="")
+        if is_negative:
+            num_word = f"minus {num_word}"
+        return f" {num_word} {unit_word} "
+    except Exception:
+        return f" {num_str} {unit} "
+def _expand_percent(m):
+    num_str = m.group(1)
+    try:
+        is_negative = num_str.startswith('-')
+        clean_num = num_str.lstrip('-')
+        if not clean_num:
+            clean_num = '0'
+        if '.' in clean_num:
+            parts = clean_num.split('.', 1)
+            integer_part = parts[0] or '0'
+            decimal_part = parts[1]
+            if not integer_part.isdigit() or not decimal_part.isdigit():
+                return f" {num_str} percent "
+            integer_word = _inflect.number_to_words(int(integer_part), andword="") if integer_part != '0' else 'zero'
+            decimal_words = " ".join(
+                _inflect.number_to_words(int(d), andword="") for d in decimal_part if d.isdigit()
+            )
+            num_word = f"{integer_word} point {decimal_words}"
+        else:
+            if not clean_num.isdigit():
+                return f" {num_str} percent "
+            num_word = _inflect.number_to_words(int(clean_num), andword="")
+        if is_negative:
+            num_word = f"minus {num_word}"
+        return f" {num_word} percent "
+    except Exception:
+        return f" {num_str} percent "
+def _expand_dollars(m):
+    match = m.group(1)
+    is_negative = match.startswith('-')
+    clean_match = match.lstrip('-')
+    if not clean_match:
+        clean_match = '0'
+    try:
+        if '.' in clean_match:
+            parts = clean_match.split('.', 1)
+            integer_part = parts[0] or '0'
+            decimal_part = parts[1]
+            if not integer_part.isdigit() or not decimal_part.isdigit():
+                return f" {clean_match} dollars "
+            integer_word = _inflect.number_to_words(int(integer_part), andword="") if integer_part != '0' else 'zero'
+            decimal_words = " ".join(
+                _inflect.number_to_words(int(d), andword="") for d in decimal_part if d.isdigit()
+            )
+            num_word = f"{integer_word} point {decimal_words}"
+        else:
+            if not clean_match.isdigit():
+                return f" {clean_match} dollars "
+            num_word = _inflect.number_to_words(int(clean_match), andword="")
+        if is_negative:
+            num_word = f"minus {num_word}"
+        value = float(clean_match)
+        unit = "dollar" if abs(value) == 1.0 else "dollars"
+        return f" {num_word} {unit} "
+    except Exception:
+        return f" {clean_match} dollars "
+def _expand_pounds(m):
+    num_str = m.group(1)
+    is_negative = num_str.startswith('-')
+    clean_num = num_str.lstrip('-')
+    if not clean_num:
+        clean_num = '0'
+    try:
+        if '.' in clean_num:
+            parts = clean_num.split('.', 1)
+            integer_part = parts[0] or '0'
+            decimal_part = parts[1]
+            if not integer_part.isdigit() or not decimal_part.isdigit():
+                return f" {clean_num} pounds "
+            integer_word = _inflect.number_to_words(int(integer_part), andword="") if integer_part != '0' else 'zero'
+            decimal_words = " ".join(
+                _inflect.number_to_words(int(d), andword="") for d in decimal_part if d.isdigit()
+            )
+            num_word = f"{integer_word} point {decimal_words}"
+        else:
+            if not clean_num.isdigit():
+                return f" {clean_num} pounds "
+            num_word = _inflect.number_to_words(int(clean_num), andword="")
+        if is_negative:
+            num_word = f"minus {num_word}"
+        value = float(clean_num)
+        unit = "pound" if abs(value) == 1.0 else "pounds"
+        return f" {num_word} {unit} "
+    except Exception:
+        return f" {clean_num} pounds "
+def fraction_to_words(numerator, denominator):
+    try:
+        if numerator == 1 and denominator == 2:
+            return " one half "
+        if numerator == 1 and denominator == 4:
+            return " one quarter "
+        if denominator == 2:
+            plural = " half" if numerator == 1 else " halves"
+            return f" {_inflect.number_to_words(numerator)}{plural} "
+        if denominator == 4:
+            plural = " quarter" if numerator == 1 else " quarters"
+            return f" {_inflect.number_to_words(numerator)}{plural} "
+        ordinal = _inflect.ordinal(_inflect.number_to_words(denominator))
+        return f" {_inflect.number_to_words(numerator)} {ordinal} "
+    except Exception:
+        return f" {numerator} over {denominator} "
+def _expand_fraction(m):
+    try:
+        numerator = int(m.group(1))
+        denominator = int(m.group(2))
+        return fraction_to_words(numerator, denominator)
+    except Exception:
+        return m.group(0)
+def _expand_ordinal(m):
+    try:
+        # 提取纯数字部分
+        num = int(re.sub(r"(st|nd|rd|th)", "", m.group(0)))
+        word = _inflect.number_to_words(num)
+        return f" {word} "
+    except Exception:
+        return m.group(0)
+def _expand_number(m):
+    num_str = m.group(0)
+    try:
+        is_negative = num_str.startswith('-')
+        clean_num = num_str.lstrip('-')
+        if not clean_num:
+            clean_num = '0'
+        if '.' in clean_num:
+            parts = clean_num.split('.', 1)
+            integer_part = parts[0]
+            decimal_part = parts[1]
+            if not integer_part:
+                integer_part = '0'
+            if not integer_part.isdigit() or not decimal_part.isdigit():
+                return f" {num_str} "
+            integer_word = _inflect.number_to_words(int(integer_part), andword="") if integer_part != '0' else 'zero'
+            decimal_words = " ".join(
+                _inflect.number_to_words(int(d), andword="") for d in decimal_part if d.isdigit()
+            )
+            num_word = f"{integer_word} point {decimal_words}"
+        else:
+            if not clean_num.isdigit():
+                return f" {num_str} "
+            num_word = _inflect.number_to_words(int(clean_num), andword="")
+        if is_negative:
+            num_word = f"minus {num_word}"
+        return f" {num_word} "
+    except Exception:
+        return f" {num_str} "
+def _expand_version(m):
+    prefix = m.group(1)
+    sep = m.group(2)
+    num_str = m.group(3)
+    try:
+        if '.' in num_str:
+            parts = num_str.split('.', 1)
+            integer_part = parts[0]
+            decimal_part = parts[1]
+            if not integer_part.isdigit() or not decimal_part.isdigit():
+                return m.group(0)
+            integer_word = _inflect.number_to_words(int(integer_part))
+            decimal_words = " ".join(
+                _inflect.number_to_words(int(d)) for d in decimal_part if d.isdigit()
+            )
+            word = f"{integer_word} point {decimal_words}"
+        else:
+            if not num_str.isdigit():
+                return m.group(0)
+            word = _inflect.number_to_words(int(num_str))
+    except Exception:
+        return m.group(0)
+    # 连字符替换为空格，如 gpt-4 → gpt four
+    if sep == '-':
+        return f"{prefix} {word}"
+    # 直接拼接则加空格读出：v2 → v two
+    return f"{prefix} {word}"
+def normalize_numbers(text):
+    text = re.sub(_comma_number_re, _remove_commas, text)
+    text = re.sub(_unit_re, _expand_unit, text)             # 带单位的量（含负数）
+    text = re.sub(_pounds_re, _expand_pounds, text)
+    text = re.sub(_dollars_re, _expand_dollars, text)
+    text = re.sub(_fraction_re, _expand_fraction, text)
+    text = re.sub(_percent_number_re, _expand_percent, text)
+    text = re.sub(_ordinal_re, _expand_ordinal, text)
+    text = re.sub(_version_re, _expand_version, text)       # 新增：处理 v2, gpt-4
+    text = re.sub(_number_re, _expand_number, text)         # 剩余孤立数字
+    # text = re.sub(_colon_re, _expand_colon, text)         # 冒号（可选）
+    text = re.sub(_whitespace_re, " ", text)
+    return text.strip()

talker/front/text_segment_cut.py ADDED Viewed

	@@ -0,0 +1,388 @@

+import re
+import string
+def is_chinese(text):
+    """检测文本是否包含中文"""
+    # 使用简单规则判断
+    return bool(re.search(r'[\u4e00-\u9fff]', text))
+def get_lang_type(text):
+    """
+    - 'zh': 只包含中文字符（及相关标点）。
+    - 'en': 只包含英文字符（及相关标点）。
+    - 'mixed': 同时包含中文字符和英文字符。
+    - 'other': 只包含数字、符号等，不含中英文字符。
+    """
+    has_chinese = bool(re.search(r'[\u4e00-\u9fff]', text))
+    has_english = bool(re.search(r'[a-zA-Z]', text))
+    if has_chinese and has_english:
+        return 'mixed'
+    if has_chinese:
+        return 'zh'
+    if has_english:
+        return 'en'
+    return 'other'
+def split_and_group_sentences(text, group_by_lang=False):
+    # 分句正则表达式，匹配中文和英文的句子结束标点
+    # pattern = r'(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=[.!?。！？])\s*'
+    # 仅匹配中文冒号 `：` 或英文冒号 `:` 后跟空格的情况
+    pattern = r'(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=[.!?。！？]|:(?=\s)|：)\s*'
+    sentences = [s.strip() for s in re.split(pattern, text.strip()) if s.strip()]
+    if not group_by_lang:
+        return sentences
+    # 聚合相同语言的相邻子句
+    grouped = []
+    current_group = []
+    current_lang = None
+    for sentence in sentences:
+        lang = 'zh' if is_chinese(sentence) else 'en'
+        if current_lang is None:
+            current_group.append(sentence)
+            current_lang = lang
+        elif lang == current_lang:
+            # 中文直接连接，英文添加空格
+            if lang == 'zh':
+                current_group.append(sentence)
+            else:
+                current_group.append(' ' + sentence if sentence[0].isalnum() else sentence)
+        else:
+            grouped.append(''.join(current_group))
+            current_group = [sentence]
+            current_lang = lang
+    if current_group:
+        grouped.append(''.join(current_group))
+    return grouped
+def get_semantic_length(text):
+    """
+    计算字符串的语义长度。
+    一个汉字计为1，一个连续的英文字母串（单词）计为1。
+    """
+    chinese_char_count = len(re.findall(r'[\u4e00-\u9fa5]', text))
+    # [a-zA-Z]+ 匹配一个或多个连续的英文字母
+    english_word_count = len(re.findall(r'[a-zA-Z]+', text))
+    return chinese_char_count + english_word_count
+def has_valid_content(text):
+    """
+    判断文本是否包含有效字符（非标点符号）
+    """
+    punctuation_and_whitespace = string.punctuation + string.whitespace
+    for char in text:
+        if char not in punctuation_and_whitespace:
+            return True
+    return False
+def append_text_fragment(fragments, new_text, max_len, min_tail_length):
+    """
+    将新文本追加到片段列表中，根据语义长度决定是否合并
+    """
+    # 清理 new_text 的前导标点和空格
+    new_text = new_text.lstrip("，,:;" + string.whitespace)
+    if not has_valid_content(new_text):
+        return fragments
+    if not fragments:
+        fragments.append(new_text)
+        return fragments
+    last_fragment = fragments[-1]
+    last_semantic_len = get_semantic_length(last_fragment)
+    new_semantic_len = get_semantic_length(new_text)
+    if last_semantic_len + new_semantic_len <= max_len:
+        # 特殊处理：如果上一段以句号结尾且新文本很短，则开启新段落
+        if last_fragment.endswith(("。", "！", "？")) and new_semantic_len < min_tail_length:
+            fragments.append(new_text)
+        else:
+            # 决定连接符，让中英文拼接更自然
+            separator = ""
+            # 如果上一个片段结尾不是空格，且新片段以英文/数字开头，则加个空格
+            if not last_fragment.endswith(" ") and re.match(r'^[a-zA-Z0-9]', new_text):
+                 separator = " "
+            fragments[-1] += separator + new_text
+    else:
+        # 开启新段落时，new_text 已经被清理过了
+        fragments.append(new_text)
+    return fragments
+def split_long_fragment(text_fragment, max_len):
+    """
+    分割超长的文本片段。优先沿标点(clause边界)切,避免从字中间断开。
+    仅当单个 clause 自身仍超 max_len 时,才退化到字/词级切分(保底不卡死)。
+    """
+    fragment_semantic_len = get_semantic_length(text_fragment)
+    if fragment_semantic_len <= max_len:
+        return [text_fragment]
+    # —— 第一优先级:沿标点切 clause ——
+    # 标点(中文，、；。！？ + 英文 ,;!?!) 及列表分隔(空格+[-•·]) 作为切分边界。
+    # 用 split 保留分隔符并并回前一 clause, 使子串拼接 == 原文(位置校准仍能 find 到)。
+    clause_pattern = r'([，、；。！？,;!?]|\s+[-•·])'
+    parts = re.split(clause_pattern, text_fragment)
+    merged_clauses = []
+    for part in parts:
+        if part == "":
+            continue
+        if merged_clauses and re.fullmatch(clause_pattern, part):
+            merged_clauses[-1] += part          # 标点/分隔 并回前一段
+        else:
+            merged_clauses.append(part)
+    fragments = []
+    current_fragment = ""
+    for clause in merged_clauses:
+        clause_semantic_len = get_semantic_length(clause)
+        # 单个 clause 自身超 max_len: 保底字/词级切分(避免无限卡死)
+        if clause_semantic_len > max_len:
+            if current_fragment:
+                fragments.append(current_fragment)
+                current_fragment = ""
+            fragments.extend(_split_by_char_word(clause, max_len))
+            continue
+        current_semantic_len = get_semantic_length(current_fragment)
+        if current_semantic_len + clause_semantic_len <= max_len:
+            current_fragment += clause
+        else:
+            if current_fragment:
+                fragments.append(current_fragment)
+            current_fragment = clause
+    if current_fragment:
+        fragments.append(current_fragment)
+    return fragments
+def _split_by_char_word(text, max_len):
+    """
+    保底切分:按字(汉字)/词(英文)等基本语义单元切,用于单个 clause 仍超长时。
+    """
+    fragments = []
+    current = ""
+    tokens = re.findall(r'(\s+|[，、；。！？,;!?]|[\u4e00-\u9fa5]|[a-zA-Z]+|[^a-zA-Z\u4e00-\u9fa5]+)', text)
+    for tok in tokens:
+        tok_len = get_semantic_length(tok)
+        cur_len = get_semantic_length(current)
+        if cur_len + tok_len <= max_len:
+            current += tok
+        else:
+            if current:
+                fragments.append(current)
+            if tok_len > max_len:
+                fragments.append(tok)
+                current = ""
+            else:
+                current = tok
+    if current:
+        fragments.append(current)
+    return fragments
+def calibrate_positions(fragments, positions, original_text):
+    """
+    校准fragments和positions的对齐关系
+    """
+    calibrated_positions = {}
+    current_global_pos = 0
+    for frag_idx, fragment in enumerate(fragments):
+        # 在原始文本中查找当前片段的位置
+        frag_len = len(fragment)
+        found_pos = original_text.find(fragment, current_global_pos)
+        if found_pos == -1:
+            # 如果找不到，尝试放宽条件（如忽略空格差异）
+            simplified_frag = fragment.replace(" ", "")
+            simplified_original = original_text.replace(" ", "")
+            found_pos = simplified_original.find(simplified_frag, current_global_pos)
+            if found_pos != -1:
+                # 找到后映射回原始位置
+                non_space_count = original_text[:found_pos].count(" ")
+                found_pos += non_space_count
+        if found_pos == -1:
+            # 如果仍然找不到，使用原positions中的值（降级处理）
+            if frag_idx in positions:
+                calibrated_positions[frag_idx] = positions[frag_idx]
+            else:
+                calibrated_positions[frag_idx] = (current_global_pos, current_global_pos + frag_len)
+            continue
+        calibrated_positions[frag_idx] = (found_pos, found_pos + frag_len)
+        current_global_pos = found_pos + frag_len
+    return calibrated_positions
+def cut_text_by_semantic_length(text, max_semantic_length=50, min_tail_length=5):
+    """
+    按照语义长度划分文本，保持语义单元的完整性，并返回位置信息
+    Args:
+        text: 输入文本
+        max_semantic_length: 最大语义长度限制
+        min_tail_length: 尾部最小长度
+    Returns:
+        dict: {
+            "fragments": List[str],  # 分割后的文本片段列表
+            "positions": Dict[int, Tuple[int, int]]  # 子句位置信息（key为顺序，value为(起始index, 结束index)）
+        }
+    """
+    if not has_valid_content(text):
+        return {"fragments": [], "positions": {}}
+    # 保存原始文本用于位置计算
+    original_text = text
+    # 定义一个独特的占位符
+    DOT_PLACEHOLDER = "##DOT##"
+    # 保护小数点和特定缩写中的点
+    # 使用正则表达式查找数字中的点（如 3.14）和常见缩写中的点（如 U.S.A.）
+    # 并将它们替换为占位符
+    # `\d\.\d` 匹配数字间的小数点
+    # `([A-Z])\.([A-Z])` 匹配大写字母间的点
+    processed_text = re.sub(r'(\d)\.(\d)', r'\1' + DOT_PLACEHOLDER + r'\2', text)
+    for _ in range(3):
+        processed_text = re.sub(r'([A-Z])\.([A-Z])', r'\1' + DOT_PLACEHOLDER + r'\2', processed_text)
+    processed_text = processed_text.replace("\n", " ").replace("。，", "。")
+    if get_semantic_length(processed_text) <= max_semantic_length:
+        # 返回前恢复占位符
+        return {
+            "fragments": [processed_text.replace(DOT_PLACEHOLDER, ".")],
+            "positions": {0: (0, len(text))}  # 整个文本的位置
+        }
+    # 标点标准化（只在处理时使用，不影响原始位置计算）
+    normalized_text = processed_text.replace(".", "。").replace("!", "！").replace("?", "？").replace(",", "，")
+    result_fragments = []
+    position_map = {}
+    global_offset = 0  # 跟踪在原始文本中的位置
+    # 第一阶段：按句子分割
+    sentences = []
+    sentence_positions = []
+    current_sentence = ""
+    start_idx = 0
+    for i, char in enumerate(normalized_text):
+        current_sentence += char
+        if char in "。！？": # 移除了 '.' 和 '!' '?'
+            sentence = current_sentence.strip()
+            if sentence:
+                sentences.append(sentence)
+                sentence_positions.append((start_idx, i + 1))  # 包含标点的结束位置
+            current_sentence = ""
+            start_idx = i + 1
+    # 处理最后一个句子
+    if current_sentence:
+        # 如果最后一部分没有标点，可以补上，也可以不补，保持原逻辑
+        sentences.append(current_sentence.strip())
+        sentence_positions.append((start_idx, len(normalized_text)))
+        if not sentences[-1].endswith(("。", "！", "？")):
+             sentences[-1] += "。"
+    # 第二阶段：处理每个句子
+    fragment_counter = 0
+    for sent_idx, (sentence, (sent_start, sent_end)) in enumerate(zip(sentences, sentence_positions)):
+        clauses = []
+        clause_positions = []
+        current_clause = ""
+        clause_start = 0
+        for i, char in enumerate(sentence):
+            current_clause += char
+            if char in "，;；": # 移除了 ','
+                clause = current_clause.strip()
+                if clause and has_valid_content(clause):
+                    clauses.append(clause)
+                    clause_positions.append((clause_start, i + 1))  # 在句子中的相对位置
+                elif clause and clauses:
+                    clauses[-1] += clause
+                    clause_positions[-1] = (clause_positions[-1][0], i + 1)
+                current_clause = ""
+                clause_start = i + 1
+        if current_clause:
+            clause = current_clause.strip()
+            if clause and has_valid_content(clause):
+                clauses.append(clause)
+                clause_positions.append((clause_start, len(sentence)))
+            elif clause and clauses:
+                clauses[-1] += clause
+                clause_positions[-1] = (clause_positions[-1][0], len(sentence))
+        # 第三阶段：处理每个子句
+        i = 0
+        while i < len(clauses):
+            clause = clauses[i]
+            clause_start_in_sent, clause_end_in_sent = clause_positions[i]
+            # 计算在原始文本中的绝对位置
+            abs_start = sent_start + clause_start_in_sent
+            abs_end = sent_start + clause_end_in_sent
+            clause_semantic_len = get_semantic_length(clause)
+            # 如果当前子句很短，则尝试与下一个子句聚合
+            if clause_semantic_len < min_tail_length and i + 1 < len(clauses):
+                next_clause = clauses[i + 1]
+                next_start, next_end = clause_positions[i + 1]
+                combined_clause = clause + next_clause
+                combined_semantic_len = get_semantic_length(combined_clause)
+                # 如果聚合后的子句不超过最大长度，则聚合
+                if combined_semantic_len <= max_semantic_length:
+                    result_fragments = append_text_fragment(result_fragments, combined_clause, max_semantic_length, min_tail_length)
+                    # 合并位置信息
+                    merged_position = (abs_start, sent_start + next_end)
+                    position_map[fragment_counter] = merged_position
+                    fragment_counter += 1
+                    i += 2
+                    continue
+            # 如果子句仍然超过最大长度，则进一步分割
+            if clause_semantic_len > max_semantic_length:
+                sub_fragments = split_long_fragment(clause, max_semantic_length)
+                # 等分位置（简化处理，实际应根据语义分割调整）
+                frag_len = len(clause)
+                sub_frag_count = len(sub_fragments)
+                for j, frag in enumerate(sub_fragments):
+                    result_fragments = append_text_fragment(result_fragments, frag, max_semantic_length, min_tail_length)
+                    frag_len_part = len(frag)
+                    frag_start = abs_start + j * (frag_len // sub_frag_count)
+                    frag_end = frag_start + frag_len_part
+                    position_map[fragment_counter] = (frag_start, frag_end)
+                    fragment_counter += 1
+            else:
+                result_fragments = append_text_fragment(result_fragments, clause, max_semantic_length, min_tail_length)
+                position_map[fragment_counter] = (abs_start, abs_end)
+                fragment_counter += 1
+            i += 1
+    # 最终返回结果前，恢复所有片段中的占位符
+    final_result = [frag.replace(DOT_PLACEHOLDER, ".") for frag in result_fragments]
+    calibrated_positions = calibrate_positions(final_result, position_map, original_text)
+    return {
+        "fragments": final_result,
+        "positions": calibrated_positions
+    }

talker/front/toolkit.py ADDED Viewed

	@@ -0,0 +1,240 @@

+import re
+from typing import Iterable
+TOKENIZE_PATTERN = r"(?:[a-zA-Z]\.)+|[a-zA-Z]+(?:['\-][a-zA-Z]+)*|\d+(?:\.\d+)?|[\u4e00-\u9fff]|\s+|\S"
+def tokenize_mixed_text(text: str):
+    """
+    将混合文本字符串处理为标记列表，并保留原始的空格。
+    """
+    return re.findall(TOKENIZE_PATTERN, text)
+def tokenize_mixed_text_iterator(text_iterator: Iterable[str]):
+    """
+    Args:
+        text_iterator (Iterable[str]): 一个产出字符串的迭代器。
+            例如，一个打开的文件对象，或一个生成器表达式。
+    Yields:
+        Iterator[str]: 逐一产出文本中的标记（单词、中文、标点、空格等）。
+    """
+    # 函数本身就是一个生成器，将从 re.finditer 返回的迭代器中逐一产出结果
+    for chunk in text_iterator:
+        # re.finditer 返回一个迭代器，而不是列表
+        for match in re.finditer(TOKENIZE_PATTERN, chunk):
+            yield match.group(0)
+def split_uppercase_acronyms(text):
+    """
+    将连续大写字母（缩写词）逐字母用空格隔开。
+    单个大写字母或首字母大写的单词不受影响。
+    """
+    return re.sub(r'[A-Z]{2,}', lambda m: ' '.join(m.group(0)), text)
+def replace_parens_with_sep(text, target_sep='，'):
+    """
+    将中英文括号替换为target_sep。
+    """
+    for ch in '()（）':
+        text = text.replace(ch, target_sep)
+    return text
+def clean_text_formatting(text):
+    if not text:
+        return text
+    # 定义开头需要被替换为空格的“杂质标点”
+    # 保留了：() {} [] （） 【】 "" '' “” ‘’ 《》 等成对符号
+    # 替换掉：， 。 ！ ？ ； ： 、 …… 等容易出现在开头且没意义的标点
+    leading_punct_pattern = r'^[，。！？；：、……\.\!\?\-\—]+'
+    # 替换开头标点
+    text = re.sub(leading_punct_pattern, ' ', text)
+    # # 聚合连续空格
+    # text = re.sub(r'\s+', ' ', text)
+    # 去除首尾多余空格
+    return text.strip()
+def chinese_punc_en2cn(text):
+    """
+    含中文文本清理：包含中文即处理，多个堆叠标点最终只保留一个。
+    1. 增加 URL和小数点 保护，防止链接内部符号被误转。
+    2. 增强标点优先级逻辑，确保顿号（、）正确去重。
+    3. 修复异类标点堆叠的匹配规则。
+    4、修复开头符号
+    """
+    # # 允许的字符范围：汉字、中文标点、全角符号、空格、以及常见的英文标点
+    # # 明确排除：a-zA-Z（英文字母）、0-9（数字）
+    # # gen_tag = 'tn'
+    # if re.search(r'[a-zA-Z0-9]', text):
+    #     return text
+    # 仅检查是否包含中文，无中文则直接返回原文本
+    # gen_tag = 'tn_v1'
+    if not text or not re.search(r'[\u4e00-\u9fa5]', text):
+        return clean_text_formatting(text)
+    # URL 占位保护
+    urls = re.findall(r'https?://[^\s，。！？]+', text)
+    for i, url in enumerate(urls):
+        text = text.replace(url, f"URLHOLDER{i}")
+    # 保护数字间的小数点，防止被 punc_map 误转为 。
+    text = re.sub(r'(?<=\d)\.(?=\d)', 'DOTPLACEHOLDER', text)
+    # 基础符号转换
+    text = text.replace('...', '……')
+    punc_map = {
+        ',': '，', '.': '。', ';': '；', ':': '：', '!': '！', '?': '？',
+        '(': '（', ')': '）', '[': '【', ']': '】', '<': '《', '>': '》', '_': '——'
+    }
+    map_pattern = re.compile('|'.join(re.escape(k) for k in punc_map.keys()))
+    text = map_pattern.sub(lambda m: punc_map[m.group(0)], text)
+    # 还原小数点
+    text = text.replace('DOTPLACEHOLDER', '.')
+    # 修复：引号计数与转换 (改用闭包计数，避免 := 运算符在 re.sub 中的潜在崩溃)
+    dq_count = [0]
+    sq_count = [0]
+    def dq_sub(_):
+        dq_count[0] += 1
+        return '“' if dq_count[0] % 2 != 0 else '”'
+    def sq_sub(_):
+        sq_count[0] += 1
+        return '‘' if sq_count[0] % 2 != 0 else '’'
+    text = re.sub(r'"', dq_sub, text)
+    text = re.sub(r"'", sq_sub, text)
+    # 彻底消除标点周边的空格
+    ch_puncs = r'，。！？；：“”‘’（）【】《》——……、'
+    text = re.sub(fr'[ \t\u00A0\u3000]*([{ch_puncs}])[ \t\u00A0\u3000]*', r'\1', text)
+    # 标点递归清理
+    while True:
+        old_text = text
+        # 压缩完全相同的重复标点
+        text = re.sub(r'([{ch_puncs}])\1+', r'\1', text)
+        # 弱标点(，；、) 遇到 强标点(。！？) 时删除弱标点
+        text = re.sub(r'[，；、]+([。！？])', r'\1', text)
+        text = re.sub(r'([。！？])[，；、]+', r'\1', text)
+        # 处理 逗号/分号 与 顿号 的堆叠
+        text = re.sub(r'[，；]+[、]|[、][，；]+', '、', text)
+        # 强标点间唯一化
+        text = re.sub(r'([。！？])[。！？]+', r'\1', text)
+        # 剩余异类弱标点堆叠唯一化
+        text = re.sub(r'([，；、])[，；、]+', r'\1', text)
+        if old_text == text:
+            break
+    # 压缩汉字/单词间连续空格
+    text = re.sub(r'[ \t\u00A0\u3000]+', ' ', text)
+    # 还原 URL
+    for i, url in enumerate(urls):
+        text = text.replace(f"URLHOLDER{i}", url)
+    return clean_text_formatting(text)
+# 弱标点结尾视为未收束，升格为句末标点（替换而非追加：
+# 纯英文不经 chinese_punc_en2cn 的标点去重，追加会留下 ",." 漏网）
+WEAK_TAIL_PUNCT = '，,、;；'
+def add_period(text):
+    """尾部无标点则添加句号；汉字结尾加'。'，否则加'.'。弱标点(，,、;；)结尾升格为句末标点"""
+    stripped = text.rstrip()
+    if not stripped:          # 空串或全空白
+        return "."
+    last_char = stripped[-1]
+    # 弱标点结尾（逗号/顿号/分号）：升格为句末标点
+    if last_char in WEAK_TAIL_PUNCT:
+        has_chinese = bool(re.search(r'[一-鿿]', stripped))   # U+4E00-U+9FFF 汉字区
+        return stripped[:-1] + ('。' if has_chinese else '.')
+    # 如果最后一个字符不是字母或数字，则认为它已是标点符号，直接返回
+    if not last_char.isalnum():
+        return stripped
+    # 判断是否为汉字（基本汉字区和扩展A区）
+    is_chinese = ('\u4e00' <= last_char <= '\u9fff' or
+                  '\u3400' <= last_char <= '\u4dbf')
+    suffix = '。' if is_chinese else '.'
+    return stripped + suffix
+def fallback_replace_non_alnum(text, replacement='，'):
+    """
+    兜底方案：将非汉字、英文、数字及基础TTS标点的字符替换为指定符号。
+    并合并连续的替换符号。
+    保留中文基础标点：，。！？；：、
+    保留英文基础标点：,.!?;:
+    保留小数点：.和空格等
+    """
+    # 白名单新增: () （） " “ ” (半角/全角括号 + 半角/中文双引号) 不再被替换成逗号
+    text = re.sub(r"[^\u4e00-\u9fffA-Za-z0-9，。！？；：、,.!?;:\. \u2019\u2018'\"()（）\u201C\u201D]", replacement, text)
+    text = re.sub(re.escape(replacement) + '+', replacement, text).strip()
+    return text
+def normalize_text_kit(text):
+    """
+    完整文本规范化流水线：
+    连续大写字母拆开（NBA → N B A）
+    (已禁用) 中英括号替换为逗号
+    兜底：非汉字/英文/数字/基础标点 → 逗号 (已把 () （） " “ ” 加入白名单)
+    英文标点→中文标点 + 标点去重 + 空格清理
+    """
+    #print(f'[normalize_text_kit] 输入                   : {text!r}')
+    # ---- 已禁用: 连续大写字母拆分（NBA → N B A）----
+    # text = split_uppercase_acronyms(text)
+    #print(f'[normalize_text_kit] step1 大写拆分后       : {text!r}')
+    # ---- 已禁用: 不再把括号替换成逗号 ----
+    # text = replace_parens_with_sep(text)
+    # print(f'[normalize_text_kit] step2 括号替换后       : {text!r}')
+    text = fallback_replace_non_alnum(text)
+    #print(f'[normalize_text_kit] step3 兜底非白名单→，  : {text!r}')
+    text = chinese_punc_en2cn(text)
+    #print(f'[normalize_text_kit] step4 英文标点→中文    : {text!r}')
+    text = re.sub(r'\s+', ' ', text).strip()
+    #print(f'[normalize_text_kit] step5 空格清理后       : {text!r}')
+    return text
+def strip_markdown_code(text):
+    """
+    清除 Markdown 行内代码标记（反引号）。
+    支持单反引号 `code` 和双反引号 ``code`` 两种格式。
+    """
+    # 双反引号优先匹配（避免单反引号先匹配到内层）
+    text = re.sub(r'``(.+?)``', r'\1', text)
+    # 单反引号
+    text = re.sub(r'`(.+?)`', r'\1', text)
+    # 清除残留的未配对反引号
+    text = text.replace('`', '')
+    return text
+def preprosess_text(text):
+    """
+    文本预处理：
+    清除 Markdown 行内代码标记
+    """
+    text = strip_markdown_code(text)
+    return text

talker/inference.py ADDED Viewed

	@@ -0,0 +1,443 @@

+"""Synchronous Ming Talker, adapted from talker_transformers_inference.
+Native prompt formatting, text normalization, CFM, DiT, Aggregator and AudioVAE
+are retained. Qwen2 uses the installed Transformers cache API; inference needs
+neither vLLM nor engine workers, CUDA graphs, thread pools or temporary files.
+"""
+from contextlib import nullcontext
+from functools import cached_property
+from pathlib import Path
+import random
+import re
+import numpy as np
+import onnxruntime
+from safetensors.torch import load_file
+import torch
+from torch import nn
+import torchaudio
+from torchaudio.compliance import kaldi
+from transformers import AutoTokenizer, Qwen2Config, Qwen2Model
+from .AudioVAE.configuration_audio_vae import AudioVAEconfig
+from .AudioVAE.modeling_audio_vae import AudioVAE
+from .configuration_bailing_talker import BailingTalkerConfig
+from .front.number_en import normalize_numbers
+from .front.text_segment_cut import cut_text_by_semantic_length, is_chinese
+from .front.toolkit import (
+    add_period,
+    normalize_text_kit,
+    preprosess_text,
+    tokenize_mixed_text_iterator,
+)
+from .talker_module.aggregator import Aggregator
+from .talker_module.cfm import CFM, get_epss_timesteps
+from .talker_module.dit import DiT
+from .talker_tn.talker_tn import TalkerTN
+from .tts_prompt import (
+    DEFAULT_TTS_PROMPT,
+    append_optional_target_language,
+    build_task_instruction,
+    contains_paralinguistic_tag,
+    preserve_multilingual_text,
+    protect_paralinguistic_tags,
+    resolve_request_language,
+    restore_paralinguistic_tags,
+    target_language_from_prompt,
+)
+class MingTalker(nn.Module):
+    def __init__(self, path: Path):
+        super().__init__()
+        self.path = path
+        self.config = BailingTalkerConfig.from_pretrained(path)
+        self.tokenizer = AutoTokenizer.from_pretrained(path / "llm")
+        llm_config = Qwen2Config.from_pretrained(path / "llm")
+        llm_config._attn_implementation = "sdpa"
+        self.model = Qwen2Model(llm_config)
+        self.cfm = CFM(
+            DiT(llm_cond_dim=llm_config.hidden_size, **self.config.flowmodel),
+            steps=self.config.steps,
+        )
+        self.aggregator = Aggregator(
+            llm_input_dim=llm_config.hidden_size, **self.config.aggregator
+        )
+        self.stop_head = nn.Linear(llm_config.hidden_size, 2)
+        self.spk_head = nn.Linear(192, llm_config.hidden_size)
+        self.normalizer = TalkerTN()
+        self.latent_dim = 64
+        self.patch_size = self.config.patch_size
+        self.history_patch_size = self.config.history_patch_size
+    @classmethod
+    def from_pretrained(cls, path, *, dtype=torch.bfloat16, device="cuda:0"):
+        path = Path(path).resolve()
+        model = cls(path).to(dtype=dtype)
+        weights = load_file(str(path / "model.safetensors"))
+        # Recent x-transformers computes RoPE frequencies instead of persisting them.
+        state_keys = model.state_dict().keys()
+        weights = {
+            key: value
+            for key, value in weights.items()
+            if not (key.endswith("rotary_embed.inv_freq") and key not in state_keys)
+        }
+        model.load_state_dict(weights, strict=True)
+        vae_config = AudioVAEconfig.from_pretrained(path / "vae")
+        vae_config._attn_implementation = "eager"
+        for section in (
+            vae_config.enc_kwargs["backbone"],
+            vae_config.dec_kwargs["backbone"],
+        ):
+            section["_attn_implementation"] = "eager"
+            section["attn_implementation"] = "eager"
+        model.audio_vae = AudioVAE(vae_config).to(dtype=dtype)
+        model.audio_vae.load_state_dict(
+            load_file(str(path / "vae/model.safetensors")), strict=True
+        )
+        return model.to(device).eval()
+    @property
+    def device(self):
+        return next(self.model.parameters()).device
+    @property
+    def dtype(self):
+        return next(self.model.parameters()).dtype
+    @cached_property
+    def speaker_encoder(self):
+        options = onnxruntime.SessionOptions()
+        options.graph_optimization_level = (
+            onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
+        )
+        options.intra_op_num_threads = 2
+        return onnxruntime.InferenceSession(
+            str(self.path / "campplus.onnx"),
+            sess_options=options,
+            providers=["CPUExecutionProvider"],
+        )
+    def reference_features(self, reference, continuation):
+        waveform = torch.as_tensor(reference["samples"], dtype=torch.float32).reshape(
+            1, -1
+        )
+        sample_rate = reference["sample_rate"]
+        speaker_waveform = torchaudio.functional.resample(waveform, sample_rate, 16000)
+        features = kaldi.fbank(
+            speaker_waveform, num_mel_bins=80, dither=0, sample_frequency=16000
+        )
+        features -= features.mean(dim=0, keepdim=True)
+        embedding = self.speaker_encoder.run(
+            None,
+            {self.speaker_encoder.get_inputs()[0].name: features.unsqueeze(0).numpy()},
+        )[0]
+        speaker = [
+            self.spk_head(
+                torch.as_tensor(
+                    embedding, device=self.device, dtype=self.dtype
+                ).reshape(1, 192)
+            )
+        ]
+        if not continuation:
+            return speaker, None, None
+        waveform = torchaudio.functional.resample(
+            waveform, sample_rate, self.audio_vae.config.sample_rate
+        )
+        alignment = (
+            self.audio_vae.encoder.hop_size
+            * max(1, self.audio_vae.encoder.patch_size)
+            * self.patch_size
+        )
+        waveform = nn.functional.pad(waveform, ((-waveform.shape[-1]) % alignment, 0))
+        latents, _ = self.audio_vae.encode_latent(
+            waveform.to(device=self.device, dtype=self.dtype),
+            torch.tensor([waveform.shape[-1]], device=self.device),
+        )
+        embeddings = self.aggregator(
+            latents.reshape(-1, self.patch_size, self.latent_dim)
+        )
+        return (
+            speaker,
+            latents,
+            embeddings.reshape(1, -1, self.model.config.hidden_size),
+        )
+    def text_fragments(self, value, max_length=50):
+        """Mirror the legacy TTS-family buffering before TN."""
+        streaming_text = []
+        count = 0
+        for index, ele in enumerate(tokenize_mixed_text_iterator(value)):
+            if len(ele) == 0:
+                continue
+            should_process = False
+            if ele[-1] in "！？。!?" and (
+                len(streaming_text) >= 12 or (count > 0 and len(streaming_text) >= 8)
+            ):
+                should_process = True
+                streaming_text.append(ele)
+            elif ele[-1] in "，," and len(streaming_text) >= 36:
+                should_process = True
+                streaming_text.append(ele)
+            elif (
+                ele[-1] == "."
+                and (
+                    len(streaming_text) >= 12
+                    or (count > 0 and len(streaming_text) >= 8)
+                )
+                and (not bool(re.search("[0-9]", streaming_text[-1][-1])))
+            ):
+                should_process = True
+                streaming_text.append(ele)
+            elif ele[-1] == "\n":
+                if len(streaming_text) > 0:
+                    if re.search("[\\u4e00-\\u9fff]", "".join(streaming_text)):
+                        if re.search("[\\u4e00-\\u9fff]", streaming_text[-1][-1]):
+                            ele = "，"
+                            streaming_text.append(ele)
+                    else:
+                        if len(ele) > 1 and re.search("[a-zA-Z]", ele[-2]):
+                            ele = ele[:-1] + "."
+                        else:
+                            ele = ele[:-1]
+                        streaming_text.append(ele)
+                if len(streaming_text) >= 12 or (
+                    count > 0 and len(streaming_text) >= 8
+                ):
+                    should_process = True
+            else:
+                streaming_text.append(ele)
+                continue
+            if should_process:
+                buffered_text = preprosess_text("".join(streaming_text))
+                text_list = cut_text_by_semantic_length(buffered_text, max_length)[
+                    "fragments"
+                ]
+                count += len(text_list)
+                yield from text_list
+                streaming_text = []
+        if streaming_text and re.search(
+            "[a-zA-Z\\u4e00-\\u9fff1-9]", "".join(streaming_text)
+        ):
+            buffered_text = preprosess_text("".join(streaming_text))
+            buffered_text = add_period(buffered_text)
+            yield from cut_text_by_semantic_length(buffered_text, max_length)[
+                "fragments"
+            ]
+    def input_embeddings(
+        self, text, prompt, instruction, speaker, prompt_text, prompt_embeddings
+    ):
+        encode = self.tokenizer.encode
+        speaker_tokens = []
+        for index, _ in enumerate(speaker or []):
+            speaker_tokens += (
+                encode(f"  speaker_{index + 1}:")
+                + encode("<|vision_start|>")
+                + encode("<|vision_pad|>")
+                + encode("<|vision_end|>\n")
+            )
+        continuation = prompt_embeddings is not None and prompt_text is not None
+        tokens = (
+            encode("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n")
+            + encode("<|im_start|>user\n")
+            + encode(prompt)
+            + speaker_tokens
+            + encode(" Text input:\n")
+            + (encode(prompt_text) if continuation else [])
+            + encode(text)
+            + encode("<|im_end|>\n")
+            + encode("<|im_start|>assistant\n")
+            + (encode(instruction) + encode("<|im_end|>") if instruction else [])
+            + encode("<audio>")
+            + (
+                encode("<audioPatch>") * prompt_embeddings.shape[1]
+                if continuation
+                else []
+            )
+        )
+        ids = torch.tensor([tokens], device=self.device)
+        embeddings = self.model.get_input_embeddings()(ids)
+        if speaker:
+            positions = torch.where(ids[0] == encode("<|vision_start|>")[0])[0]
+            for position, embedding in zip(positions, speaker):
+                embeddings[0, position + 1] = embedding
+        if continuation:
+            start = torch.where(ids[0] == encode("<audio>")[0])[0][0] + 1
+            embeddings[0, start : start + prompt_embeddings.shape[1]] = (
+                prompt_embeddings[0]
+            )
+        return embeddings
+    def generate_latents(
+        self, embeddings, prompt_latents, max_steps, cfg, sigma, temperature
+    ):
+        history = embeddings.new_zeros(1, self.history_patch_size, self.latent_dim)
+        if prompt_latents is not None:
+            tail = prompt_latents[:, -self.history_patch_size :]
+            history[:, -tail.shape[1] :] = tail
+        past = None
+        latents = []
+        for step in range(max_steps):
+            output = self.model(
+                inputs_embeds=embeddings, past_key_values=past, use_cache=True
+            )
+            past = output.past_key_values
+            hidden = output.last_hidden_state[:, -1:]
+            noise = torch.randn(
+                (1, self.patch_size, self.latent_dim),
+                device=self.device,
+                dtype=self.dtype,
+            )
+            timesteps = get_epss_timesteps(self.config.steps, self.device, self.dtype)
+            sde_noise = torch.randn(
+                (self.config.steps, *noise.shape), device=self.device, dtype=self.dtype
+            )
+            latent = self.cfm.sample(
+                hidden,
+                history,
+                noise,
+                timesteps,
+                hidden.new_tensor([cfg, sigma, temperature]),
+                sde_noise,
+            )
+            embeddings = self.aggregator(latent)
+            latents.append(latent)
+            history = torch.cat([history, latent], dim=1)[:, -self.history_patch_size :]
+            stop = self.stop_head(hidden[:, -1]).softmax(dim=-1)[0, 1]
+            if step > 3 and stop.item() > 0.5:
+                break
+        return torch.cat(latents, dim=1)
+    @torch.inference_mode()
+    def synthesize(
+        self,
+        text,
+        *,
+        ip_name="灵小柔",
+        reference=None,
+        prompt_text=None,
+        language=None,
+        max_steps=1000,
+        seed=None,
+        cfg=2.0,
+        sigma=0.25,
+        temperature=0.0,
+    ):
+        context = (
+            torch.cuda.device(self.device)
+            if self.device.type == "cuda"
+            else nullcontext()
+        )
+        with context, torch.autocast(device_type=self.device.type, dtype=self.dtype):
+            if seed is not None:
+                random.seed(seed)
+                np.random.seed(seed)
+                torch.manual_seed(seed)
+            text = re.sub(r"^[A-Za-z]{1,16}\t", "", text)
+            task = "IP" if reference is None else "TTS"
+            language, text = resolve_request_language(
+                text, language=language, taskname=task
+            )
+            prompt = append_optional_target_language(DEFAULT_TTS_PROMPT, language)
+            instruction = (
+                build_task_instruction("IP", ip=ip_name) if reference is None else None
+            )
+            skip_tn = preserve_multilingual_text(target_language_from_prompt(prompt))
+            speaker, prompt_latents, prompt_embeddings = (
+                self.reference_features(reference, prompt_text is not None)
+                if reference is not None
+                else (None, None, None)
+            )
+            chunks = []
+            for fragment in self.text_fragments(text):
+                protected = contains_paralinguistic_tag(fragment)
+                fragment = (
+                    protect_paralinguistic_tags(fragment) if protected else fragment
+                )
+                if not skip_tn:
+                    if not is_chinese(fragment):
+                        fragment = normalize_numbers(fragment)
+                    fragment = normalize_text_kit(
+                        self.normalizer.normalize(fragment)
+                    ).lstrip("，")
+                if protected:
+                    fragment = restore_paralinguistic_tags(fragment)
+                embeddings = self.input_embeddings(
+                    fragment,
+                    prompt,
+                    instruction,
+                    speaker,
+                    prompt_text,
+                    prompt_embeddings,
+                )
+                latents = self.generate_latents(
+                    embeddings, prompt_latents, max_steps, cfg, sigma, temperature
+                )
+                waveform, _, _ = self.audio_vae.decode(
+                    latents, use_cache=False, last_chunk=True
+                )
+                chunks.append(
+                    self.silence_holder(
+                        waveform.detach().float().cpu(),
+                        self.audio_vae.config.sample_rate,
+                    )[0]
+                )
+            return {
+                "audio": torch.cat(chunks, dim=-1),
+                "sr": self.audio_vae.config.sample_rate,
+            }
+    @staticmethod
+    def silence_holder(
+        speech, sample_rate, sil_cache=None, last_chunk=True, sil_th=1e-3, last_sil=0.3
+    ):
+        if speech.numel() == 0:
+            assert not last_chunk
+            return speech, sil_cache
+        frame_step, frame_size = int(sample_rate * 0.1), int(sample_rate * 0.1)
+        if sil_cache is None:
+            sil_cache = {"holder": [], "buffer": []}
+        if sil_cache["buffer"]:
+            speech = torch.cat([*sil_cache["buffer"], speech], dim=-1)
+            sil_cache["buffer"] = []
+        if speech.shape[-1] < frame_size:
+            sil_cache["buffer"].append(speech)
+            if last_chunk:
+                speech = torch.cat(sil_cache["holder"] + sil_cache["buffer"], dim=-1)
+                return speech[..., : int(last_sil * sample_rate)], sil_cache
+            return torch.zeros(
+                (*speech.shape[:-1], 0), device=speech.device, dtype=speech.dtype
+            ), sil_cache
+        num_frame = (speech.shape[-1] - frame_size) // frame_step + 1
+        cur_len = (num_frame - 1) * frame_step + frame_size
+        if speech.shape[-1] > cur_len:
+            sil_cache["buffer"].append(speech[..., cur_len:])
+            speech = speech[..., :cur_len]
+        spe_frames = speech.unfold(-1, frame_size, frame_step)
+        scores = spe_frames.abs().mean(dim=-1)
+        scores = scores.mean(dim=list(range(scores.dim() - 1)))
+        idx = scores.shape[0] - 1
+        while idx >= 0:
+            if scores[idx] > sil_th:
+                break
+            idx -= 1
+        if idx < 0:
+            sil_cache["holder"].append(speech)
+            if last_chunk:
+                speech = torch.cat(sil_cache["holder"] + sil_cache["buffer"], dim=-1)
+                return speech[..., : int(last_sil * sample_rate)], sil_cache
+            return torch.zeros(
+                (*speech.shape[:-1], 0), device=speech.device, dtype=speech.dtype
+            ), sil_cache
+        non_sil_len = idx * frame_step + frame_size
+        if last_chunk:
+            non_sil_len += int(last_sil * sample_rate)
+        speech = torch.cat([*sil_cache["holder"], speech[..., :non_sil_len]], dim=-1)
+        sil_cache["holder"] = []
+        if non_sil_len < speech.shape[-1]:
+            sil_cache["holder"].append(speech[..., non_sil_len:])
+        return speech, sil_cache

talker/llm/added_tokens.json ADDED Viewed

	@@ -0,0 +1,75 @@

+{
+  "</answer>": 151691,
+  "</audio>": 151707,
+  "</body>": 151681,
+  "</gen_audio>": 151710,
+  "</gen_image>": 151701,
+  "</html>": 151679,
+  "</image>": 151696,
+  "</prompt>": 151672,
+  "</table>": 151683,
+  "</td>": 151687,
+  "</think>": 151689,
+  "</thinker_prefix>": 151674,
+  "</tool_call>": 151658,
+  "</tr>": 151685,
+  "</video>": 151698,
+  "</vp>": 151676,
+  "<answer>": 151690,
+  "<asr>": 151714,
+  "<audio>": 151706,
+  "<audioHere>": 151711,
+  "<audioPatch>": 151705,
+  "<audio_bos>": 151665,
+  "<audio_eos>": 151666,
+  "<audio_pad>": 151667,
+  "<body>": 151680,
+  "<end_of_audio>": 151704,
+  "<end_of_chunk>": 151703,
+  "<framePatch>": 151712,
+  "<gen_audio>": 151709,
+  "<gen_audioPatch>": 151708,
+  "<gen_image>": 151700,
+  "<gen_imagePatch>": 151699,
+  "<html>": 151678,
+  "<image>": 151695,
+  "<imageHere>": 151702,
+  "<imagePatch>": 151694,
+  "<prompt>": 151671,
+  "<table>": 151682,
+  "<td>": 151686,
+  "<text>": 151713,
+  "<text_bos>": 151668,
+  "<text_eos>": 151669,
+  "<text_pad>": 151670,
+  "<think>": 151688,
+  "<thinker_prefix>": 151673,
+  "<tool_call>": 151657,
+  "<tr>": 151684,
+  "<tts>": 151715,
+  "<video>": 151697,
+  "<vp>": 151675,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|startoftext|>": 151692,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652,
+  "[gMASK]": 151693,
+  "[item]": 151677
+}

talker/llm/config.json ADDED Viewed

	@@ -0,0 +1,27 @@

+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

talker/llm/generation_config.json ADDED Viewed

	@@ -0,0 +1,14 @@

+{
+  "bos_token_id": 151643,
+  "pad_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_p": 0.8,
+  "top_k": 20,
+  "transformers_version": "4.37.0"
+}

talker/llm/merges.txt ADDED Viewed

The diff for this file is too large to render. See raw diff

talker/llm/special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,137 @@

+{
+  "additional_special_tokens": [
+    {
+      "content": "[item]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<html>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</html>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<body>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</body>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<answer>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "</answer>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[gMASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

talker/llm/tokenizer.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:87d54ca3b44ea2f822ba39a0a45cb1b61644bd6606d9d3536dea75367bffb06d
+size 11431389

talker/llm/tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,619 @@

+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<audio_bos>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "<audio_eos>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<audio_pad>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "<text_bos>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151669": {
+      "content": "<text_eos>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151670": {
+      "content": "<text_pad>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151671": {
+      "content": "<prompt>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151672": {
+      "content": "</prompt>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151673": {
+      "content": "<thinker_prefix>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151674": {
+      "content": "</thinker_prefix>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151675": {
+      "content": "<vp>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151676": {
+      "content": "</vp>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151677": {
+      "content": "[item]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151678": {
+      "content": "<html>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151679": {
+      "content": "</html>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151680": {
+      "content": "<body>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151681": {
+      "content": "</body>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151682": {
+      "content": "<table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151683": {
+      "content": "</table>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151684": {
+      "content": "<tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151685": {
+      "content": "</tr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151686": {
+      "content": "<td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151687": {
+      "content": "</td>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151688": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151689": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151690": {
+      "content": "<answer>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151691": {
+      "content": "</answer>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151692": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151693": {
+      "content": "[gMASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151694": {
+      "content": "<imagePatch>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151695": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151696": {
+      "content": "</image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151697": {
+      "content": "<video>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151698": {
+      "content": "</video>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151699": {
+      "content": "<gen_imagePatch>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151700": {
+      "content": "<gen_image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151701": {
+      "content": "</gen_image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151702": {
+      "content": "<imageHere>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151703": {
+      "content": "<end_of_chunk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151704": {
+      "content": "<end_of_audio>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151705": {
+      "content": "<audioPatch>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151706": {
+      "content": "<audio>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151707": {
+      "content": "</audio>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151708": {
+      "content": "<gen_audioPatch>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151709": {
+      "content": "<gen_audio>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151710": {
+      "content": "</gen_audio>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151711": {
+      "content": "<audioHere>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151712": {
+      "content": "<framePatch>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151713": {
+      "content": "<text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151714": {
+      "content": "<asr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151715": {
+      "content": "<tts>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "[item]",
+    "<html>",
+    "</html>",
+    "<body>",
+    "</body>",
+    "<table>",
+    "</table>",
+    "<tr>",
+    "</tr>",
+    "<td>",
+    "</td>",
+    "<think>",
+    "</think>",
+    "<answer>",
+    "</answer>",
+    "<|startoftext|>",
+    "[gMASK]"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

talker/llm/vocab.json ADDED Viewed

The diff for this file is too large to render. See raw diff

talker/model.safetensors ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:7df7677cf9e9d76ecd3d8ea07606bb04249c72010f71827dd3daca3dfa7fd040
+size 1398191068