안녕하세요. 답장이 늦어 죄송합니다. 평가는 ko_ref와 model prediction에 대해서 ScareBLEU를 사용하였습니다.
실험에 사용한 추론코드와 평가 코드 공유드립니다. 감사합니다.

python inference_translation_eeve.py -g 3 -d "eval_dataset/flores.csv" -m "yanolja/EEVE-Korean-Instruct-10.8B-v1.0"
python inference_translation_seagull.py -g 3 -d "eval_dataset/flores.csv" -m "kuotient/Seagull-13b-translation"
python inference_translation_kullm.py -g 3 -d "eval_dataset/flores.csv" -m "nlpai-lab/KULLM3"
python inference_translation_synatra.py -g 3 -d "eval_dataset/flores.csv" -m "maywell/Synatra-7B-v0.3-Translation"

# python inference_translation_base.py
import os
import argparse

parser = argparse.ArgumentParser()
parser.add_argument("-m", "--model_name", type=str, default="meta-llama/Meta-Llama-3-8B-Instruct")
parser.add_argument("-d", "--dataset_path", type=str,  default="gemini/ko-eng-dataset.csv")
parser.add_argument("-g", "--gpu_id", type=int,  default=0)
args = parser.parse_args()
print(args)
os.environ["CUDA_VISIBLE_DEVICES"]=str(args.gpu_id)

import torch
import evaluate
import pandas as pd

from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained(args.model_name)
# tokenizer.pad_token = tokenizer.eos_token
model = AutoModelForCausalLM.from_pretrained(
    args.model_name,
    # device_map="auto",
    torch_dtype=torch.bfloat16,
).to('cuda')
model.eval()

def apply_template(example):
    SYSTEM_PROMPT=f"당신은 번역기 입니다. 영어를 한국어로 번역하세요." # ours
    conversation = {"messages": [
                        {'role': 'system', 'content': SYSTEM_PROMPT},
                        {'role': 'user', 'content':example["en_ref"]}
                    ]}
    return conversation

# datasets
tc_dataset = load_dataset("csv", data_files=args.dataset_path, split="train")
dataset = tc_dataset.map(apply_template, remove_columns=tc_dataset.features, batched=False, num_proc=64)
print(dataset)

# inference
output_list = []
for idx, data in enumerate(dataset):
    inputs = tokenizer.apply_chat_template(data['messages'],tokenize=True, add_generation_prompt=True, return_tensors='pt').to("cuda")
    # print(tokenizer.batch_decode(inputs))
    outputs = model.generate(inputs,
                             pad_token_id=tokenizer.eos_token_id,
                             max_new_tokens=512)
    output_decode = tokenizer.decode(outputs[0][len(inputs[0]):], skip_special_tokens=True)
    print(f'{idx}:',output_decode)
    output_list.append(output_decode)

df = pd.DataFrame(tc_dataset)
df['ko_pred']=output_list
df = df[['ko_pred', 'ko_ref', 'en_ref', 'source']]

model_name = args.model_name.split('/')[-1]
output_path = 'inference_' + args.dataset_path.split('.')[-2]
print(output_path)
os.makedirs(output_path, exist_ok=True)
df.to_json(f'{output_path}/{model_name}_eval_result.json', lines=True, orient='records', force_ascii=False)

python eval_translation.py -i inference_eval_dataset/ko_news_eval40/nllb-finetuned-en2ko_eval_result.json
python eval_translation.py -i inference_eval_dataset/ko_news_eval40/EEVE-Korean-Instruct-10.8B-v1.0_eval_result.json
python eval_translation.py -i inference_eval_dataset/ko_news_eval40/Synatra-7B-v0.3-Translation_eval_result.json
python eval_translation.py -i inference_eval_dataset/ko_news_eval40/KULLM3_eval_result.json

# python eval_translation.py
import os
import argparse
import evaluate
import pandas as pd

parser = argparse.ArgumentParser()
parser.add_argument("-i", "--inference_path", type=str,  default="result/nayohanllama3-8b-it-translation-271k_eval_result.json")
args = parser.parse_args()
print(args)

# evaluate sacrebleu
metric = evaluate.load("sacrebleu")
def compute_metrics(eval_preds):
    decoded_preds, decoded_labels = eval_preds
    result = metric.compute(predictions=decoded_preds, references=decoded_labels)
    result = {"bleu": result["score"]}
    result = {k: round(v, 2) for k, v in result.items()}
    return result

# eval result to json
df = pd.read_json(args.inference_path, lines=True, orient='records')
result = []
for source in df['source'].unique():
    df_source = df[df['source']==source].reset_index(drop=True)
    eval_preds = [df_source['ko_pred'], df_source['ko_ref']]
    eval_result = compute_metrics(eval_preds)
    # print(eval_result)
    eval_result['source'] = source
    result.append(eval_result)

output_df = pd.DataFrame(result, columns=['source', 'bleu'])
output_df = output_df.sort_values(by=['source'])
print(output_df)
output_path = '/'.join(args.inference_path.split('/')[:-1]) + '/eval'
output_file = args.inference_path.split('/')[-1]
os.makedirs(output_path, exist_ok=True)
output_df.to_json(f'{output_path}/{output_file}', lines=True, orient='records', force_ascii=False)

make_eval_dataset.py

import pandas as pd
from datasets import load_dataset

# flores
eval_dataset = load_dataset('traintogpb/aihub-flores-koen-integrated-sparta-30k')
df = pd.DataFrame(eval_dataset['test'])
df = df.drop('ko_ref_xcomet', axis=1)
df.to_csv('eval_dataset/flores.csv', index=False)

# iwlst2023
iwlst_en_ko_ban = load_dataset('shreevigneshs/iwslt-2023-en-ko-train-val-split-0.1', split='f_test')
iwlst_en_ko_zon = load_dataset('shreevigneshs/iwslt-2023-en-ko-train-val-split-0.1', split='if_test')

df = iwlst_en_ko_ban.to_pandas()
df = df[["en", "ko"]]
df.columns=["en_ref", "ko_ref"]
df['source'] = 'iwlst_en_ko_ban'
df.to_csv('iwlst_en_ko_banmal.csv', index=False)#, encoding='utf-8-sig')
print(df)

df = iwlst_en_ko_zon.to_pandas()
df = df[["en", "ko"]]
df.columns=["en_ref", "ko_ref"]
df['source'] = 'iwlst_en_ko_zon'
df.to_csv('iwlst_en_ko_zondae.csv', index=False)#, encoding='utf-8-sig')
print(df)