# Invalid Label value Error coming during Squad2.0 fine tuning

**URL:** <https://discuss.huggingface.co/t/invalid-label-value-error-coming-during-squad2-0-fine-tuning/2818>\
**Category:** Beginners\
**Created:** [December 14, 2020, 1:28pm UTC](https://discuss.huggingface.co/t/invalid-label-value-error-coming-during-squad2-0-fine-tuning/2818 "2020-12-14T13:28:51Z")\
**Posts on this page:** 1\
**Page:** 1

<div class="post-metadata">

**Author:** ![dearc](https://sea2.discourse-cdn.com/hellohellohello/user_avatar/discuss.huggingface.co/dearc/32/826_2.png) [@dearc](https://discuss.huggingface.co/u/dearc)\
**Post date:** [December 14, 2020, 1:28pm UTC](https://discuss.huggingface.co/t/invalid-label-value-error-coming-during-squad2-0-fine-tuning/2818/1 "2020-12-14T13:28:51Z")

</div>

Hello Everyone, I am new to this forum. Please be patient with me.

I was trying to finetune the Squad2.0 dataset as per instructions shared in [HF custom dataset page](https://huggingface.co/transformers/master/custom_datasets.html#qa-squad) (TF version of the instructions).

Even though I followed everystep to the last detail the  
`model.fit(train_dataset.shuffle(1000).batch(16), epochs=3, batch_size=16)` step is failing with the below error

 ![image](https://us1.discourse-cdn.com/hellohellohello/original/2X/8/8520c08ae5f9c642beff8d47c94a1d25a231ffee.png)

I did check the `train_answers` everyquestion has a required answer, so not sure what I am missing here.

The steps are exactly same as mentioned in the docs:

```
import transformers
import tensorflow as tf
from transformers import TFDistilBertForQuestionAnswering

from pathlib import Path

def read_squad(path):

    path = Path(path)
    with open(path, 'rb') as f:
        squad_dict = json.load(f)

    contexts = []
    questions = []
    answers = []
    for group in squad_dict['data']:
        for passage in group['paragraphs']:
            context = passage['context']
            for qa in passage['qas']:
                question = qa['question']
                for answer in qa['answers']:
                    contexts.append(context)
                    questions.append(question)
                    answers.append(answer)

    return contexts, questions, answers

train_contexts, train_questions, train_answers = read_squad('squad/train-v2.0.json')
val_contexts, val_questions, val_answers = read_squad('squad/dev-v2.0.json')

def add_end_idx(answers, contexts):
      for answer, context in zip(answers, contexts):
        gold_text = answer['text']
        start_idx = answer['answer_start']
        end_idx = start_idx + len(gold_text)

        # sometimes squad answers are off by a character or two – fix this
        if context[start_idx:end_idx] == gold_text:
            answer['answer_end'] = end_idx
        elif context[start_idx-1:end_idx-1] == gold_text:
            answer['answer_start'] = start_idx - 1
            answer['answer_end'] = end_idx - 1 # When the gold label is off by one character
        elif context[start_idx-2:end_idx-2] == gold_text:
            answer['answer_start'] = start_idx - 2
            answer['answer_end'] = end_idx - 2 # When the gold label is off by two characters

add_end_idx(train_answers, train_contexts)
add_end_idx(val_answers, val_contexts)

from transformers import DistilBertTokenizerFast
tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')

train_encodings = tokenizer(train_contexts, train_questions, truncation=True, padding=True)
val_encodings = tokenizer(val_contexts, val_questions, truncation=True, padding=True)

def add_token_positions(encodings, answers):
    start_positions = []
    end_positions = []
    for i in range(len(answers)):
        start_positions.append(encodings.char_to_token(i, answers[i]['answer_start']))
        end_positions.append(encodings.char_to_token(i, answers[i]['answer_end'] - 1))
        # if None, the answer passage has been truncated
        if start_positions[-1] is None:
            start_positions[-1] = tokenizer.model_max_length
        if end_positions[-1] is None:
            end_positions[-1] = tokenizer.model_max_length
    encodings.update({'start_positions': start_positions, 'end_positions': end_positions})

add_token_positions(train_encodings, train_answers)
add_token_positions(val_encodings, val_answers)

train_dataset = tf.data.Dataset.from_tensor_slices((
    {key: train_encodings[key] for key in ['input_ids', 'attention_mask']},
    {key: train_encodings[key] for key in ['start_positions', 'end_positions']}
))
val_dataset = tf.data.Dataset.from_tensor_slices((
    {key: val_encodings[key] for key in ['input_ids', 'attention_mask']},
    {key: val_encodings[key] for key in ['start_positions', 'end_positions']}
))

model = TFDistilBertForQuestionAnswering.from_pretrained("distilbert-base-uncased")

# Keras will expect a tuple when dealing with labels
train_dataset = train_dataset.map(lambda x, y: (x, (y['start_positions'], y['end_positions'])))

loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
model.distilbert.return_dict = False # if using 🤗 Transformers >3.02, make sure outputs are tuples

optimizer = tf.keras.optimizers.Adam(learning_rate=5e-5)
model.compile(optimizer=optimizer, loss=loss) # can also use any keras loss fn
model.fit(train_dataset.shuffle(1000).batch(16), epochs=3, batch_size=16)

```