Use BOS token as a fallback

huggingface · Apr 30, 2024 · a01cc68 · a01cc68
1 parent 6bc93c7
commit a01cc68
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 0 deletions.
diff --git a/examples/pytorch/question-answering/run_qa.py b/examples/pytorch/question-answering/run_qa.py
@@ -436,6 +436,8 @@ def prepare_train_features(examples):
  input_ids = tokenized_examples["input_ids"][i]
  if tokenizer.cls_token_id in input_ids:
  cls_index = input_ids.index(tokenizer.cls_token_id)
+ elif tokenizer.bos_token_id in input_ids:
+ cls_index = input_ids.index(tokenizer.bos_token_id)
  else:
  cls_index = 0
 

diff --git a/examples/pytorch/question-answering/run_qa_beam_search.py b/examples/pytorch/question-answering/run_qa_beam_search.py
@@ -419,6 +419,8 @@ def prepare_train_features(examples):
  input_ids = tokenized_examples["input_ids"][i]
  if tokenizer.cls_token_id in input_ids:
  cls_index = input_ids.index(tokenizer.cls_token_id)
+ elif tokenizer.bos_token_id in input_ids:
+ cls_index = input_ids.index(tokenizer.bos_token_id)
  else:
  cls_index = 0
  tokenized_examples["cls_index"].append(cls_index)
@@ -539,6 +541,8 @@ def prepare_validation_features(examples):
  # Find the CLS token in the input ids.
  if tokenizer.cls_token_id in input_ids:
  cls_index = input_ids.index(tokenizer.cls_token_id)
+ elif tokenizer.bos_token_id in input_ids:
+ cls_index = input_ids.index(tokenizer.bos_token_id)
  else:
  cls_index = 0
  tokenized_examples["cls_index"].append(cls_index)

diff --git a/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py b/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py
@@ -446,6 +446,8 @@ def prepare_train_features(examples):
  input_ids = tokenized_examples["input_ids"][i]
  if tokenizer.cls_token_id in input_ids:
  cls_index = input_ids.index(tokenizer.cls_token_id)
+ elif tokenizer.bos_token_id in input_ids:
+ cls_index = input_ids.index(tokenizer.bos_token_id)
  else:
  cls_index = 0
  tokenized_examples["cls_index"].append(cls_index)
@@ -568,6 +570,8 @@ def prepare_validation_features(examples):
  # Find the CLS token in the input ids.
  if tokenizer.cls_token_id in input_ids:
  cls_index = input_ids.index(tokenizer.cls_token_id)
+ elif tokenizer.bos_token_id in input_ids:
+ cls_index = input_ids.index(tokenizer.bos_token_id)
  else:
  cls_index = 0
  tokenized_examples["cls_index"].append(cls_index)

diff --git a/examples/pytorch/question-answering/run_qa_no_trainer.py b/examples/pytorch/question-answering/run_qa_no_trainer.py
@@ -515,6 +515,8 @@ def prepare_train_features(examples):
  input_ids = tokenized_examples["input_ids"][i]
  if tokenizer.cls_token_id in input_ids:
  cls_index = input_ids.index(tokenizer.cls_token_id)
+ elif tokenizer.bos_token_id in input_ids:
+ cls_index = input_ids.index(tokenizer.bos_token_id)
  else:
  cls_index = 0