[FIX] Fix bug in finetune.py
This commit is contained in:
parent
cf5e7db067
commit
1720dfa670
|
@ -123,7 +123,7 @@ def train(
|
|||
)
|
||||
tokenizer.padding_side = "left" # Allow batched inference
|
||||
|
||||
def tokenize(prompt):
|
||||
def tokenize(prompt, add_eos_token=True):
|
||||
# there's probably a way to do this with the tokenizer settings
|
||||
# but again, gotta move fast
|
||||
result = tokenizer(
|
||||
|
|
Loading…
Reference in New Issue