[FIX] Fix bug in finetune.py

2023-05-23 16:57:44 +08:00 · 2023-05-23 16:57:44 +08:00 · 1720dfa670
parent cf5e7db067
commit 1720dfa670
1 changed files with 1 additions and 1 deletions
--- a/finetune.py
+++ b/finetune.py
@ -123,7 +123,7 @@ def train(
    )
    tokenizer.padding_side = "left"  # Allow batched inference

-    def tokenize(prompt):
+    def tokenize(prompt, add_eos_token=True):
        # there's probably a way to do this with the tokenizer settings
        # but again, gotta move fast
        result = tokenizer(