p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Solve MASS codex warning.

This commit is contained in:
liuchongming74 2020-06-29 16:19:23 +08:00
parent 2f1b0dc531
commit 490f949617
1 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
model_zoo/mass/src/utils/byte_pair_encoding.py
View File

@ -16,8 +16,8 @@
import os import os
import subprocess import subprocess
ENCODER = "subword-nmt apply-bpe -c {codes} -i {input} -o {output}" ENCODER = "subword-nmt apply-bpe -c"
LEARN_DICT = "subword-nmt get-vocab -i {input} -o {dict_path}" LEARN_DICT = "subword-nmt get-vocab -i"
def bpe_encode(codes_path, src_path, output_path, dict_path): def bpe_encode(codes_path, src_path, output_path, dict_path):
@ -43,10 +43,10 @@ def bpe_encode(codes_path, src_path, output_path, dict_path):
raise FileNotFoundError("Dir not found.") raise FileNotFoundError("Dir not found.")
# Encoding. # Encoding.
print(f" | Applying BPE encoding.") print(" | Applying BPE encoding.")
subprocess.call(ENCODER.format(codes=codes_path, input=src_path, output=output_path), commands = ENCODER.split() + [codes_path] + ["-i"] + [src_path] + ["-o"] + [output_path]
shell=True) subprocess.call(commands)
print(f" | Fetching vocabulary from single file.") print(" | Fetching vocabulary from single file.")
# Learn vocab. # Learn vocab.
subprocess.call(LEARN_DICT.format(input=output_path, dict_path=dict_path), commands = LEARN_DICT.split() + [output_path] + ["-o"] + [dict_path]
shell=True) subprocess.call(commands)