fix: Change the tokenizer to accept quotes

This commit is contained in:
Clément Renault 2018-10-17 17:00:49 +02:00
parent af91bfa11f
commit 98899d3ea0
1 changed files with 2 additions and 2 deletions

View File

@ -24,7 +24,7 @@ impl<'a> Tokens<'a> {
fn new(string: &str) -> Tokens {
Tokens {
index: 0,
inner: string.trim_matches(&[' ', '.', ';', ',', '!', '?', '-'][..]),
inner: string.trim_matches(&[' ', '.', ';', ',', '!', '?', '-', '\'', '"'][..]),
}
}
}
@ -62,7 +62,7 @@ impl<'a> Iterator for Tokens<'a> {
for (i, c) in self.inner.char_indices() {
let separator = match c {
'.' | ';' | ',' | '!' | '?' | '-' => Some(Long),
' ' => Some(Short),
' ' | '\'' | '"' => Some(Short),
_ => None,
};