INNER CODE UNIT · Python
pre_puct_position
R1j1t/contextualSpellCheck · contextualSpellCheck/contextualSpellCheck.py:600
pre_puct_position = -1
for char_position in range(text_len):
if unicodedata.category(text[char_position]).startswith("P"):
# print("current_pos is {} and sub_token append {}"
# .format(char_position,text[char_position]))
sub_tokens.append(text[char_position])
# print("pre_pos is {}, cur is {} , pre to current is {}"
# .format(pre_puct_position,char_position,text[pre_puct_position+1:char_position]))
if (
pre_puct_position >= 0
and text[pre_puct_position + 1 : char_position] != ""
):
# print("pre_pos is {}, cur is {} , pre to current is {}"
# .format(pre_puct_position,char_position,text[pre_puct_position+1:char_position]))
sub_tokens.append(
text[pre_puct_position + 1 : char_position]
)
pre_puct_position = char_position