INNER CODE UNIT · Python
_unit_info
liyucheng09/Selective_Context · app/app.py:134
def _unit_info(tokens, self_info, units):
current_unit_idx = 0
current_position = 0
unit_self_info = [[] for _ in range(len(units))]
for idx, (token, info) in enumerate(zip(tokens, self_info)):
current_position += len(token)
if current_position == len(units[current_unit_idx]):
unit_self_info[current_unit_idx].append(info)
current_position = current_position - len(units[current_unit_idx])
current_unit_idx += 1
elif current_position > len(units[current_unit_idx]):
counter_ = 1
current_position = current_position - len(units[current_unit_idx])
current_unit_idx += 1
while current_position >= len(units[current_unit_idx]):
counter_ += 1
current_position = current_position - len(units[current_unit_idx])