INNER CODE UNIT · Python
count_token
HIT-SCIR/Chinese-Mixtral-8x7B · data/utils.py:18
def count_token(
dataset_name2dsNratio: Dict[str, Tuple[Dataset, float]],
sequence_length: int = 2048,
):
statistics = {}
for ds_name, (ds, ratio) in dataset_name2dsNratio.items():
statistics[ds_name] = len(ds) * ratio * sequence_length / 10 ** 9
return statistics
def count_all_token(
sequence_length: int = 2048,
):
# TODO refactor with `count_token`
statistics = {}