INNER CODE UNIT · Python
parse_dataset_name_and_ratio
HIT-SCIR/Chinese-Mixtral-8x7B · data/utils.py:9
def parse_dataset_name_and_ratio(datasets: List[str]):
# Parse ["0.1:wudao", "0.2:slimpajama"] into {"wudao": 0.1, "slimpajama": 0.2}
dataset_name_and_ratio = {}
for dataset in datasets:
ratio, name = dataset.split(":")
dataset_name_and_ratio[name] = float(ratio)
return dataset_name_and_ratio
def count_token(
dataset_name2dsNratio: Dict[str, Tuple[Dataset, float]],
sequence_length: int = 2048,
):
statistics = {}
for ds_name, (ds, ratio) in dataset_name2dsNratio.items():
statistics[ds_name] = len(ds) * ratio * sequence_length / 10 ** 9