INNER CODE UNIT · Python
build_dataset
HIT-SCIR/Chinese-Mixtral-8x7B · train.py:102
def build_dataset(train_datasets: List[str], valid_datasets: List[str]):
with open("./data/datasets.toml", "rb") as f:
ds_info = tomli.load(f)
train_data_name2ratio = parse_dataset_name_and_ratio(train_datasets) # {"wudao": 0.1, "slimpajama": 0.2}
train_data_name2pathNratio = {} # {"wudao": ("/path/to/wudao", 0.1), "slimpajama": ("/path/to/slimpajama", 0.2)}
for ds_name, ratio in train_data_name2ratio.items():
ds = ds_info[ds_name]
assert "train" in ds["splits"]
train_data_name2pathNratio[ds_name] = (ds["root"].format(DATA_DIR="./data", name=ds_name) + "/" +
ds["encoded"].format(name=ds_name, split="train"),
ratio)
valid_data_name2path = {} # {"wudao": "/path/to/wudao", "slimpajama": "/path/to/slimpajama"}
for ds_name in valid_datasets:
ds = ds_info[ds_name]
assert "dev" in ds["splits"]
valid_data_name2path[ds_name] = (ds["root"].format(DATA_DIR="./data", name=ds_name) + "/" +