INNER CODE UNIT · Python

main

FoundationVision/Liquid · data_process/convert_DCLM_data.py:19

def main(args):
    input_path = args.input_path
    if not os.path.exists(args.temp_path):
        os.makedirs(args.temp_path)
        
    files_list = []
    for dirpath, dirnames, filenames in os.walk(input_path):
        for file in filenames:
            if file.endswith('zst'):
                full_path = os.path.join(dirpath, file)
                files_list.append(full_path)
    files_list.sort()
    # files_list = files_list[:10] 
    for idx in tqdm(range(len(files_list))):
        filepath = files_list[idx]
        alldata_in_one_path = list(read_jsonl_zst(filepath))
        all_datas = []
        for data in alldata_in_one_path:

View source record →