INNER CODE UNIT · Python

read_jsonl_zst

FoundationVision/Liquid · data_process/convert_DCLM_data.py:10

def read_jsonl_zst(file_path):
    with open(file_path, 'rb') as file:
        decompressor = zstd.ZstdDecompressor()
        stream_reader = decompressor.stream_reader(file)
        stream = io.TextIOWrapper(stream_reader, encoding = "utf-8")
        for line in stream:
            yield json.loads(line)


def main(args):
    input_path = args.input_path
    if not os.path.exists(args.temp_path):
        os.makedirs(args.temp_path)
        
    files_list = []
    for dirpath, dirnames, filenames in os.walk(input_path):
        for file in filenames:
            if file.endswith('zst'):

View source record →