INNER CODE UNIT · Python

parse_file

HIT-SCIR/Chinese-Mixtral-8x7B · data/parquet2jsonl.py:9

def parse_file(filename):
    pds = pq.read_pandas(filename, columns=None).to_pandas()
    return pds.to_json(
        path_or_buf=None,
        orient="records",
        lines=True,
        date_format="iso",
        date_unit="us",
        compression="gzip",
    ).split("\n")


def main(
    target_jsonl: str = "./data/en/DKYoon-SlimPajama-6B.jsonl",
    root: str = "./data/en/DKYoon-SlimPajama-6B/data",
):
    root = Path(root)
    target_jsonl = Path(target_jsonl)

View source record →