INNER CODE UNIT · Python

build_falcon_app

tensorchord/modelz-llm · src/modelz_llm/falcon_service.py:109

def build_falcon_app(args: argparse.Namespace):
    if args.dry_run:
        snapshot_download(repo_id=args.model)
        snapshot_download(repo_id=args.emb_model)
        return

    llm_uds_path = UNIX_DOMAIN_SOCKET_PATH.format("llm")
    emb_uds_path = UNIX_DOMAIN_SOCKET_PATH.format("emb")
    barrier = mp.get_context("spawn").Barrier(3)
    model_proc = run_server(
        llm_uds_path, barrier, LLM, model_name=args.model, device=args.device
    )
    emb_proc = run_server(
        emb_uds_path, barrier, Emb, model_name=args.emb_model, device=args.device
    )
    barrier.wait()
    # wait to detect the proc exitcode
    time.sleep(1)

View source record →