INNER CODE UNIT · Python

ChatCompletions

tensorchord/modelz-llm · src/modelz_llm/falcon_service.py:30

class ChatCompletions:
    def __init__(self, client: Client) -> None:
        self.client = client

    async def on_post(self, req: Request, resp: Response):
        buf = await req.stream.readall()
        try:
            chat_req = ChatCompletionRequest.from_bytes(buf=buf)
        except msgspec.ValidationError as err:
            logger.info(f"Failed to parse request: {err}")
            # return 400 otherwise the client will retry
            resp.status = falcon.HTTP_400
            resp.data = ErrorResponse.from_validation_err(err, str(buf)).to_json()
            return

        comp = await self.client.request(chat_req)
        logger.info(comp)
        if isinstance(comp, Exception):

View source record →