INNER CODE UNIT · Python
Args
huggingface/transformers-bloom-inference · inference_server/server.py:30
class Args:
def __init__(self) -> None:
self.deployment_framework = os.getenv("DEPLOYMENT_FRAMEWORK", HF_ACCELERATE)
self.model_name = os.getenv("MODEL_NAME")
self.model_class = os.getenv("MODEL_CLASS")
self.dtype = get_torch_dtype(os.getenv("DTYPE"))
self.allowed_max_new_tokens = int(os.getenv("ALLOWED_MAX_NEW_TOKENS", 100))
self.max_input_length = int(os.getenv("MAX_INPUT_LENGTH", 512))
self.max_batch_size = int(os.getenv("MAX_BATCH_SIZE", 4))
self.debug = parse_bool(os.getenv("DEBUG", "false"))
# ------------------------------------------------------
args = Args()
model = ModelDeployment(args, True)
query_ids = QueryID()
app = Flask(__name__)
# ------------------------------------------------------