INNER CODE UNIT · Python
tokenize
hyperonym/basaran · basaran/model.py:183
def tokenize(self, text):
"""Tokenize a string into a tensor of token IDs."""
batch = self.tokenizer.encode(text, return_tensors="pt")
return batch[0].to(self.device)
def generate(self, input_ids, logprobs=0, **kwargs):
"""Generate a stream of predicted tokens using the language model."""
# Store the original batch size and input length.
batch_size = input_ids.shape[0]
input_length = input_ids.shape[-1]
# Separate model arguments from generation config.
config = self.model.generation_config
config = copy.deepcopy(config)
kwargs = config.update(**kwargs)
kwargs["output_attentions"] = False
kwargs["output_hidden_states"] = False