INNER CODE UNIT · Python
_doc_to_example_text_list
dragondrop-cloud/cloud-concierge · nlpengine/main.py:157
def _doc_to_example_text_list(doc: str) -> List[str]:
"""
Converts a document into a series of individual pieces of text, separated by sentence.
"""
return [text for text in doc.split(".") if text != " "]
def _split_into_train_and_evaluation_data(training_data_examples: List) -> dict:
"""
Splits the training and evaluation data out of training_data_examples with an 80/20 train/eval split.
"""
train_set_cutoff = int(len(training_data_examples) * 0.8)
shuffle(training_data_examples)
return {
"train": training_data_examples[:train_set_cutoff],
"evaluate": training_data_examples[train_set_cutoff:],