INNER CODE UNIT · Python
new_size
UKPLab/gpl · gpl/train.py:88
if new_size == -1:
new_size = math.ceil(
250e3 / 3
) # Here use ceil to make the QPP == 3 if the corpus is large enough
logger.info(f"Automatically set `new_size` to {new_size}")
resize(evaluation_data, path_to_generated_data, new_size, use_train_qrels)
else:
corpus_path = os.path.join(evaluation_data, "corpus.jsonl")
os.system(f"cp {corpus_path} {path_to_generated_data}")
#### Adjust the QQP automatically, if needed ####
if queries_per_passage == -1:
assert "corpus.jsonl" in os.listdir(
path_to_generated_data
), "At least corpus should exist!"
corpus = GenericDataLoader(path_to_generated_data).load_corpus()
if len(corpus) * 3 < 250e3:
queries_per_passage = math.ceil(