INNER CODE UNIT · Python
_get_probs
NorskRegnesentral/skweak · skweak/aggregation.py:154
def _get_probs(self, agg_df: pandas.DataFrame, min_threshold=0.1) \
-> Dict[Tuple[int,int],Dict[str,float]]:
"""Takes as input a 2D dataframe of shape (nb_entries, nb_labels)
assocating each token/span to the probability of an output label, and
returns a dictionary associating each (start,end) span/token to a
mapping from labels to probabilities"""
raise NotImplementedError("must implement _get_probs")
def _get_sources_to_aggregate(self, doc):
"""Returns the sources to consider (and filtering out the ones to avoid,
such as aggretated sources)"""
sources = [source for source in doc.spans if len(doc.spans[source]) > 0
and not doc.spans[source].attrs.get("aggregated", False)
and not doc.spans[source].attrs.get("avoid_in_aggregation", False)
and (not hasattr(self, "weights")
or np.sum(self.weights.get(source, 1)) > 0) #type:ignore