INNER CODE UNIT · Python
filter_observations
NorskRegnesentral/skweak · skweak/aggregation.py:120
def filter_observations(self, obs:pandas.DataFrame) -> pandas.DataFrame:
# We count the votes for each label on all sources
def count_fun(x):
return np.bincount(x[x>=0], minlength=len(self.observed_labels))
label_votes = np.apply_along_axis(count_fun, 1, obs.values).astype(np.float32)
out_label_votes = label_votes.dot(self._get_vote_matrix())
relevant_rows = (out_label_votes.sum(axis=1) > 0.0)
return obs[relevant_rows] #type: ignore
@abstractmethod
def aggregate(self, obs: pandas.DataFrame) -> pandas.DataFrame:
"""Takes as input a 2D dataframe of shape (nb_entries, nb_sources)
associating each token/span to a set of observations from labelling
sources, and returns a 2D dataframe of shape (nb_entries, nb_labels)