INNER CODE UNIT · Python
combine_conds
datajuicer/data-juicer · app.py:201
def combine_conds(conds, num_samples):
"""Combine the per-OP boolean conditions into a single mask over samples.
``conds`` is empty whenever none of the OPs in the config is covered by
``op_stats_dict``. ``np.all`` over an empty list collapses to a scalar,
which ``DataFrame.loc`` then reads as a label instead of a boolean mask and
raises ``KeyError: True``, so keep all the samples explicitly in that case.
"""
if not conds:
return np.ones(num_samples, dtype=bool)
return np.all([list(cond.values())[0] for cond in conds], axis=0)
op_stats_dict = {
"alphanumeric_filter": [StatsKeys.alpha_token_ratio, StatsKeys.alnum_ratio],
"average_line_length_filter": [StatsKeys.avg_line_length],
"character_repetition_filter": [StatsKeys.char_rep_ratio],
"flagged_words_filter": [StatsKeys.flagged_words_ratio],