INNER CODE UNIT · Python

calc_metric

linzhiqiu/t2v_metrics · dataset.py:151

def calc_metric(gold_scores, metric_scores, variant: str="pairwise_acc_with_tie_optimization", sample_rate=1.0):
    gold_scores = np.array(gold_scores)
    metric_scores = np.array(metric_scores)
    assert gold_scores.shape == metric_scores.shape
    if gold_scores.ndim == 1:
        # No grouping
        gold_scores = gold_scores.reshape(1, -1)
        metric_scores = metric_scores.reshape(1, -1)
    else:
        # Group by item (last dim is number of system)
        pass
    
    # Calculate metric using KendallTau (including Pairwise Accuracy)
    if variant == "pairwise_acc_with_tie_optimization":
        import tau_optimization
        result = tau_optimization.tau_optimization(metric_scores, gold_scores, tau_optimization.TauSufficientStats.acc_23, sample_rate=sample_rate)
        return result.best_tau, result.best_threshold
    elif variant == "pairwise_acc_ignore_tie":

View source record →