Day 71: Quantitative Evaluation Metrics & Statistical Confidence

import re
from collections import Counter

def normalize_answer(s):
def remove_articles(text):
return re.sub(r'\b(a|an|the)\b', ' ', text)
def white_space_fix(text):
return ' '.join(text.split())
def remove_punc(text):
return re.sub(r'[^a-zA-Z0-9 ]', '', text)
def lower(text):
return text.lower()
return white_space_fix(remove_articles(remove_punc(lower(s))))

def compute_token_f1(prediction, ground_truth):
pred_tokens = normalize_answer(prediction).split()
gold_tokens = normalize_answer(ground_truth).split()
em = 1.0 if pred_tokens == gold_tokens else 0.0
if len(pred_tokens) == 0 or len(gold_tokens) == 0:
return {'em': em, 'f1': 1.0 if pred_tokens == gold_tokens else 0.0}
common = Counter(pred_tokens) & Counter(gold_tokens)
num_same = sum(common.values())
if num_same == 0:
return {'em': em, 'f1': 0.0}
precision = 1.0 * num_same / len(pred_tokens)
recall = 1.0 * num_same / len(gold_tokens)
f1 = (2 * precision * recall) / (precision + recall)
return {'em': em, 'f1': f1}

def compute_rouge_l(candidate, reference, beta=1.0):
cand_tokens = candidate.lower().split()
ref_tokens = reference.lower().split()
m, n = len(ref_tokens), len(cand_tokens)
if m == 0 or n == 0:
return {'precision': 0.0, 'recall': 0.0, 'f1': 0.0}
dp = [[0] * (n + 1) for _ in range(m + 1)]
for i in range(1, m + 1):
for j in range(1, n + 1):
if ref_tokens[i - 1] == cand_tokens[j - 1]:
dp[i][j] = dp[i - 1][j - 1] + 1
else:
dp[i][j] = max(dp[i - 1][j], dp[i][j - 1])
lcs = dp[m][n]
r_lcs = lcs / m
p_lcs = lcs / n
if r_lcs + p_lcs == 0:
return {'precision': 0.0, 'recall': 0.0, 'f1': 0.0}
f_lcs = ((1 + beta**2) * r_lcs * p_lcs) / (r_lcs + beta**2 * p_lcs)
return {'precision': p_lcs, 'recall': r_lcs, 'f1': f_lcs}

def rank_with_length_penalty(candidates, alpha=0.7):
# candidates is a dict: {label: (total_log_prob, token_length)}
scored = {}
for label, (log_prob, length) in candidates.items():
penalty = (5.0 + length) alpha / (6.0 alpha)
scored[label] = log_prob / penalty
ranked = sorted(scored.items(), key=lambda x: x[1], reverse=True)
return ranked

import numpy as np

def bootstrap_confidence_interval(scores, num_resamples=10000, alpha=0.05, seed=42):
rng = np.random.default_rng(seed)
n = len(scores)
resampled_means = np.zeros(num_resamples)
for b in range(num_resamples):
indices = rng.choice(n, size=n, replace=True)
resampled_means[b] = np.mean(scores[indices])
lower_pct = (alpha / 2.0) * 100.0
upper_pct = (1.0 - alpha / 2.0) * 100.0
ci_lower = float(np.percentile(resampled_means, lower_pct))
ci_upper = float(np.percentile(resampled_means, upper_pct))
return {
'mean': float(np.mean(scores)),
'ci_lower': ci_lower,
'ci_upper': ci_upper,
'ci_width': ci_upper - ci_lower
}