#!/usr/bin/env python3 """Run the frozen MRI-002 historical holdout validation. This script is fully offline. It reads the accepted normalized historical comments and prior calibration/eligible discussion files, excludes calibration articleIDs, applies the frozen exposure-matched early-alignment procedure, and writes only compact holdout outputs. It does not modify existing historical files and does not run LLM classification, sentiment analysis, topic labeling, category classification, embeddings, or MRI interpretation. """ from __future__ import annotations import hashlib import html import math import re import sys from pathlib import Path from typing import Any import numpy as np import pandas as pd from scipy.optimize import minimize from scipy.special import logsumexp from scipy.stats import norm from sklearn.decomposition import TruncatedSVD from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.preprocessing import normalize ROOT = Path("historical") COMMENTS_MASTER = ROOT / "comments_2017_2018_master.parquet" ELIGIBLE_DISCUSSIONS = ROOT / "mri_high_engagement_discussions.csv" CALIBRATION_DISCUSSIONS = ROOT / "mri_predictive_120_discussions.csv" SUMMARY_TXT = ROOT / "mri_holdout_summary.txt" DISCUSSION_RESULTS_CSV = ROOT / "mri_holdout_discussion_results.csv" CONTROL_TARGET_PER_WINNER = 3 FUTURE_WINNERS_PER_DISCUSSION = 10 MATCH_WINDOWS = [0.01, 0.025, 0.05] PAIR_ID_PREFIX = "H" TOKEN_SEED = "mri002-historical-holdout-frozen-v1" class HoldoutError(RuntimeError): """Expected validation or preparation failure.""" def require_inputs() -> None: missing = [ str(path) for path in [COMMENTS_MASTER, ELIGIBLE_DISCUSSIONS, CALIBRATION_DISCUSSIONS] if not path.exists() ] if missing: raise HoldoutError(f"Missing required input file(s): {', '.join(missing)}") def parse_bool(series: pd.Series) -> pd.Series: if series.dtype == bool: return series return series.astype(str).str.strip().str.lower().isin(["true", "1", "yes"]) def stable_token(prefix: str, *parts: Any, length: int = 14) -> str: text = "|".join("" if part is None else str(part) for part in parts) digest = hashlib.sha256(text.encode("utf-8")).hexdigest()[:length].upper() return f"{prefix}{digest}" def valid_text(series: pd.Series) -> pd.Series: return series.notna() & series.astype(str).str.strip().ne("") def word_count(value: Any) -> int: text = "" if value is None or pd.isna(value) else str(value) return len(re.findall(r"\S+", text.strip())) def clean_text(value: Any) -> str: text = "" if value is None or pd.isna(value) else str(value) text = html.unescape(text) text = re.sub(r"<[^>]+>", " ", text) text = re.sub(r"https?://\S+|www\.\S+", " ", text) text = text.lower() text = re.sub(r"\s+", " ", text).strip() return text def load_holdout_discussions() -> tuple[pd.DataFrame, set[str]]: eligible = pd.read_csv(ELIGIBLE_DISCUSSIONS, dtype={"articleID": str}) calibration = pd.read_csv(CALIBRATION_DISCUSSIONS, dtype={"articleID": str}) calibration_ids = set(calibration["articleID"].astype(str)) holdout = eligible[~eligible["articleID"].astype(str).isin(calibration_ids)].copy() holdout = holdout.sort_values(["publication_date", "articleID"], kind="mergesort").reset_index(drop=True) if holdout["articleID"].duplicated().any(): duplicates = holdout.loc[holdout["articleID"].duplicated(), "articleID"].tolist() raise HoldoutError("Duplicate holdout articleID values: " + ", ".join(duplicates[:10])) return holdout, calibration_ids def load_holdout_top_level_comments(article_ids: set[str]) -> pd.DataFrame: comments = pd.read_parquet( COMMENTS_MASTER, columns=[ "commentID", "articleID", "commentBody", "createDate", "recommendations", "editorsSelection", "is_top_level", ], ) comments["articleID"] = comments["articleID"].astype(str) comments = comments[comments["articleID"].isin(article_ids) & parse_bool(comments["is_top_level"])].copy() comments["commentID"] = comments["commentID"].astype(str) comments["createDate"] = pd.to_numeric(comments["createDate"], errors="coerce") comments["recommendations"] = pd.to_numeric(comments["recommendations"], errors="coerce") comments["editorsSelection"] = parse_bool(comments["editorsSelection"]) comments["has_valid_text"] = valid_text(comments["commentBody"]) comments["has_valid_timestamp"] = comments["createDate"].notna() comments["has_valid_recommendations"] = comments["recommendations"].notna() comments["is_valid_candidate"] = ( comments["has_valid_text"] & comments["has_valid_timestamp"] & comments["has_valid_recommendations"] ) return comments def add_order_and_future_targets(comments: pd.DataFrame) -> pd.DataFrame: ordered = comments.sort_values( ["articleID", "createDate", "commentID"], ascending=[True, True, True], kind="mergesort", ).copy() ordered["comment_order"] = ordered.groupby("articleID").cumcount() + 1 ordered["top_level_comment_total"] = ordered.groupby("articleID")["commentID"].transform("size") ordered["comment_order_pct"] = ordered["comment_order"] / ordered["top_level_comment_total"] ordered["is_early_25"] = ordered["comment_order"].le(25) ordered["is_late_25"] = ordered["comment_order"].gt(ordered["top_level_comment_total"] - 25) future_candidates = ordered[ ordered["comment_order"].gt(25) & ordered["is_valid_candidate"] ].sort_values( ["articleID", "recommendations", "createDate", "commentID"], ascending=[True, False, True, True], kind="mergesort", ).copy() future_candidates["future_recognition_rank"] = ( future_candidates.groupby("articleID").cumcount() + 1 ) ordered["future_recognition_rank"] = pd.Series(pd.NA, index=ordered.index, dtype="Int64") ordered.loc[future_candidates.index, "future_recognition_rank"] = future_candidates[ "future_recognition_rank" ].astype("Int64") ordered["is_future_top_10"] = ( ordered["future_recognition_rank"].le(FUTURE_WINNERS_PER_DISCUSSION).fillna(False) ) ordered["word_count"] = ordered["commentBody"].map(word_count).astype("Int64") ordered["clean_text"] = ordered["commentBody"].map(clean_text) return ordered def validate_holdout_population(flagged: pd.DataFrame, holdout_ids: set[str]) -> None: summary = flagged.groupby("articleID").agg( top_level_comments=("commentID", "size"), early_count=("is_early_25", "sum"), future_count=("is_future_top_10", "sum"), missing_text=("has_valid_text", lambda values: int((~values).sum())), missing_timestamp=("has_valid_timestamp", lambda values: int((~values).sum())), missing_recommendations=("has_valid_recommendations", lambda values: int((~values).sum())), duplicate_comment_ids=("commentID", lambda values: int(values.duplicated().sum())), ) missing_articles = sorted(holdout_ids - set(summary.index.astype(str))) if missing_articles: raise HoldoutError("No comments loaded for articleID(s): " + ", ".join(missing_articles[:10])) failures = summary[ summary["top_level_comments"].lt(100) | summary["early_count"].ne(25) | summary["future_count"].ne(FUTURE_WINNERS_PER_DISCUSSION) | summary["missing_timestamp"].gt(0) | summary["missing_recommendations"].gt(0) | summary["duplicate_comment_ids"].gt(0) ] if not failures.empty: raise HoldoutError( "Holdout eligibility validation failed for articleID(s): " + ", ".join(failures.index.astype(str).tolist()[:10]) ) def choose_controls_for_winner( article_pool: pd.DataFrame, winner: pd.Series, used_control_ids: set[str], ) -> pd.DataFrame: base_pool = article_pool[ article_pool["comment_order"].gt(25) & article_pool["is_valid_candidate"] & ~article_pool["is_future_top_10"] ].copy() base_pool["_order_pct_difference"] = ( base_pool["comment_order_pct"] - winner["comment_order_pct"] ).abs() unused_pool = base_pool[~base_pool["commentID"].isin(used_control_ids)].copy() selected = pd.DataFrame(columns=base_pool.columns) for window in MATCH_WINDOWS: candidates = unused_pool[unused_pool["_order_pct_difference"].le(window)].sort_values( ["_order_pct_difference", "createDate", "commentID"], ascending=[True, True, True], kind="mergesort", ) if len(candidates) >= CONTROL_TARGET_PER_WINNER: selected = candidates.head(CONTROL_TARGET_PER_WINNER).copy() break if selected.empty: selected = unused_pool.sort_values( ["_order_pct_difference", "createDate", "commentID"], ascending=[True, True, True], kind="mergesort", ).head(CONTROL_TARGET_PER_WINNER).copy() if len(selected) < CONTROL_TARGET_PER_WINNER: needed = CONTROL_TARGET_PER_WINNER - len(selected) fallback = base_pool[~base_pool["commentID"].isin(set(selected["commentID"]))].sort_values( ["_order_pct_difference", "createDate", "commentID"], ascending=[True, True, True], kind="mergesort", ).head(needed) selected = pd.concat([selected, fallback], ignore_index=False) return selected.head(CONTROL_TARGET_PER_WINNER).copy() def build_candidates(flagged: pd.DataFrame, discussions: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]: meta = discussions[["articleID", "headline", "publication_date", "sectionName", "top_level_comments"]].copy() comments = flagged.merge(meta, on="articleID", how="left", suffixes=("", "_discussion")) early = comments[comments["is_early_25"]][ ["articleID", "commentID", "clean_text", "comment_order"] ].copy() early["early_doc_id"] = early.apply( lambda row: stable_token("E_", TOKEN_SEED, row["articleID"], row["commentID"]), axis=1, ) rows: list[dict[str, Any]] = [] pair_number = 0 for article_id, article in comments.groupby("articleID", sort=False): article = article.sort_values(["comment_order", "commentID"], kind="mergesort") future_winners = article[article["is_future_top_10"]].sort_values( ["future_recognition_rank", "createDate", "commentID"], ascending=[True, True, True], kind="mergesort", ) used_control_ids: set[str] = set() for _, winner in future_winners.iterrows(): pair_number += 1 pair_id = f"{PAIR_ID_PREFIX}{pair_number:05d}" rows.append( { "candidate_id": stable_token("HC_", TOKEN_SEED, article_id, pair_id, winner["commentID"]), "articleID": article_id, "headline": winner["headline"], "publication_date": winner["publication_date"], "sectionName": winner["sectionName"], "pair_id": pair_id, "candidate_role": "FUTURE_WINNER", "winner_status": 1, "commentID": winner["commentID"], "commentBody": winner["commentBody"], "clean_text": winner["clean_text"], "createDate": winner["createDate"], "comment_order": winner["comment_order"], "comment_order_pct": winner["comment_order_pct"], "recommendations": winner["recommendations"], "word_count": winner["word_count"], "matched_to_future_commentID": winner["commentID"], "order_pct_difference": 0.0, } ) controls = choose_controls_for_winner(article, winner, used_control_ids) for _, control in controls.iterrows(): used_control_ids.add(str(control["commentID"])) rows.append( { "candidate_id": stable_token("HC_", TOKEN_SEED, article_id, pair_id, control["commentID"]), "articleID": article_id, "headline": control["headline"], "publication_date": control["publication_date"], "sectionName": control["sectionName"], "pair_id": pair_id, "candidate_role": "MATCHED_CONTROL", "winner_status": 0, "commentID": control["commentID"], "commentBody": control["commentBody"], "clean_text": control["clean_text"], "createDate": control["createDate"], "comment_order": control["comment_order"], "comment_order_pct": control["comment_order_pct"], "recommendations": control["recommendations"], "word_count": control["word_count"], "matched_to_future_commentID": winner["commentID"], "order_pct_difference": abs( control["comment_order_pct"] - winner["comment_order_pct"] ), } ) candidates = pd.DataFrame(rows) return early, candidates def compute_blinded_alignment(early: pd.DataFrame, candidates: pd.DataFrame) -> pd.DataFrame: blind_candidates = candidates[ ["candidate_id", "articleID", "pair_id", "clean_text", "comment_order", "comment_order_pct", "word_count"] ].copy() if set(blind_candidates.columns).intersection( {"candidate_role", "recommendations", "future_recognition_rank", "winner_status"} ): raise HoldoutError("Blinded candidate frame contains outcome-revealing columns.") early_docs = early[["early_doc_id", "articleID", "clean_text"]].rename( columns={"early_doc_id": "doc_id"} ) early_docs["doc_type"] = "EARLY" candidate_docs = blind_candidates[["candidate_id", "articleID", "clean_text"]].rename( columns={"candidate_id": "doc_id"} ) candidate_docs["doc_type"] = "CANDIDATE" docs = pd.concat([early_docs, candidate_docs], ignore_index=True) vectorizer = TfidfVectorizer( stop_words="english", ngram_range=(1, 2), min_df=2, sublinear_tf=True, max_features=50000, lowercase=False, ) tfidf = vectorizer.fit_transform(docs["clean_text"].fillna("")) if tfidf.shape[1] <= 100: raise HoldoutError(f"Frozen SVD requires more than 100 features; got {tfidf.shape[1]}.") svd = TruncatedSVD(n_components=100, random_state=42) reduced = normalize(svd.fit_transform(tfidf), norm="l2", axis=1) early_indices = np.flatnonzero(docs["doc_type"].eq("EARLY").to_numpy()) candidate_indices = np.flatnonzero(docs["doc_type"].eq("CANDIDATE").to_numpy()) early_vectors = pd.DataFrame( { "articleID": docs.iloc[early_indices]["articleID"].to_numpy(), "vector_index": early_indices, } ) centroids: dict[str, np.ndarray] = {} for article_id, group in early_vectors.groupby("articleID", sort=False): centroid = reduced[group["vector_index"].to_numpy()].mean(axis=0) norm = np.linalg.norm(centroid) centroids[str(article_id)] = centroid / norm if norm else centroid candidate_doc_rows = docs.iloc[candidate_indices].reset_index(drop=False).rename( columns={"index": "vector_index"} ) similarities = [] for row in candidate_doc_rows.itertuples(index=False): centroid = centroids.get(str(row.articleID)) if centroid is None: similarity = np.nan else: similarity = float(reduced[int(row.vector_index)].dot(centroid)) similarities.append({"candidate_id": row.doc_id, "early_alignment_similarity": similarity}) return pd.DataFrame(similarities) def score_pairs(candidates_with_similarity: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]: pair_rows = [] for pair_id, group in candidates_with_similarity.groupby("pair_id", sort=False): winner = group[group["winner_status"].eq(1)].iloc[0] controls = group[group["winner_status"].eq(0)] winner_sim = float(winner["early_alignment_similarity"]) control_sims = controls["early_alignment_similarity"].astype(float) control_mean = float(control_sims.mean()) pairwise_wins = int((winner_sim > control_sims).sum()) pair_rows.append( { "articleID": winner["articleID"], "headline": winner["headline"], "publication_date": winner["publication_date"], "sectionName": winner["sectionName"], "pair_id": pair_id, "winner_similarity": winner_sim, "mean_control_similarity": control_mean, "winner_minus_mean_control_similarity": winner_sim - control_mean, "pairwise_comparisons": len(controls), "pairwise_winner_wins": pairwise_wins, "winner_gt_mean_control": bool(winner_sim > control_mean), "winner_top_of_four": bool(winner_sim > float(control_sims.max())), "median_control_order_pct_difference": float(controls["order_pct_difference"].median()), "p90_control_order_pct_difference": float(controls["order_pct_difference"].quantile(0.90)), "controls_within_001": int(controls["order_pct_difference"].le(0.01).sum()), "controls_within_0025": int(controls["order_pct_difference"].le(0.025).sum()), } ) pair_results = pd.DataFrame(pair_rows) discussion_results = pair_results.groupby("articleID", sort=False).agg( publication_date=("publication_date", "first"), headline=("headline", "first"), sectionName=("sectionName", "first"), matched_sets=("pair_id", "size"), pairwise_comparisons=("pairwise_comparisons", "sum"), pairwise_winner_wins=("pairwise_winner_wins", "sum"), winner_gt_mean_control_sets=("winner_gt_mean_control", "sum"), top_of_four_winner_sets=("winner_top_of_four", "sum"), mean_winner_similarity=("winner_similarity", "mean"), mean_control_similarity=("mean_control_similarity", "mean"), mean_winner_minus_mean_control_similarity=("winner_minus_mean_control_similarity", "mean"), median_control_order_pct_difference=("median_control_order_pct_difference", "median"), p90_control_order_pct_difference=("p90_control_order_pct_difference", "median"), controls_within_001=("controls_within_001", "sum"), controls_within_0025=("controls_within_0025", "sum"), ).reset_index() discussion_results["pairwise_win_rate"] = ( discussion_results["pairwise_winner_wins"] / discussion_results["pairwise_comparisons"] ) discussion_results["winner_gt_mean_control_rate"] = ( discussion_results["winner_gt_mean_control_sets"] / discussion_results["matched_sets"] ) discussion_results["top_of_four_accuracy"] = ( discussion_results["top_of_four_winner_sets"] / discussion_results["matched_sets"] ) discussion_results["discussion_direction_positive"] = discussion_results[ "mean_winner_minus_mean_control_similarity" ].gt(0) return pair_results, discussion_results def fit_conditional_logit(candidates_with_similarity: pd.DataFrame) -> tuple[pd.DataFrame, str]: regression = candidates_with_similarity.copy() regression["log1p_word_count"] = np.log1p(regression["word_count"].astype(float)) for source, target in [ ("early_alignment_similarity", "standardized_early_alignment_similarity"), ("log1p_word_count", "standardized_log1p_word_count"), ]: mean = regression[source].mean() std = regression[source].std(ddof=0) if not std or pd.isna(std): regression[target] = 0.0 else: regression[target] = (regression[source] - mean) / std y = regression["winner_status"].astype(float).to_numpy() x_columns = ["standardized_early_alignment_similarity", "standardized_log1p_word_count"] x = regression[x_columns].astype(float) groups = regression["pair_id"].astype(str) try: from statsmodels.discrete.conditional_models import ConditionalLogit model = ConditionalLogit(y, x, groups=groups) result = model.fit(disp=False, maxiter=200) conf = result.conf_int() rows = [] for column in x_columns: coefficient = float(result.params[column]) ci_low = float(conf.loc[column, 0]) ci_high = float(conf.loc[column, 1]) rows.append( { "term": column, "coefficient": coefficient, "odds_ratio": math.exp(coefficient), "ci_low": ci_low, "ci_high": ci_high, "odds_ratio_ci_low": math.exp(ci_low), "odds_ratio_ci_high": math.exp(ci_high), "p_value": float(result.pvalues[column]), } ) return pd.DataFrame(rows), "statsmodels ConditionalLogit" except Exception as exc: return fit_conditional_logit_fallback(regression, x_columns), f"custom scipy conditional likelihood fallback ({exc})" def fit_conditional_logit_fallback(regression: pd.DataFrame, x_columns: list[str]) -> pd.DataFrame: x = regression[x_columns].astype(float).to_numpy() y = regression["winner_status"].astype(float).to_numpy() group_codes = pd.factorize(regression["pair_id"].astype(str), sort=False)[0] group_indices = [np.flatnonzero(group_codes == code) for code in np.unique(group_codes)] case_indices = [indices[np.flatnonzero(y[indices] == 1)[0]] for indices in group_indices] def objective(beta: np.ndarray) -> tuple[float, np.ndarray]: nll = 0.0 grad = np.zeros_like(beta) for indices, case_index in zip(group_indices, case_indices): xg = x[indices] eta = xg @ beta log_denom = logsumexp(eta) nll += log_denom - float(x[case_index] @ beta) probs = np.exp(eta - log_denom) grad += probs @ xg - x[case_index] return nll, grad result = minimize( fun=lambda beta: objective(beta)[0], x0=np.zeros(len(x_columns)), jac=lambda beta: objective(beta)[1], method="BFGS", options={"maxiter": 500, "gtol": 1e-6}, ) beta = result.x hessian = np.zeros((len(x_columns), len(x_columns))) for indices in group_indices: xg = x[indices] eta = xg @ beta probs = np.exp(eta - logsumexp(eta)) weighted_second = xg.T @ (probs[:, None] * xg) mean = probs @ xg hessian += weighted_second - np.outer(mean, mean) cov = np.linalg.pinv(hessian) se = np.sqrt(np.diag(cov)) rows = [] for index, column in enumerate(x_columns): coefficient = float(beta[index]) z_value = coefficient / float(se[index]) p_value = float(2 * norm.sf(abs(z_value))) ci_low = coefficient - 1.96 * float(se[index]) ci_high = coefficient + 1.96 * float(se[index]) rows.append( { "term": column, "coefficient": coefficient, "odds_ratio": math.exp(coefficient), "ci_low": ci_low, "ci_high": ci_high, "odds_ratio_ci_low": math.exp(ci_low), "odds_ratio_ci_high": math.exp(ci_high), "p_value": p_value, } ) return pd.DataFrame(rows) def validation_metrics( holdout: pd.DataFrame, calibration_ids: set[str], flagged: pd.DataFrame, candidates: pd.DataFrame, candidates_with_similarity: pd.DataFrame, pair_results: pd.DataFrame, discussion_results: pd.DataFrame, ) -> dict[str, Any]: future = candidates_with_similarity[candidates_with_similarity["winner_status"].eq(1)] controls = candidates_with_similarity[candidates_with_similarity["winner_status"].eq(0)] holdout_ids = set(holdout["articleID"].astype(str)) calibration_overlap = len(holdout_ids.intersection(calibration_ids)) duplicate_controls = int(controls.duplicated(subset=["articleID", "commentID"]).sum()) missing_candidate_text = int( (candidates["commentBody"].isna() | candidates["commentBody"].astype(str).str.strip().eq("")).sum() ) missing_candidate_timestamps = int(candidates["createDate"].isna().sum()) missing_candidate_recommendations = int(candidates["recommendations"].isna().sum()) missing_similarity = int(candidates_with_similarity["early_alignment_similarity"].isna().sum()) control_count = len(controls) pairwise_comparisons = int(pair_results["pairwise_comparisons"].sum()) pairwise_wins = int(pair_results["pairwise_winner_wins"].sum()) return { "holdout_discussions": len(holdout), "future_winners": len(future), "matched_controls": len(controls), "target_matched_controls": len(future) * CONTROL_TARGET_PER_WINNER, "pairwise_win_rate": pairwise_wins / pairwise_comparisons, "winner_gt_mean_control_rate": float(pair_results["winner_gt_mean_control"].mean()), "top_of_four_accuracy": float(pair_results["winner_top_of_four"].mean()), "discussion_direction_positive_fraction": float( discussion_results["discussion_direction_positive"].mean() ), "median_abs_order_pct_difference": float(controls["order_pct_difference"].median()), "p90_abs_order_pct_difference": float(controls["order_pct_difference"].quantile(0.90)), "controls_within_001_pct": float(controls["order_pct_difference"].le(0.01).mean()), "controls_within_0025_pct": float(controls["order_pct_difference"].le(0.025).mean()), "controls_within_005_pct": float(controls["order_pct_difference"].le(0.05).mean()), "missing_candidate_text": missing_candidate_text, "missing_candidate_timestamps": missing_candidate_timestamps, "missing_candidate_recommendations": missing_candidate_recommendations, "missing_similarity": missing_similarity, "duplicate_reused_controls_within_article": duplicate_controls, "calibration_article_ids_in_holdout": calibration_overlap, "controls_with_comment_order_le_25": int(controls["comment_order"].le(25).sum()), "controls_that_are_future_winners": int( controls["commentID"].isin(set(future["commentID"].astype(str))).sum() ), "early_rows": int(flagged["is_early_25"].sum()), "feature_count": None, "control_count": control_count, } def write_discussion_results(discussion_results: pd.DataFrame, holdout: pd.DataFrame) -> pd.DataFrame: keep = [ "articleID", "top_level_comments", "total_comments", "total_recommendations", "max_recommendations", "median_recommendations", "editors_picks", ] enriched = discussion_results.merge( holdout[[column for column in keep if column in holdout.columns]], on="articleID", how="left", ) ordered_columns = [ "articleID", "publication_date", "headline", "sectionName", "top_level_comments", "total_comments", "matched_sets", "pairwise_comparisons", "pairwise_winner_wins", "pairwise_win_rate", "winner_gt_mean_control_sets", "winner_gt_mean_control_rate", "top_of_four_winner_sets", "top_of_four_accuracy", "mean_winner_similarity", "mean_control_similarity", "mean_winner_minus_mean_control_similarity", "discussion_direction_positive", "median_control_order_pct_difference", "p90_control_order_pct_difference", "controls_within_001", "controls_within_0025", "total_recommendations", "max_recommendations", "median_recommendations", "editors_picks", ] enriched = enriched[ordered_columns].sort_values(["publication_date", "articleID"], kind="mergesort") enriched.to_csv(DISCUSSION_RESULTS_CSV, index=False) return enriched def format_regression_table(regression_results: pd.DataFrame) -> str: lines = [] for row in regression_results.to_dict("records"): lines.append( "- {term}: coefficient={coefficient:.6f}; odds_ratio={odds_ratio:.6f}; " "95% CI [{odds_ratio_ci_low:.6f}, {odds_ratio_ci_high:.6f}]; p={p_value:.6g}".format( **row ) ) return "\n".join(lines) def write_summary( metrics: dict[str, Any], regression_results: pd.DataFrame, regression_method: str, ) -> None: text = f"""MRI-002 Historical Holdout Summary Status: frozen holdout validation; no method tuning was performed. Inputs remained local/offline. Existing historical master and calibration files were not modified. Eligibility - Source eligible high-engagement population: historical/mri_high_engagement_discussions.csv - Calibration exclusions: historical/mri_predictive_120_discussions.csv - Calibration articleIDs in holdout: {metrics['calibration_article_ids_in_holdout']} - Holdout discussions: {metrics['holdout_discussions']} Matching Validation - Future winners: {metrics['future_winners']} - Target matched controls: {metrics['target_matched_controls']} - Actual matched controls: {metrics['matched_controls']} - Median absolute order-percentile difference: {metrics['median_abs_order_pct_difference']:.6f} - 90th percentile order-percentile difference: {metrics['p90_abs_order_pct_difference']:.6f} - Controls within +/-0.01: {metrics['controls_within_001_pct']:.2%} - Controls within +/-0.025: {metrics['controls_within_0025_pct']:.2%} - Controls within +/-0.05: {metrics['controls_within_005_pct']:.2%} - Duplicate/reused controls within article: {metrics['duplicate_reused_controls_within_article']} - Controls with comment_order <= 25: {metrics['controls_with_comment_order_le_25']} - Controls that are FUTURE_TOP_10 winners: {metrics['controls_that_are_future_winners']} Missing Data - Candidate rows missing text: {metrics['missing_candidate_text']} - Candidate rows missing timestamps: {metrics['missing_candidate_timestamps']} - Candidate rows missing recommendation counts: {metrics['missing_candidate_recommendations']} - Candidate rows missing early-alignment similarity: {metrics['missing_similarity']} Frozen Text Alignment Method - Text sources used for representation: EARLY_25 text and blinded matched candidate text only. - Cleaning: HTML tags and URLs removed; text lowercased. - TF-IDF: English stop words, word unigrams+bigrams, min_df=2, sublinear_tf=True, max_features=50000. - TruncatedSVD: n_components=100, random_state=42. - Vectors normalized before cosine similarity. - Semantic representation was created before candidate roles were revealed for metric calculation. - Control selection used comment_order_pct distance, createDate, and commentID only; it did not use recommendations or text. Holdout Early-Alignment Metrics - Pairwise early-alignment win rate: {metrics['pairwise_win_rate']:.6f} - Winner > mean-control rate: {metrics['winner_gt_mean_control_rate']:.6f} - Top-of-four accuracy: {metrics['top_of_four_accuracy']:.6f} - Chance baseline for top-of-four accuracy: 0.250000 - Fraction of holdout discussions with mean winner-minus-control similarity > 0: {metrics['discussion_direction_positive_fraction']:.6f} Length Robustness - Conditional logistic regression groups: matched pair_id sets. - Model: winner_status ~ standardized_early_alignment_similarity + standardized_log1p_word_count. - Fitting method: {regression_method} {format_regression_table(regression_results)} Compact Outputs - historical/mri_holdout_summary.txt - historical/mri_holdout_discussion_results.csv """ SUMMARY_TXT.write_text(text, encoding="utf-8") def print_report(metrics: dict[str, Any], regression_results: pd.DataFrame, regression_method: str) -> None: print("MRI-002 historical holdout report") print(f"holdout discussions: {metrics['holdout_discussions']}") print(f"future winners: {metrics['future_winners']}") print(f"matched controls: {metrics['matched_controls']}") print(f"median absolute order-percentile difference: {metrics['median_abs_order_pct_difference']:.6f}") print(f"% controls within +/-0.01: {metrics['controls_within_001_pct']:.2%}") print(f"% controls within +/-0.025: {metrics['controls_within_0025_pct']:.2%}") print( "missing text/timestamps/recommendations/similarity: " f"{metrics['missing_candidate_text']}/{metrics['missing_candidate_timestamps']}/" f"{metrics['missing_candidate_recommendations']}/{metrics['missing_similarity']}" ) print(f"duplicate/reused controls within article: {metrics['duplicate_reused_controls_within_article']}") print(f"calibration articleIDs in holdout: {metrics['calibration_article_ids_in_holdout']}") print("semantic representation frozen before role reveal: True") print(f"pairwise early-alignment win rate: {metrics['pairwise_win_rate']:.6f}") print(f"winner > mean-control rate: {metrics['winner_gt_mean_control_rate']:.6f}") print(f"top-of-four accuracy: {metrics['top_of_four_accuracy']:.6f}") print( "fraction discussions with mean winner-minus-control similarity > 0: " f"{metrics['discussion_direction_positive_fraction']:.6f}" ) print("conditional logistic regression:") print(f" method: {regression_method}") for row in regression_results.to_dict("records"): print( " {term}: coefficient={coefficient:.6f}, odds_ratio={odds_ratio:.6f}, " "95% CI [{odds_ratio_ci_low:.6f}, {odds_ratio_ci_high:.6f}], p={p_value:.6g}".format( **row ) ) print(f"wrote {SUMMARY_TXT}") print(f"wrote {DISCUSSION_RESULTS_CSV}") def main() -> int: try: require_inputs() holdout, calibration_ids = load_holdout_discussions() holdout_ids = set(holdout["articleID"].astype(str)) comments = load_holdout_top_level_comments(holdout_ids) flagged = add_order_and_future_targets(comments) validate_holdout_population(flagged, holdout_ids) early, candidates = build_candidates(flagged, holdout) similarities = compute_blinded_alignment(early, candidates) representation_frozen_before_role_reveal = True candidates_with_similarity = candidates.merge(similarities, on="candidate_id", how="left") if not representation_frozen_before_role_reveal: raise HoldoutError("Semantic representation was not frozen before role reveal.") pair_results, discussion_results = score_pairs(candidates_with_similarity) regression_results, regression_method = fit_conditional_logit(candidates_with_similarity) metrics = validation_metrics( holdout, calibration_ids, flagged, candidates, candidates_with_similarity, pair_results, discussion_results, ) write_discussion_results(discussion_results, holdout) write_summary(metrics, regression_results, regression_method) print_report(metrics, regression_results, regression_method) except HoldoutError as exc: print(f"Stopped: {exc}", file=sys.stderr) return 1 return 0 if __name__ == "__main__": raise SystemExit(main())