#!/usr/bin/env python3 import csv import json import math import random import re import ssl import statistics import time import urllib.error import urllib.request from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone from pathlib import Path PROJECT = Path("mri_commercial_panel") BFCL = PROJECT / "frame_audit_001_bfcl" OUT = PROJECT / "outputs" / "mri_rank_before_work" OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" SEED = 47001 TRIALS_PER_CELL = 12 MAX_PROJECTED_COST = 10.00 TEMPERATURE = 0.2 MAX_TOKENS = 450 CTX = ssl.create_default_context(cafile="/etc/ssl/cert.pem") JUDGES = [ { "judge_family": "OpenAI", "model": "openai/gpt-5.6-sol", "provider_slug": "azure/eu", "prompt_per_m": 5.00, "completion_per_m": 30.00, }, { "judge_family": "Anthropic", "model": "anthropic/claude-opus-5", "provider_slug": "amazon-bedrock", "prompt_per_m": 5.00, "completion_per_m": 25.00, }, { "judge_family": "Google", "model": "google/gemini-2.5-pro", "provider_slug": "google-vertex/global", "prompt_per_m": 1.25, "completion_per_m": 10.00, }, { "judge_family": "xAI", "model": "x-ai/grok-4.5", "provider_slug": "xai/zdr", "prompt_per_m": 2.00, "completion_per_m": 6.00, }, ] SCENARIOS = { "Profile A": { "title": "High-volume low-risk automation", "preferred_model": "Qwen3-14B (Prompt)", "aggregate_model": "Claude-Opus-4-5-20251101 (FC)", "alternative_model": "Qwen3-14B (Prompt)", "environment": ( "A product team is choosing a tool-calling model for routine CRM, ticket triage, " "and back-office automations. Individual errors are reviewable and low consequence, " "but call volume is high." ), "conditions": ( "Satisfactory performance means strong single-turn tool-call capability, reliable " "relevance/irrelevance behavior, low benchmark cost, and low measured tail latency." ), "metrics": [ ("single-turn capability", "higher is better", "single_turn_score"), ("relevance/irrelevance behavior", "higher is better", "relevance_behavior_score"), ("benchmark cost", "lower is better", "total_cost_usd"), ("measured P95 latency", "lower is better", "latency_95th_percentile_s"), ], }, "Profile C": { "title": "Multi-turn enterprise agent", "preferred_model": "Claude-Opus-4-5-20251101 (FC)", "aggregate_model": "Claude-Opus-4-5-20251101 (FC)", "alternative_model": "GLM-4.6 (FC thinking)", "environment": ( "An enterprise AI team is choosing a model for multi-step agent workflows such as " "account changes, procurement requests, technical support, workflow orchestration, " "and knowledge-work automation." ), "conditions": ( "Satisfactory performance means strong multi-turn completion, memory behavior, web-search " "behavior, relevance/irrelevance behavior, acceptable latency, and cost as a secondary factor." ), "metrics": [ ("multi-turn performance", "higher is better", "multi_turn_score"), ("memory performance", "higher is better", "memory_score"), ("web-search performance", "higher is better", "web_search_score"), ("relevance/irrelevance behavior", "higher is better", "relevance_behavior_score"), ("benchmark cost", "lower is better", "total_cost_usd"), ("measured P95 latency", "lower is better", "latency_95th_percentile_s"), ], }, } CONDITIONS = ["EVIDENCE_FIRST", "CENTER_FIRST", "MRI_ORDER"] def now_iso(): return datetime.now(timezone.utc).replace(microsecond=0).isoformat() def load_api_key(): for line in (PROJECT / ".env").read_text(encoding="utf-8").splitlines(): if line.startswith("OPENROUTER_API_KEY="): return line.split("=", 1)[1].strip().strip('"').strip("'") raise RuntimeError("OPENROUTER_API_KEY missing") def load_profile_rows(): p = BFCL / "data" / "processed" / "bfcl_profile_results.csv" rows = {} with p.open(encoding="utf-8", newline="") as f: for row in csv.DictReader(f): rows[(row["profile_id"], row["model_identifier"])] = row return rows def fnum(value): if value is None or value == "": return None return float(str(value).replace("$", "")) def fmt(value, lower=False): x = fnum(value) if x is None: return "N/A" if lower and x >= 10: return f"{x:.2f}" if x >= 100: return f"{x:.2f}" return f"{x:.3f}".rstrip("0").rstrip(".") def verify_phase1(profile_rows): a_q = profile_rows[("A", "Qwen3-14B (Prompt)")] a_c = profile_rows[("A", "Claude-Opus-4-5-20251101 (FC)")] checks = [ ("single_turn_score", "higher", fnum(a_q["single_turn_score"]), fnum(a_c["single_turn_score"])), ("relevance_behavior_score", "higher", fnum(a_q["relevance_behavior_score"]), fnum(a_c["relevance_behavior_score"])), ("total_cost_usd", "lower", fnum(a_q["total_cost_usd"]), fnum(a_c["total_cost_usd"])), ("latency_95th_percentile_s", "lower", fnum(a_q["latency_95th_percentile_s"]), fnum(a_c["latency_95th_percentile_s"])), ] ok_every = all((q >= c if direction == "higher" else q <= c) for _, direction, q, c in checks) better_one = any((q > c if direction == "higher" else q < c) for _, direction, q, c in checks) if not (ok_every and better_one): raise RuntimeError("Profile A pair cannot be represented as Qwen at least as good on every displayed field and better on at least one.") c_w = profile_rows[("C", "Claude-Opus-4-5-20251101 (FC)")] c_alt = profile_rows[("C", "GLM-4.6 (FC thinking)")] return checks, c_w, c_alt def card_profiles(scenario_name, label_map, order, profile_rows, include_rank=False): scenario = SCENARIOS[scenario_name] profile_id = "A" if scenario_name == "Profile A" else "C" lines = [] for label in order: model = label_map[label] row = profile_rows[(profile_id, model)] lines.append(f"{label}:") if include_rank: lines.append(f"- General BFCL overall rank: {int(float(row['published_rank']))}") for metric_name, direction, col in scenario["metrics"]: value = fmt(row[col], lower=("lower" in direction)) suffix = " (lower is better)" if "lower" in direction else " (higher is better)" unit = "" if col == "total_cost_usd": value = "$" + value if col == "latency_95th_percentile_s": unit = " seconds" lines.append(f"- {metric_name}: {value}{unit}{suffix}") return "\n".join(lines) def rank_block(scenario_name, label_map, order, profile_rows): profile_id = "A" if scenario_name == "Profile A" else "C" lines = ["General BFCL overall ranks:"] for label in order: model = label_map[label] row = profile_rows[(profile_id, model)] lines.append(f"- {label}: BFCL overall rank {int(float(row['published_rank']))}") return "\n".join(lines) def evidence_block(scenario_name, label_map, order, profile_rows): scenario = SCENARIOS[scenario_name] return ( f"Work environment:\n{scenario['environment']}\n\n" f"Conditions of satisfactory performance:\n{scenario['conditions']}\n\n" f"Anonymized performance profiles:\n{card_profiles(scenario_name, label_map, order, profile_rows, include_rank=False)}" ) def final_prompt(condition, scenario_name, label_map, order, profile_rows): base = evidence_block(scenario_name, label_map, order, profile_rows) if condition == "EVIDENCE_FIRST": display = base elif condition == "CENTER_FIRST": display = rank_block(scenario_name, label_map, order, profile_rows) + "\n\n" + base else: display = base return ( display + "\n\nChoose the model you would select for this work environment. " + "Return JSON only with keys: final_choice_label, confidence_0_100, stated_reasoning, " + "cited_overall_rank, cited_scenario_specific_evidence. Use only labels shown above." ) def provisional_prompt(scenario_name, label_map, order, profile_rows): return ( evidence_block(scenario_name, label_map, order, profile_rows) + "\n\nMake a provisional selection before seeing any general BFCL overall rank. " + "Return JSON only with keys: provisional_choice_label, confidence_0_100, stated_reasoning, " + "cited_overall_rank, cited_scenario_specific_evidence. Use only labels shown above." ) def mri_final_prompt(scenario_name, label_map, order, profile_rows): return ( rank_block(scenario_name, label_map, order, profile_rows) + "\n\nNow make a final selection. You may keep or change your provisional selection. " + "Return JSON only with keys: final_choice_label, confidence_0_100, stated_reasoning, " + "cited_overall_rank, cited_scenario_specific_evidence, changed_after_rank_disclosure. Use only labels shown above." ) def system_prompt(): return ( "You are an independent decision judge. Select exactly one of the anonymized model labels for the described work environment. " "Use the displayed evidence only. Do not assume unstated model identity or hidden evidence. Return compact JSON only, without markdown." ) def make_payload(judge, messages): return { "model": judge["model"], "messages": messages, "temperature": TEMPERATURE, "max_tokens": MAX_TOKENS, "provider": { "zdr": True, "data_collection": "deny", "allow_fallbacks": False, "only": [judge["provider_slug"]], }, "usage": {"include": True}, } def call_openrouter(api_key, judge, messages): payload = make_payload(judge, messages) req = urllib.request.Request( OPENROUTER_URL, data=json.dumps(payload).encode("utf-8"), method="POST", headers={ "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", "X-Title": "MRI Technical Pilot 001", "HTTP-Referer": "http://localhost", "X-OpenRouter-Cache": "false", "Cache-Control": "no-store", }, ) with urllib.request.urlopen(req, timeout=240, context=CTX) as resp: return json.loads(resp.read().decode("utf-8")) def extract_content(response): try: content = response["choices"][0]["message"]["content"] except Exception: return "" if isinstance(content, str): return content.strip() if isinstance(content, list): parts = [] for item in content: if isinstance(item, dict): text = item.get("text") or item.get("content") if isinstance(text, str): parts.append(text) elif isinstance(item, str): parts.append(item) return "\n".join(parts).strip() return "" def parse_json(text): if not text: raise ValueError("empty content") t = text.strip() if t.startswith("```"): t = re.sub(r"^```(?:json)?\s*", "", t, flags=re.I) t = re.sub(r"\s*```$", "", t) start = t.find("{") end = t.rfind("}") if start >= 0 and end > start: t = t[start : end + 1] return json.loads(t) def normalize_bool(v): if isinstance(v, bool): return v if isinstance(v, str): return v.strip().lower() in {"true", "yes", "1"} return False def normalize_label(value): if not isinstance(value, str): return "" s = value.strip() m = re.search(r"\bModel\s*([AB])\b", s, flags=re.I) if m: return "Model " + m.group(1).upper() if s.upper() in {"A", "B"}: return "Model " + s.upper() return s def usage_cost(response, judge): usage = response.get("usage") or {} if isinstance(usage, dict) and usage.get("cost") is not None: try: return float(usage.get("cost") or 0) except Exception: pass pin = int((usage or {}).get("prompt_tokens") or 0) cout = int((usage or {}).get("completion_tokens") or 0) return (pin / 1_000_000) * judge["prompt_per_m"] + (cout / 1_000_000) * judge["completion_per_m"] def generate_trials(profile_rows): rng = random.Random(SEED) trials = [] trial_id = 1 for judge in JUDGES: for scenario_name, scenario in SCENARIOS.items(): models = [scenario["aggregate_model"], scenario["alternative_model"]] for condition in CONDITIONS: for rep in range(1, TRIALS_PER_CELL + 1): labels = ["Model A", "Model B"] rng.shuffle(labels) label_map = {labels[0]: models[0], labels[1]: models[1]} order = ["Model A", "Model B"] rng.shuffle(order) trials.append({ "trial_id": f"T{trial_id:04d}", "judge_family": judge["judge_family"], "judge_model": judge["model"], "provider_slug": judge["provider_slug"], "scenario": scenario_name, "condition": condition, "replicate": rep, "label_map": label_map, "candidate_order": order, }) trial_id += 1 return trials def projected_cost(): # Conservative estimate. MRI_ORDER uses two calls; other conditions use one. input_tokens = 1300 output_tokens = 220 total = 0.0 calls = 0 for judge in JUDGES: model_calls = TRIALS_PER_CELL * len(SCENARIOS) * (1 + 1 + 2) calls += model_calls total += model_calls * ((input_tokens / 1_000_000) * judge["prompt_per_m"] + (output_tokens / 1_000_000) * judge["completion_per_m"]) return calls, total def write_prereg(profile_rows, phase1_checks, c_w, c_alt, trials, projected_calls, projected_usd): OUT.mkdir(parents=True, exist_ok=True) a_q = profile_rows[("A", "Qwen3-14B (Prompt)")] a_c = profile_rows[("A", "Claude-Opus-4-5-20251101 (FC)")] text = f"""# MRI TECHNICAL PILOT 001 # 47 TO 1 - THE RANK BEFORE THE WORK Preregistered at: {now_iso()} ## Core Question When the underlying evidence and working conditions remain constant, does revealing an authoritative overall rank change which model a decision system selects? ## Frozen Data Sources - `frame_audit_001_bfcl/data/processed/bfcl_profile_results.csv` - `frame_audit_001_bfcl/data/processed/bfcl_v4_baseline.csv` - Registered Profile A and Profile C results only. No BFCL source data, calculations, thresholds, profile definitions, prior reports, website files, manuscript files, or additional benchmarks are changed or used. ## Phase 1 Verification Profile A pair: | Displayed field | Direction | Qwen3-14B (Prompt) | Claude-Opus-4-5-20251101 (FC) | Verification | | --- | --- | ---: | ---: | --- | | Single-turn capability | Higher better | {float(a_q['single_turn_score']):.3f} | {float(a_c['single_turn_score']):.3f} | Qwen at least as good | | Relevance/irrelevance behavior | Higher better | {float(a_q['relevance_behavior_score']):.3f} | {float(a_c['relevance_behavior_score']):.3f} | Qwen better | | Benchmark cost | Lower better | ${float(a_q['total_cost_usd']):.2f} | ${float(a_c['total_cost_usd']):.2f} | Qwen better | | Measured P95 latency | Lower better | {float(a_q['latency_95th_percentile_s']):.2f}s | {float(a_c['latency_95th_percentile_s']):.2f}s | Qwen better | Profile A proceeds because Qwen is at least as good on every relevant displayed field and better on at least one. Profile C control pair: - BFCL aggregate winner: `Claude-Opus-4-5-20251101 (FC)`, Profile C rank {int(float(c_w['profile_rank']))}, Profile C score {float(c_w['profile_weighted_score']):.2f}. - Strongest appropriate frozen alternative: `GLM-4.6 (FC thinking)`, Profile C rank {int(float(c_alt['profile_rank']))}, Profile C score {float(c_alt['profile_weighted_score']):.2f}. ## Presentation Conditions Condition 1: EVIDENCE FIRST. The judge sees the work environment, conditions of satisfactory performance, and anonymized performance profiles. General BFCL rank is not displayed. Condition 2: CENTER FIRST. The judge sees general BFCL overall ranks first, then exactly the same work environment and anonymized performance profiles used in Condition 1. Condition 3: MRI ORDER. The judge first sees the same evidence-first card and must make a provisional selection. Only after that response is recorded, the rank information is revealed and the judge makes a final selection. The evidence, metrics, candidate data, and wording are held constant within each scenario. Only the presence and order of overall-rank information changes. ## Judge Families | Judge family | Fixed model ID | ZDR provider endpoint | Settings | | --- | --- | --- | --- | """ + "\n".join( f"| {j['judge_family']} | `{j['model']}` | `{j['provider_slug']}` | temperature {TEMPERATURE}, max_tokens {MAX_TOKENS}, provider.zdr=true, provider.data_collection=deny, provider.allow_fallbacks=false |" for j in JUDGES ) + f""" OpenRouter model and ZDR metadata were queried immediately before the pilot. `openrouter/auto` is not used. ## Trial Plan - Judge families: {len(JUDGES)} - Scenarios: {len(SCENARIOS)} (`Profile A`, `Profile C`) - Conditions: {len(CONDITIONS)} - Randomized trials per judge/scenario/condition: {TRIALS_PER_CELL} - Decision records: {len(trials)} - Completion calls projected: {projected_calls} - Randomization seed: {SEED} - Projected cost cap: ${MAX_PROJECTED_COST:.2f} - Projected cost before calls: ${projected_usd:.2f} ## Capture Fields Each decision record will capture judge family, scenario, condition, label mapping, candidate order, provisional choice where applicable, final choice, confidence, stated reasoning, overall-rank citation, scenario-specific evidence citation, whether the decision changed after rank disclosure, parsing/compliance errors, and usage/cost if returned. ## MRI Measures 1. CENTER PULL: change in selection rate of the BFCL aggregate winner under CENTER FIRST versus EVIDENCE FIRST. 2. TASK-FIT ACCURACY: selection rate of the frozen scenario-specific preferred option. 3. DECISION REGRET: best frozen scenario score minus selected option's frozen scenario score. 4. VARIATION SUPPRESSION: EVIDENCE FIRST scenario-evidence citation rate minus CENTER FIRST scenario-evidence citation rate. 5. CORRECTION CAPACITY: MRI ORDER provisional-to-final change frequency and direction after rank disclosure. 6. CROSS-MODEL CONSISTENCY: directional appearance across independent judge families. ## Verdict Rule PILOT PASS if, in Profile A, CENTER FIRST increases selection of the BFCL aggregate winner by at least 15 percentage points relative to EVIDENCE FIRST; the increase also raises decision regret; MRI ORDER produces lower decision regret than CENTER FIRST; and the pattern appears directionally in at least three of four judge families; while Profile C does not show a material loss in task-fit accuracy under MRI ORDER. PILOT MIXED if rank affects decisions but the effect is inconsistent, small, or not corrected by MRI ordering. PILOT FAIL if rank information produces no consistent center pull or MRI ordering does not improve the decision. """ (OUT / "00_PREREGISTRATION.md").write_text(text, encoding="utf-8") def write_cards(profile_rows, trials): examples = [] for scenario_name in SCENARIOS: scenario = SCENARIOS[scenario_name] default_map = {"Model A": scenario["aggregate_model"], "Model B": scenario["alternative_model"]} order = ["Model A", "Model B"] examples.append(f"## {scenario_name}: {scenario['title']}\n\n### Evidence Block\n\n{evidence_block(scenario_name, default_map, order, profile_rows)}\n\n### Rank Block\n\n{rank_block(scenario_name, default_map, order, profile_rows)}") text = f"""# Experimental Cards These are the exact wording templates used for MRI TECHNICAL PILOT 001. Candidate label assignment and display order are randomized on every run. The canonical examples below use one illustrative label assignment; actual mappings are recorded in `03_RESULTS.csv`. Condition 1, EVIDENCE FIRST, displays only the Evidence Block. Condition 2, CENTER FIRST, displays the Rank Block first, followed by the same Evidence Block. Condition 3, MRI ORDER, first displays the Evidence Block and records a provisional selection; only then does it display the Rank Block and record the final selection. """ + "\n\n".join(examples) (OUT / "01_EXPERIMENTAL_CARDS.md").write_text(text, encoding="utf-8") def run_trial(api_key, profile_rows, trial): judge = next(j for j in JUDGES if j["judge_family"] == trial["judge_family"]) messages = [{"role": "system", "content": system_prompt()}] raw_events = [] result = { **trial, "provisional_choice_label": "", "provisional_choice_model": "", "final_choice_label": "", "final_choice_model": "", "confidence": "", "stated_reasoning": "", "cited_overall_rank": "", "cited_scenario_specific_evidence": "", "decision_changed_after_rank_disclosure": "", "parsing_or_compliance_errors": "", "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, "actual_returned_model": "", } try: if trial["condition"] == "MRI_ORDER": user1 = provisional_prompt(trial["scenario"], trial["label_map"], trial["candidate_order"], profile_rows) messages1 = messages + [{"role": "user", "content": user1}] response1 = call_openrouter(api_key, judge, messages1) content1 = extract_content(response1) raw_events.append(("provisional", content1, response1)) obj1 = parse_json(content1) plabel = normalize_label(obj1.get("provisional_choice_label")) result["provisional_choice_label"] = plabel result["provisional_choice_model"] = trial["label_map"].get(plabel, "") result["actual_returned_model"] = response1.get("model") or result["actual_returned_model"] u1 = response1.get("usage") or {} result["input_tokens"] += int(u1.get("prompt_tokens") or 0) result["output_tokens"] += int(u1.get("completion_tokens") or 0) result["cost_usd"] += usage_cost(response1, judge) messages2 = messages1 + [{"role": "assistant", "content": content1}, {"role": "user", "content": mri_final_prompt(trial["scenario"], trial["label_map"], trial["candidate_order"], profile_rows)}] response2 = call_openrouter(api_key, judge, messages2) content2 = extract_content(response2) raw_events.append(("final", content2, response2)) obj2 = parse_json(content2) flabel = normalize_label(obj2.get("final_choice_label")) result["final_choice_label"] = flabel result["final_choice_model"] = trial["label_map"].get(flabel, "") result["confidence"] = obj2.get("confidence_0_100", "") result["stated_reasoning"] = str(obj2.get("stated_reasoning", "")) result["cited_overall_rank"] = normalize_bool(obj2.get("cited_overall_rank")) result["cited_scenario_specific_evidence"] = normalize_bool(obj2.get("cited_scenario_specific_evidence")) result["decision_changed_after_rank_disclosure"] = (plabel != flabel) if plabel and flabel else "" result["actual_returned_model"] = response2.get("model") or result["actual_returned_model"] u2 = response2.get("usage") or {} result["input_tokens"] += int(u2.get("prompt_tokens") or 0) result["output_tokens"] += int(u2.get("completion_tokens") or 0) result["cost_usd"] += usage_cost(response2, judge) else: user = final_prompt(trial["condition"], trial["scenario"], trial["label_map"], trial["candidate_order"], profile_rows) response = call_openrouter(api_key, judge, messages + [{"role": "user", "content": user}]) content = extract_content(response) raw_events.append(("final", content, response)) obj = parse_json(content) flabel = normalize_label(obj.get("final_choice_label")) result["final_choice_label"] = flabel result["final_choice_model"] = trial["label_map"].get(flabel, "") result["confidence"] = obj.get("confidence_0_100", "") result["stated_reasoning"] = str(obj.get("stated_reasoning", "")) result["cited_overall_rank"] = normalize_bool(obj.get("cited_overall_rank")) result["cited_scenario_specific_evidence"] = normalize_bool(obj.get("cited_scenario_specific_evidence")) result["decision_changed_after_rank_disclosure"] = "" result["actual_returned_model"] = response.get("model") or "" u = response.get("usage") or {} result["input_tokens"] += int(u.get("prompt_tokens") or 0) result["output_tokens"] += int(u.get("completion_tokens") or 0) result["cost_usd"] += usage_cost(response, judge) except urllib.error.HTTPError as exc: try: body = exc.read().decode("utf-8", "replace") except Exception: body = "" result["parsing_or_compliance_errors"] = f"HTTPError {exc.code}: {body[:300]}" raw_events.append(("error", result["parsing_or_compliance_errors"], {"error": result["parsing_or_compliance_errors"]})) except Exception as exc: result["parsing_or_compliance_errors"] = f"{type(exc).__name__}: {exc}" return result, raw_events def selected_score(profile_rows, scenario, model): profile_id = "A" if scenario == "Profile A" else "C" if not model: return None row = profile_rows.get((profile_id, model)) if not row: return None return fnum(row["profile_weighted_score"]) def add_outcomes(profile_rows, row): scenario = SCENARIOS[row["scenario"]] final_model = row["final_choice_model"] row["selected_is_bfcl_aggregate_winner"] = final_model == scenario["aggregate_model"] row["selected_is_scenario_preferred"] = final_model == scenario["preferred_model"] best = selected_score(profile_rows, row["scenario"], scenario["preferred_model"]) chosen = selected_score(profile_rows, row["scenario"], final_model) if best is None or chosen is None: row["decision_regret"] = "" else: row["decision_regret"] = max(0.0, best - chosen) if row["condition"] == "MRI_ORDER": prov_model = row["provisional_choice_model"] if prov_model and final_model: if prov_model != scenario["aggregate_model"] and final_model == scenario["aggregate_model"]: row["rank_disclosure_direction"] = "toward_aggregate_winner" elif prov_model == scenario["aggregate_model"] and final_model != scenario["aggregate_model"]: row["rank_disclosure_direction"] = "away_from_aggregate_winner" elif prov_model != final_model: row["rank_disclosure_direction"] = "other_change" else: row["rank_disclosure_direction"] = "no_change" else: row["rank_disclosure_direction"] = "" else: row["rank_disclosure_direction"] = "" return row def rate(rows, pred): valid = [r for r in rows if not r["parsing_or_compliance_errors"] and r["final_choice_model"]] if not valid: return None, 0 return sum(1 for r in valid if pred(r)) / len(valid), len(valid) def mean_regret(rows): vals = [float(r["decision_regret"]) for r in rows if r.get("decision_regret") not in ("", None) and not r["parsing_or_compliance_errors"]] if not vals: return None return sum(vals) / len(vals) def build_metrics(rows): metrics = [] groups = [("ALL", None)] + [(j["judge_family"], j["judge_family"]) for j in JUDGES] for scenario in SCENARIOS: for group_name, fam in groups: subset = [r for r in rows if r["scenario"] == scenario and (fam is None or r["judge_family"] == fam)] ef = [r for r in subset if r["condition"] == "EVIDENCE_FIRST"] cf = [r for r in subset if r["condition"] == "CENTER_FIRST"] mo = [r for r in subset if r["condition"] == "MRI_ORDER"] ef_agg, ef_n = rate(ef, lambda r: r["selected_is_bfcl_aggregate_winner"]) cf_agg, cf_n = rate(cf, lambda r: r["selected_is_bfcl_aggregate_winner"]) ef_pref, _ = rate(ef, lambda r: r["selected_is_scenario_preferred"]) cf_pref, _ = rate(cf, lambda r: r["selected_is_scenario_preferred"]) mo_pref, mo_n = rate(mo, lambda r: r["selected_is_scenario_preferred"]) ef_evid, _ = rate(ef, lambda r: bool(r["cited_scenario_specific_evidence"])) cf_evid, _ = rate(cf, lambda r: bool(r["cited_scenario_specific_evidence"])) mo_changed = [r for r in mo if not r["parsing_or_compliance_errors"] and r["decision_changed_after_rank_disclosure"] != ""] changed_rate = (sum(1 for r in mo_changed if r["decision_changed_after_rank_disclosure"] is True) / len(mo_changed)) if mo_changed else None toward = sum(1 for r in mo_changed if r["rank_disclosure_direction"] == "toward_aggregate_winner") away = sum(1 for r in mo_changed if r["rank_disclosure_direction"] == "away_from_aggregate_winner") metrics.append({ "scenario": scenario, "judge_family": group_name, "evidence_first_valid_n": ef_n, "center_first_valid_n": cf_n, "mri_order_valid_n": mo_n, "center_pull_pp": "" if ef_agg is None or cf_agg is None else (cf_agg - ef_agg) * 100, "evidence_first_aggregate_selection_rate": "" if ef_agg is None else ef_agg, "center_first_aggregate_selection_rate": "" if cf_agg is None else cf_agg, "evidence_first_task_fit_accuracy": "" if ef_pref is None else ef_pref, "center_first_task_fit_accuracy": "" if cf_pref is None else cf_pref, "mri_order_task_fit_accuracy": "" if mo_pref is None else mo_pref, "evidence_first_mean_regret": "" if mean_regret(ef) is None else mean_regret(ef), "center_first_mean_regret": "" if mean_regret(cf) is None else mean_regret(cf), "mri_order_mean_regret": "" if mean_regret(mo) is None else mean_regret(mo), "variation_suppression_pp": "" if ef_evid is None or cf_evid is None else (ef_evid - cf_evid) * 100, "evidence_first_scenario_evidence_citation_rate": "" if ef_evid is None else ef_evid, "center_first_scenario_evidence_citation_rate": "" if cf_evid is None else cf_evid, "mri_order_change_rate_after_rank_disclosure": "" if changed_rate is None else changed_rate, "mri_order_changes_toward_aggregate": toward, "mri_order_changes_away_from_aggregate": away, }) return metrics def verdict(metrics): all_a = next(m for m in metrics if m["scenario"] == "Profile A" and m["judge_family"] == "ALL") all_c = next(m for m in metrics if m["scenario"] == "Profile C" and m["judge_family"] == "ALL") try: center_pull = float(all_a["center_pull_pp"]) regret_increase = float(all_a["center_first_mean_regret"]) > float(all_a["evidence_first_mean_regret"]) mri_lower_regret = float(all_a["mri_order_mean_regret"]) < float(all_a["center_first_mean_regret"]) c_no_loss = float(all_c["mri_order_task_fit_accuracy"]) >= float(all_c["evidence_first_task_fit_accuracy"]) - 0.10 except Exception: return "PILOT FAIL", {"reason": "insufficient valid metrics"} directional_families = 0 family_details = {} for j in JUDGES: m = next(x for x in metrics if x["scenario"] == "Profile A" and x["judge_family"] == j["judge_family"]) try: cp = float(m["center_pull_pp"]) cr = float(m["center_first_mean_regret"]) er = float(m["evidence_first_mean_regret"]) mr = float(m["mri_order_mean_regret"]) ok = cp > 0 and cr > er and mr < cr except Exception: ok = False cp = "" family_details[j["judge_family"]] = {"center_pull_pp": cp, "directional_pattern": ok} if ok: directional_families += 1 if center_pull >= 15 and regret_increase and mri_lower_regret and directional_families >= 3 and c_no_loss: return "PILOT PASS", family_details if center_pull != 0 or regret_increase or directional_families > 0: return "PILOT MIXED", family_details return "PILOT FAIL", family_details def write_outputs(profile_rows, trials, results, raw_events_by_trial, metrics, final_verdict, verdict_details, total_cost): # Raw responses raw_lines = ["# Raw Responses", "", "Every raw model response content is preserved below. API keys and request bodies are not included.", ""] for r in results: raw_lines.append(f"## {r['trial_id']} - {r['judge_family']} - {r['scenario']} - {r['condition']}") raw_lines.append("") raw_lines.append(f"- Label mapping: {json.dumps(r['label_map'], sort_keys=True)}") raw_lines.append(f"- Candidate order: {json.dumps(r['candidate_order'])}") raw_lines.append("") for phase, content, response in raw_events_by_trial.get(r["trial_id"], []): raw_lines.append(f"### {phase}") raw_lines.append("") raw_lines.append("```text") raw_lines.append(str(content)) raw_lines.append("```") usage = response.get("usage") if isinstance(response, dict) else None model = response.get("model") if isinstance(response, dict) else None raw_lines.append("") raw_lines.append(f"- Returned model field present: {bool(model)}") raw_lines.append(f"- Usage: `{json.dumps(usage, sort_keys=True) if usage else ''}`") raw_lines.append("") (OUT / "02_RAW_RESPONSES.md").write_text("\n".join(raw_lines), encoding="utf-8") result_fields = [ "trial_id", "judge_family", "judge_model", "provider_slug", "scenario", "condition", "replicate", "label_map_json", "candidate_order_json", "provisional_choice_label", "provisional_choice_model", "final_choice_label", "final_choice_model", "confidence", "stated_reasoning", "cited_overall_rank", "cited_scenario_specific_evidence", "decision_changed_after_rank_disclosure", "rank_disclosure_direction", "selected_is_bfcl_aggregate_winner", "selected_is_scenario_preferred", "decision_regret", "input_tokens", "output_tokens", "cost_usd", "actual_returned_model", "parsing_or_compliance_errors", ] with (OUT / "03_RESULTS.csv").open("w", encoding="utf-8", newline="") as f: w = csv.DictWriter(f, fieldnames=result_fields) w.writeheader() for r in results: row = dict(r) row["label_map_json"] = json.dumps(r["label_map"], sort_keys=True) row["candidate_order_json"] = json.dumps(r["candidate_order"]) w.writerow({k: row.get(k, "") for k in result_fields}) metric_fields = list(metrics[0].keys()) with (OUT / "04_MRI_METRICS.csv").open("w", encoding="utf-8", newline="") as f: w = csv.DictWriter(f, fieldnames=metric_fields) w.writeheader() w.writerows(metrics) all_a = next(m for m in metrics if m["scenario"] == "Profile A" and m["judge_family"] == "ALL") all_c = next(m for m in metrics if m["scenario"] == "Profile C" and m["judge_family"] == "ALL") errors = [r for r in results if r["parsing_or_compliance_errors"]] total_valid = len(results) - len(errors) report = f"""# MRI TECHNICAL PILOT 001 Technical Report # 47 TO 1 - THE RANK BEFORE THE WORK Completed at: {now_iso()} ## Scope This pilot used only frozen BFCL Profile A and Profile C results. It did not alter BFCL data, profiles, thresholds, calculations, prior reports, or website files. It did not use the manuscript and did not introduce another benchmark. ## Experimental Design - Judge families: {', '.join(j['judge_family'] for j in JUDGES)} - Decision records planned: {len(trials)} - Decision records completed without parsing/API errors: {total_valid} - Completion calls: EVIDENCE FIRST and CENTER FIRST used one call per decision; MRI ORDER used a provisional call followed by a rank-disclosure final call. - Randomization seed: {SEED} - Generation settings: temperature {TEMPERATURE}, max_tokens {MAX_TOKENS}, fixed model IDs, ZDR provider endpoints, data collection denied, fallbacks disabled. - Total returned/estimated OpenRouter cost: ${total_cost:.4f} ## Phase 1 Pair Verification Profile A comparison was valid for the causal pilot: Qwen3-14B (Prompt) was at least as good as Claude-Opus-4-5-20251101 (FC) on every displayed Profile A-relevant field and better on at least one. In fact, Qwen was slightly higher on single-turn capability, higher on relevance/irrelevance behavior, lower cost, and lower P95 latency. Profile C control pair used the BFCL aggregate winner, Claude-Opus-4-5-20251101 (FC), and the strongest appropriate frozen Profile C alternative, GLM-4.6 (FC thinking). ## Main Metrics ### Profile A - CENTER PULL: {float(all_a['center_pull_pp']):.2f} percentage points. - EVIDENCE FIRST aggregate-winner selection rate: {float(all_a['evidence_first_aggregate_selection_rate']):.3f}. - CENTER FIRST aggregate-winner selection rate: {float(all_a['center_first_aggregate_selection_rate']):.3f}. - EVIDENCE FIRST task-fit accuracy: {float(all_a['evidence_first_task_fit_accuracy']):.3f}. - CENTER FIRST task-fit accuracy: {float(all_a['center_first_task_fit_accuracy']):.3f}. - MRI ORDER task-fit accuracy: {float(all_a['mri_order_task_fit_accuracy']):.3f}. - EVIDENCE FIRST mean regret: {float(all_a['evidence_first_mean_regret']):.3f}. - CENTER FIRST mean regret: {float(all_a['center_first_mean_regret']):.3f}. - MRI ORDER mean regret: {float(all_a['mri_order_mean_regret']):.3f}. - VARIATION SUPPRESSION: {float(all_a['variation_suppression_pp']):.2f} percentage points. - MRI ORDER change rate after rank disclosure: {float(all_a['mri_order_change_rate_after_rank_disclosure']):.3f}. - MRI ORDER changes toward aggregate winner: {all_a['mri_order_changes_toward_aggregate']}. - MRI ORDER changes away from aggregate winner: {all_a['mri_order_changes_away_from_aggregate']}. ### Profile C - CENTER PULL: {float(all_c['center_pull_pp']):.2f} percentage points. - EVIDENCE FIRST task-fit accuracy: {float(all_c['evidence_first_task_fit_accuracy']):.3f}. - CENTER FIRST task-fit accuracy: {float(all_c['center_first_task_fit_accuracy']):.3f}. - MRI ORDER task-fit accuracy: {float(all_c['mri_order_task_fit_accuracy']):.3f}. - EVIDENCE FIRST mean regret: {float(all_c['evidence_first_mean_regret']):.3f}. - CENTER FIRST mean regret: {float(all_c['center_first_mean_regret']):.3f}. - MRI ORDER mean regret: {float(all_c['mri_order_mean_regret']):.3f}. ## Cross-Model Consistency {json.dumps(verdict_details, indent=2, sort_keys=True)} ## Errors Parsing or compliance errors: {len(errors)} of {len(results)} decision records. ## Pre-Registered Verdict {final_verdict} """ (OUT / "05_TECHNICAL_REPORT.md").write_text(report, encoding="utf-8") plain = f"""# Plain Language Result The pilot tested whether showing the general BFCL overall rank before the work-specific evidence would pull independent decision systems toward the aggregate winner. In Profile A, the work-specific evidence favored Qwen3-14B (Prompt): it was at least as good on the displayed task-fit fields and better on cost, latency, and relevance behavior. The experiment then compared evidence-first decisions against decisions where the overall BFCL rank was shown first. The measured center pull for Profile A was {float(all_a['center_pull_pp']):.2f} percentage points. Mean decision regret was {float(all_a['center_first_mean_regret']):.3f} under CENTER FIRST and {float(all_a['mri_order_mean_regret']):.3f} under MRI ORDER. Profile C served as a control case where the aggregate winner was also the frozen scenario-specific preferred option. MRI ORDER task-fit accuracy in Profile C was {float(all_c['mri_order_task_fit_accuracy']):.3f}. This pilot does not establish commercial value, does not create a new benchmark, and does not revise BFCL. It only tests whether authoritative rank placement can change model-selection behavior when the underlying work-specific evidence remains constant. {final_verdict} """ (OUT / "06_PLAIN_LANGUAGE_RESULT.md").write_text(plain, encoding="utf-8") def main(): OUT.mkdir(parents=True, exist_ok=True) api_key = load_api_key() profile_rows = load_profile_rows() phase1_checks, c_w, c_alt = verify_phase1(profile_rows) trials = generate_trials(profile_rows) projected_calls, projected_usd = projected_cost() write_prereg(profile_rows, phase1_checks, c_w, c_alt, trials, projected_calls, projected_usd) write_cards(profile_rows, trials) if projected_usd > MAX_PROJECTED_COST: raise RuntimeError(f"Projected cost ${projected_usd:.2f} exceeds cap ${MAX_PROJECTED_COST:.2f}; stopping before calls.") results = [] raw_events_by_trial = {} total_cost = 0.0 max_workers = 8 with ThreadPoolExecutor(max_workers=max_workers) as pool: futures = {pool.submit(run_trial, api_key, profile_rows, trial): trial for trial in trials} for i, fut in enumerate(as_completed(futures), 1): result, raw_events = fut.result() result = add_outcomes(profile_rows, result) results.append(result) raw_events_by_trial[result["trial_id"]] = raw_events total_cost += float(result.get("cost_usd") or 0.0) if i % 24 == 0: print(f"completed {i}/{len(trials)} cost=${total_cost:.4f}", flush=True) results.sort(key=lambda r: r["trial_id"]) metrics = build_metrics(results) final_verdict, verdict_details = verdict(metrics) write_outputs(profile_rows, trials, results, raw_events_by_trial, metrics, final_verdict, verdict_details, total_cost) print("FINAL_VERDICT", final_verdict) print("TOTAL_COST", f"{total_cost:.4f}") if __name__ == "__main__": main()