#!/usr/bin/env python3 import argparse import csv import hashlib import json import math import os import random import re import urllib.request import urllib.error from datetime import datetime, timezone from html import unescape from html.parser import HTMLParser from pathlib import Path from urllib.parse import unquote, urlparse from typing import Any, Dict, List, Optional USER_AGENT = 'AFI-Needle-Test-Lab/6.2.5' _FETCH_CACHE: Dict[str, Dict[str, Any]] = {} _COMPONENT_CACHE: Dict[tuple, Dict[str, Any]] = {} _PATH_REFEREE_CACHE: Dict[tuple, Dict[str, Any]] = {} _COMPLETION_REFEREE_CACHE: Dict[tuple, Dict[str, Any]] = {} _SPECIALIST_TEMPORAL_CACHE: Dict[tuple, Dict[str, Any]] = {} _V6_EX_ANTE_GATE_CACHE: Dict[tuple, Dict[str, Any]] = {} _V6_BENCHMARK_SIGNAL_CACHE: Dict[tuple, Dict[str, Any]] = {} _V6_COMPLETION_BENCHMARK_CACHE: Dict[tuple, Dict[str, Any]] = {} MEDIA_MARKET_DIR = Path(__file__).resolve().parent / 'media_markets' REPORTING_PROFILE_DIR = Path(__file__).resolve().parent / 'reporting_profiles' # Massachusetts VoteFlow uses City/Town geography while the existing AFI media # market files are county/FIPS based. The bridge below resolves each MA # municipality to its county FIPS and caches the result on disk. MA_TIGER_COUSUB_URL = ( "https://tigerweb.geo.census.gov/tigerwebmain/Files/acs25/" "tigerweb_acs25_cousub_ma.html" ) MA_MUNICIPAL_COUNTY_CACHE_PATH = Path( os.environ.get( "AFI_MA_MUNICIPAL_COUNTY_CACHE", str(MEDIA_MARKET_DIR / "ma_municipality_counties.cache.json"), ) ) MA_MUNICIPAL_COUNTY_FALLBACK_CACHE_PATH = Path( "/tmp/afi_needle_ma_municipality_counties.cache.json" ) _MA_MUNICIPAL_COUNTY_CACHE: Optional[Dict[str, str]] = None _MA_MUNICIPAL_COUNTY_CACHE_SOURCE: Optional[str] = None _MA_MUNICIPAL_COUNTY_LOAD_ATTEMPTED = False def fetch_json(url: str) -> Dict[str, Any]: if url in _FETCH_CACHE: return _FETCH_CACHE[url] # Live VoteFlow can invoke the Needle against the engine's local race # directory. This avoids waiting for the public data host/CDN to reflect a # snapshot that was written only milliseconds earlier. parsed = urlparse(url) if parsed.scheme == "file": local_path = Path(unquote(parsed.path)) with local_path.open("r", encoding="utf-8") as f: data = json.load(f) elif parsed.scheme == "" and Path(url).is_file(): with Path(url).open("r", encoding="utf-8") as f: data = json.load(f) else: req = urllib.request.Request( url, headers={'User-Agent': USER_AGENT, 'Accept': 'application/json'} ) with urllib.request.urlopen(req, timeout=20) as r: data = json.load(r) _FETCH_CACHE[url] = data return data def save_json(path: Path, data: Dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(path.suffix + '.tmp') with tmp.open('w', encoding='utf-8') as f: json.dump(data, f, indent=2) tmp.replace(path) def slug_base(url: str) -> str: return url.rstrip('/') + '/' def candidate_map(data: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: out = {} for c in data.get('candidates') or []: name = str(c.get('name') or '').strip() if not name: continue out[name] = { 'name': name, 'party': c.get('party'), 'color': c.get('color'), 'votes': int(c.get('votes') or 0), 'share': float(c.get('percent') or 0.0) / 100.0, 'major_candidate': bool(c.get('major_candidate', True)), } return out def shares_from_votes(cands: Dict[str, Dict[str, Any]]) -> Dict[str, float]: total = sum(c['votes'] for c in cands.values()) return {name: (c['votes'] / total if total else 0.0) for name, c in cands.items()} def rank(cands: Dict[str, Dict[str, Any]], shares: Dict[str, float]) -> List[str]: return sorted(cands.keys(), key=lambda n: (shares.get(n, 0.0), cands[n]['votes']), reverse=True) def _score_projection(base_url, snapshot_name, snapshot, final, projected_shares, model_name, method_note, extra_diagnostics=None): snap_candidates = candidate_map(snapshot) final_candidates = candidate_map(final) all_names = sorted(set(snap_candidates) | set(final_candidates) | set(projected_shares)) snap_shares = shares_from_votes(snap_candidates) final_shares = shares_from_votes(final_candidates) projected_shares = {n: max(0.0, float(projected_shares.get(n, 0.0))) for n in all_names} total_projected = sum(projected_shares.values()) if total_projected > 0: projected_shares = {n: v / total_projected for n, v in projected_shares.items()} else: projected_shares = {n: snap_shares.get(n, 0.0) for n in all_names} proj_rank = sorted(all_names, key=lambda n: projected_shares.get(n, 0.0), reverse=True) final_rank = rank(final_candidates, final_shares) predicted_leader = proj_rank[0] if proj_rank else None predicted_runner = proj_rank[1] if len(proj_rank) > 1 else None final_leader = final_rank[0] if final_rank else None final_runner = final_rank[1] if len(final_rank) > 1 else None projected_margin = None if predicted_leader and predicted_runner: projected_margin = 100 * ( projected_shares.get(predicted_leader, 0.0) - projected_shares.get(predicted_runner, 0.0) ) actual_margin = None if final_leader and final_runner: actual_margin = 100 * ( final_shares.get(final_leader, 0.0) - final_shares.get(final_runner, 0.0) ) winner_projected_share = 100 * projected_shares.get(final_leader, 0.0) if final_leader else None winner_final_share = 100 * final_shares.get(final_leader, 0.0) if final_leader else None winner_share_error = ( winner_projected_share - winner_final_share if winner_projected_share is not None and winner_final_share is not None else None ) winner_projected_margin = None winner_margin_error = None if final_leader: opponent = max( (s for n, s in projected_shares.items() if n != final_leader), default=0.0, ) winner_projected_margin = 100 * (projected_shares.get(final_leader, 0.0) - opponent) winner_margin_error = ( winner_projected_margin - actual_margin if actual_margin is not None else None ) threshold = 50.0 projected_above = ( winner_projected_share >= threshold if winner_projected_share is not None else None ) final_above = winner_final_share >= threshold if winner_final_share is not None else None threshold_correct = ( projected_above == final_above if projected_above is not None and final_above is not None else None ) final_pair_projected_margin = None final_pair_margin_error = None if final_leader and final_runner: final_pair_projected_margin = 100 * ( projected_shares.get(final_leader, 0.0) - projected_shares.get(final_runner, 0.0) ) final_pair_margin_error = final_pair_projected_margin - actual_margin rows, abs_errors, sq_errors = [], [], [] for name in all_names: sc = snap_candidates.get(name, {}) fc = final_candidates.get(name, {}) ps = projected_shares.get(name, 0.0) fs = final_shares.get(name, 0.0) err = 100 * (ps - fs) abs_errors.append(abs(err)) sq_errors.append(err * err) rows.append({ "name": name, "party": sc.get("party") or fc.get("party"), "color": sc.get("color") or fc.get("color"), "major_candidate": bool(sc.get("major_candidate") or fc.get("major_candidate")), "current_votes": int(sc.get("votes") or 0), "current_share": snap_shares.get(name, 0.0), "projected_share": ps, "final_votes": int(fc.get("votes") or 0), "final_share": fs, "share_error_points": err, }) rows.sort(key=lambda x: x["projected_share"], reverse=True) current_votes = sum(c["votes"] for c in snap_candidates.values()) final_votes = sum(c["votes"] for c in final_candidates.values()) actual_counted = 100 * current_votes / final_votes if final_votes else None diagnostics = { "method": method_note, "uses_final_results_for_projection": False, "uses_final_results_for_scoring_only": True, "reporting_warning": ( "VoteFlow percent_reporting is used as a reporting-unit completion signal. " "It is not assumed to equal statewide percent of votes counted." ), } if extra_diagnostics: diagnostics.update(extra_diagnostics) return { "schema_version": 5, "model": model_name, "source": { "race_url": base_url, "snapshot": snapshot_name, "snapshot_url": base_url + snapshot_name, "final_url": base_url + "latest.json", }, "race_name": snapshot.get("election_name") or final.get("election_name"), "state": snapshot.get("province") or final.get("province"), "race_type": snapshot.get("election_type") or final.get("election_type"), "scope": snapshot.get("election_scope") or final.get("election_scope"), "timestamp": snapshot.get("last_updated"), "percent_reporting_source": snapshot.get("percent_reporting"), "votes_reported": current_votes, "final_votes_backtest_only": final_votes, "actual_vote_counted_pct_backtest_only": actual_counted, "predicted_leader": predicted_leader, "projected_margin_points": projected_margin, "final_leader": final_leader, "actual_final_margin_points": actual_margin, "leader_correct": predicted_leader == final_leader if predicted_leader and final_leader else None, "winner_projection": { "name": final_leader, "projected_share_points": winner_projected_share, "final_share_points": winner_final_share, "share_error_points": winner_share_error, "projected_margin_points": winner_projected_margin, "final_margin_points": actual_margin, "margin_error_points": winner_margin_error, }, "runoff_threshold": { "threshold_pct": threshold, "projected_winner_above_threshold": projected_above, "final_winner_above_threshold": final_above, "threshold_call_correct": threshold_correct, }, "competitive_state": { "tossup_band_points": 1.0, "projected_state": ( "TOSSUP" if winner_projected_margin is not None and abs(winner_projected_margin) <= 1.0 else ("WINNER" if winner_projected_margin is not None and winner_projected_margin > 1.0 else "OTHER") ), "final_state": ( "TOSSUP" if actual_margin is not None and abs(actual_margin) <= 1.0 else "WINNER" ), "state_call_correct": ( ( "TOSSUP" if winner_projected_margin is not None and abs(winner_projected_margin) <= 1.0 else ("WINNER" if winner_projected_margin is not None and winner_projected_margin > 1.0 else "OTHER") ) == ( "TOSSUP" if actual_margin is not None and abs(actual_margin) <= 1.0 else "WINNER" ) if winner_projected_margin is not None and actual_margin is not None else None ), }, "final_top_two": [final_leader, final_runner] if final_leader and final_runner else [], "final_top_two_projected_margin_points": final_pair_projected_margin, "final_top_two_margin_error_points": final_pair_margin_error, "candidate_share_mae_points": sum(abs_errors) / len(abs_errors) if abs_errors else None, "candidate_share_rmse_points": math.sqrt(sum(sq_errors) / len(sq_errors)) if sq_errors else None, "projection": rows, "diagnostics": diagnostics, } def project_v0(base_url: str, snapshot_name: str) -> Dict[str, Any]: base_url = slug_base(base_url) snapshot = fetch_json(base_url + snapshot_name) final = fetch_json(base_url + "latest.json") shares = shares_from_votes(candidate_map(snapshot)) return _score_projection( base_url, snapshot_name, snapshot, final, shares, "V0_BENCHMARK", "V0 freezes the current statewide candidate shares as the projected final shares." ) def county_regions(data: Dict[str, Any]) -> List[Dict[str, Any]]: """ Return the geographic layer used by the Needle. Most states use counties. Massachusetts is the exception: VoteFlow exposes hundreds of City/Town rows in addition to 14 overlapping county totals. When a substantial MA municipal layer is present, use ONLY municipalities so statewide votes are never double-counted and historical PD43+ City/Town files line up directly with the live feed. The function keeps its historical name because the rest of V6.2.2 calls this helper everywhere; the projection math itself is unchanged. """ regions = data.get("region_results") if not isinstance(regions, dict): return [] counties = [] municipalities = [] for key, region in regions.items(): if not isinstance(region, dict): continue region_type = str(region.get("type") or "").strip().casefold() item = dict(region) item["_key"] = key if region_type == "county": counties.append(item) elif region_type in { "town", "city", "municipality", "city/town", "city or town", }: municipalities.append(item) state = str( data.get("province") or data.get("state") or data.get("state_abbr") or "" ).strip().upper() # Massachusetts has 351 municipalities. A threshold well below that lets # the model keep working even if the live provider temporarily omits a few # municipalities, while avoiding accidental use of a tiny local subset. if state == "MA" and len(municipalities) >= 100: return municipalities return counties def project_v1(base_url: str, snapshot_name: str) -> Dict[str, Any]: """ V1 Geographic Completion: project each reporting county separately, then aggregate those county projections statewide. estimated county final vote = current county vote / reporting_fraction candidate projected county vote = estimate * current county candidate share 0%-reporting counties cannot yet be sized without historical turnout baselines. """ base_url = slug_base(base_url) snapshot = fetch_json(base_url + snapshot_name) final = fetch_json(base_url + "latest.json") snap_candidates = candidate_map(snapshot) v0_shares = shares_from_votes(snap_candidates) projected_votes = {name: 0.0 for name in snap_candidates} counties = county_regions(snapshot) used = 0 zero_reporting = 0 zero_votes = 0 capped = 0 covered_current_votes = 0 estimated_reporting_county_votes = 0.0 MAX_EXPANSION = 50.0 # prevents a 0.1% source reading from exploding the model for region in counties: candidate_rows = region.get("candidates") if not isinstance(candidate_rows, list): continue county_votes = {} for c in candidate_rows: if not isinstance(c, dict): continue name = str(c.get("name") or "").strip() if not name: continue county_votes[name] = int(c.get("votes") or 0) projected_votes.setdefault(name, 0.0) county_total = sum(county_votes.values()) if county_total <= 0: zero_votes += 1 continue try: reporting_pct = float(region.get("percent_reporting") or 0) except (TypeError, ValueError): reporting_pct = 0.0 if reporting_pct <= 0: zero_reporting += 1 continue fraction = min(1.0, reporting_pct / 100.0) raw_expansion = 1.0 / fraction expansion = min(raw_expansion, MAX_EXPANSION) if raw_expansion > MAX_EXPANSION: capped += 1 estimated_final = county_total * expansion covered_current_votes += county_total estimated_reporting_county_votes += estimated_final used += 1 for name, votes in county_votes.items(): projected_votes[name] += estimated_final * (votes / county_total) projected_total = sum(projected_votes.values()) fallback = projected_total <= 0 if fallback: v1_shares = v0_shares else: v1_shares = {name: votes / projected_total for name, votes in projected_votes.items()} statewide_current = sum(c["votes"] for c in snap_candidates.values()) coverage = ( 100 * covered_current_votes / statewide_current if statewide_current else 0.0 ) result = _score_projection( base_url, snapshot_name, snapshot, final, v1_shares, "V1_GEOGRAPHIC_COMPLETION", ( "V1 estimates a final vote total for every reporting county using that " "county's VoteFlow percent_reporting, preserves that county's current " "candidate shares, and sums the county projections statewide." ), { "v1_limitation": ( "Counties with 0 reported votes / 0% reporting have no turnout size " "yet. Historical county turnout baselines are the next model layer." ), "v1_max_expansion_factor": MAX_EXPANSION, } ) result["geographic"] = { "counties_total": len(counties), "counties_used": used, "counties_zero_reporting": zero_reporting, "counties_zero_votes": zero_votes, "counties_expansion_capped": capped, "current_votes_covered": covered_current_votes, "current_vote_coverage_pct": coverage, "estimated_votes_reporting_counties": estimated_reporting_county_votes, "fell_back_to_v0": fallback, } return result # Some VoteFlow runoff directories do not use the exact same office slug as # their Round 1 directories. Keep those source-layout quirks in one place so # every runoff-aware model resolves the same baseline consistently. # # Key: (STATE, RUNOFF_OFFICE_SLUG) # Value: ROUND_1_OFFICE_SLUG _RUNOFF_ROUND1_OFFICE_ALIASES = { ("SC", "Special_Senate"): "US_Senate_Special", } def round1_baseline_url(runoff_base_url: str) -> Optional[str]: """ Convert: .../STATE/Runoff/OFFICE/DISTRICT/PARTY/ into: .../STATE/ROUND1_OFFICE/DISTRICT/PARTY/latest.json Normally ROUND1_OFFICE == OFFICE. A small alias table handles VoteFlow source-layout exceptions such as South Carolina's U.S. Senate special: runoff: .../SC/Runoff/Special_Senate/00/republican/ round 1: .../SC/US_Senate_Special/00/republican/ """ runoff_base_url = slug_base(runoff_base_url) marker = "/Runoff/" if marker not in runoff_base_url: return None before, after = runoff_base_url.split(marker, 1) # `before` ends in the state slug for VoteFlow's standard structure. state_slug = unquote( before.rstrip("/").rsplit("/", 1)[-1] ).upper() parts = after.strip("/").split("/") if len(parts) < 3: return None runoff_office = unquote(parts[0]) round1_office = _RUNOFF_ROUND1_OFFICE_ALIASES.get( (state_slug, runoff_office), runoff_office, ) parts[0] = round1_office round1_path = "/".join(parts) return ( before.rstrip("/") + "/" + round1_path + "/latest.json" ) def _region_identity(region: Dict[str, Any]) -> str: """Prefer FIPS; otherwise use a stable normalized geographic name.""" fips = str(region.get("fips") or "").strip() if fips: return "fips:" + fips region_type = str(region.get("type") or "").strip().casefold() name_value = region.get("name") or region.get("_key") or "" if region_type in { "town", "city", "municipality", "city/town", "city or town", }: # Massachusetts CivicAPI disambiguates the eight municipality/county # name collisions as "Barnstable Township", "Worcester Township", etc., # while official PD43+ historical CSVs use the bare municipality name. name = normalize_ma_municipality_name(name_value) else: name = str(name_value).strip().casefold() name = re.sub(r"[^a-z0-9]+", "_", name).strip("_") return "name:" + name def _candidate_votes_in_region(region: Dict[str, Any], names: List[str]) -> Dict[str, int]: wanted = set(names) out = {name: 0 for name in names} rows = region.get("candidates") if not isinstance(rows, list): return out for c in rows: if not isinstance(c, dict): continue name = str(c.get("name") or "").strip() if name in wanted: out[name] = int(c.get("votes") or 0) return out def _normalized(values: Dict[str, float], names: List[str]) -> Dict[str, float]: total = sum(max(0.0, float(values.get(n, 0.0))) for n in names) if total <= 0: equal = 1.0 / len(names) if names else 0.0 return {n: equal for n in names} return { n: max(0.0, float(values.get(n, 0.0))) / total for n in names } def _solve_linear_system(matrix: List[List[float]], vector: List[float]) -> List[float]: """Small dense Gaussian-elimination solver used by V5 ridge regressions.""" n = len(vector) if n == 0: return [] a = [list(matrix[i]) + [float(vector[i])] for i in range(n)] for col in range(n): pivot = max(range(col, n), key=lambda r: abs(a[r][col])) if abs(a[pivot][col]) < 1e-12: continue if pivot != col: a[col], a[pivot] = a[pivot], a[col] div = a[col][col] a[col] = [x / div for x in a[col]] for row in range(n): if row == col: continue factor = a[row][col] if abs(factor) < 1e-15: continue a[row] = [ a[row][j] - factor * a[col][j] for j in range(n + 1) ] return [a[i][n] if abs(a[i][i]) > 1e-12 else 0.0 for i in range(n)] def _weighted_ridge_fit( rows: List[Dict[str, Any]], feature_names: List[str], target_key: str, ridge: float = 0.05, ) -> Dict[str, Any]: """Weighted ridge regression with an unpenalized intercept.""" if len(rows) < max(3, len(feature_names) + 1): return { "available": False, "intercept": 0.0, "coefficients": {name: 0.0 for name in feature_names}, "rows": len(rows), } p = 1 + len(feature_names) xtwx = [[0.0 for _ in range(p)] for _ in range(p)] xtwy = [0.0 for _ in range(p)] for row in rows: w = max(1e-6, float(row.get("weight") or 0.0)) x = [1.0] + [float((row.get("features") or {}).get(name) or 0.0) for name in feature_names] y = float(row.get(target_key) or 0.0) for i in range(p): xtwy[i] += w * x[i] * y for j in range(p): xtwx[i][j] += w * x[i] * x[j] scale = max(1.0, sum(float(r.get("weight") or 0.0) for r in rows)) for i in range(1, p): xtwx[i][i] += ridge * scale beta = _solve_linear_system(xtwx, xtwy) return { "available": True, "intercept": beta[0] if beta else 0.0, "coefficients": { feature_names[i]: beta[i + 1] if i + 1 < len(beta) else 0.0 for i in range(len(feature_names)) }, "rows": len(rows), } def _ridge_predict(fit: Dict[str, Any], features: Dict[str, float]) -> float: value = float(fit.get("intercept") or 0.0) for name, beta in (fit.get("coefficients") or {}).items(): value += float(beta or 0.0) * float(features.get(name) or 0.0) return value def _weighted_mean(values: List[tuple], default: float = 0.0) -> float: num = 0.0 den = 0.0 for value, weight in values: try: value = float(value) weight = float(weight) except (TypeError, ValueError): continue if weight <= 0: continue num += value * weight den += weight return num / den if den > 0 else default def _cosine_similarity(a: Dict[str, float], b: Dict[str, float], names: List[str]) -> float: dot = sum(float(a.get(n, 0.0)) * float(b.get(n, 0.0)) for n in names) aa = sum(float(a.get(n, 0.0)) ** 2 for n in names) bb = sum(float(b.get(n, 0.0)) ** 2 for n in names) if aa <= 1e-12 or bb <= 1e-12: return 0.0 return max(-1.0, min(1.0, dot / math.sqrt(aa * bb))) def _safe_reporting_fraction(region: Optional[Dict[str, Any]]) -> float: if not region: return 0.0 try: return min(1.0, max(0.0, float(region.get("percent_reporting") or 0.0) / 100.0)) except (TypeError, ValueError): return 0.0 def _model_county_rows(model_key: str, model_result: Dict[str, Any]) -> List[Dict[str, Any]]: if model_key == "v3": return (model_result.get("v3") or {}).get("counties") or [] if model_key == "market": return (model_result.get("media_markets") or {}).get("counties") or [] if model_key == "histgeo": return (model_result.get("historical_candidate_geo") or {}).get("counties") or [] if model_key == "latent": return (model_result.get("latent_coalition") or {}).get("counties") or [] if model_key == "batchbias": return (model_result.get("batch_bias") or {}).get("counties") or [] if model_key == "runoff_transfer": return (model_result.get("runoff_transfer") or {}).get("counties") or [] if model_key == "precinct": return (model_result.get("precinct_hierarchy") or {}).get("counties") or [] return [] def _model_internal_validation(model_key: str, model_result: Dict[str, Any]) -> Dict[str, Any]: if model_key == "runoff_transfer": d = model_result.get("runoff_transfer") or {} return { "available": bool(d.get("cross_validation_available")), "score": float(d.get("cross_validation_score") or 0.0), "mae_points": ( d.get("cross_validation_aggregate_margin_mae_points") if d.get("cross_validation_aggregate_margin_mae_points") is not None else d.get("cross_validation_mae_points") ), "share_mae_points": d.get("cross_validation_mae_points"), "turnout_log_mae": d.get("cross_validation_turnout_log_mae"), "aggregate_margin_mae_points": d.get( "cross_validation_aggregate_margin_mae_points" ), "evidence": float(d.get("training_evidence") or 0.0), "exam": "held-out runoff transfer + turnout + aggregate margin", } if model_key == "histgeo": d = model_result.get("historical_candidate_geo") or {} return { "available": bool(d.get("cross_validation_available")), "score": float(d.get("cross_validation_score") or 0.0), "mae_points": d.get("cross_validation_mae_points"), "evidence": float(d.get("reporting_evidence") or 0.0), "exam": "held-out historical coalition geography", } if model_key == "latent": d = model_result.get("latent_coalition") or {} return { "available": bool(d.get("cross_validation_available")), "score": float(d.get("cross_validation_score") or 0.0), "mae_points": d.get("cross_validation_mae_points"), "evidence": float(d.get("training_evidence") or 0.0), "exam": "held-out latent coalition factors", } if model_key == "batchbias": d = model_result.get("batch_bias") or {} return { "available": bool(d.get("internal_validation_available")), "score": float(d.get("internal_validation_score") or 0.0), "mae_points": d.get("internal_validation_mae_points"), "evidence": float(d.get("confidence") or 0.0), "exam": "prior-batch residual validation", } if model_key == "precinct": d = model_result.get("precinct_hierarchy") or {} return { "available": bool(d.get("available")), "score": float(d.get("confidence") or 0.0), "mae_points": None, "evidence": float(d.get("confidence") or 0.0), "exam": "precinct coverage/reliability", } if model_key == "benchmark_completion": d = model_result.get("benchmark_completion") or {} return { "available": bool(d.get("available")), "score": float(d.get("validation_quality") or 0.0), "mae_points": d.get("validation_mae_points"), "evidence": float(d.get("confidence") or 0.0), "exam": "material county-maturation benchmark completion", } return { "available": False, "score": 0.0, "mae_points": None, "evidence": 0.0, "exam": None, } def project_v2r(base_url: str, snapshot_name: str) -> Optional[Dict[str, Any]]: """ V2R: Same-Cycle Runoff Baseline. This model is only available for /Runoff/ races. It uses the final Round 1 result as information that was genuinely known before the runoff: 1. Round 1 total county turnout defines the expected geographic footprint. 2. The runoff candidates' Round 1 county votes are normalized to create a two-way (or N-way) county baseline. 3. Reporting runoff counties show how candidate support is moving relative to that baseline; a shrunk statewide live swing is learned. 4. Each county starts from the adjusted Round 1 baseline and blends toward its live runoff share as that county reports. 5. County turnout weights start at the Round 1 geographic footprint and gradually blend toward the current observed runoff footprint. The runoff latest.json is scoring-only. The Round 1 latest.json is a legitimate pre-runoff input. """ base_url = slug_base(base_url) baseline_url = round1_baseline_url(base_url) if not baseline_url: return None snapshot = fetch_json(base_url + snapshot_name) final = fetch_json(base_url + "latest.json") baseline = fetch_json(baseline_url) snap_candidates = candidate_map(snapshot) runoff_names = list(snap_candidates.keys()) if len(runoff_names) < 2: return None baseline_counties = { _region_identity(r): r for r in county_regions(baseline) } live_counties = { _region_identity(r): r for r in county_regions(snapshot) } # Build Round 1 statewide/county turnout footprint using ALL Round 1 candidates. baseline_county_totals: Dict[str, int] = {} round1_total_votes = 0 for cid, region in baseline_counties.items(): rows = region.get("candidates") if not isinstance(rows, list): continue total = sum( int(c.get("votes") or 0) for c in rows if isinstance(c, dict) ) if total > 0: baseline_county_totals[cid] = total round1_total_votes += total if round1_total_votes <= 0: return None baseline_weights = { cid: total / round1_total_votes for cid, total in baseline_county_totals.items() } # Round 1 normalized county baseline among ONLY the runoff candidates. county_baseline_shares: Dict[str, Dict[str, float]] = {} usable_baseline_counties = 0 for cid, region in baseline_counties.items(): pair_votes = _candidate_votes_in_region(region, runoff_names) if sum(pair_votes.values()) <= 0: continue county_baseline_shares[cid] = _normalized(pair_votes, runoff_names) usable_baseline_counties += 1 # Statewide fallback baseline among runoff candidates. baseline_state_candidates = candidate_map(baseline) baseline_state_pair = { n: float((baseline_state_candidates.get(n) or {}).get("votes") or 0) for n in runoff_names } baseline_state_shares = _normalized(baseline_state_pair, runoff_names) # Learn a live candidate swing relative to the Round 1 normalized baseline. swing_num = {n: 0.0 for n in runoff_names} swing_den = 0.0 reporting_geo_progress = 0.0 reporting_counties_used = 0 for cid, base_share in county_baseline_shares.items(): live = live_counties.get(cid) base_weight = baseline_weights.get(cid, 0.0) if not live or base_weight <= 0: continue try: reporting_fraction = min( 1.0, max(0.0, float(live.get("percent_reporting") or 0.0) / 100.0) ) except (TypeError, ValueError): reporting_fraction = 0.0 reporting_geo_progress += base_weight * reporting_fraction live_votes = _candidate_votes_in_region(live, runoff_names) live_total = sum(live_votes.values()) if live_total <= 0: continue live_share = _normalized(live_votes, runoff_names) # A county can be informative before it is complete, but not at full weight. local_confidence = max(0.03, math.sqrt(reporting_fraction)) weight = base_weight * local_confidence if weight <= 0: continue reporting_counties_used += 1 swing_den += weight for name in runoff_names: # Clamp individual county swing so one bizarre early county cannot # create an absurd statewide move. delta = live_share[name] - base_share[name] delta = max(-0.25, min(0.25, delta)) swing_num[name] += weight * delta raw_swing = { n: (swing_num[n] / swing_den if swing_den > 0 else 0.0) for n in runoff_names } # Shrink live swing early. Round 1 remains the anchor until there is evidence. current_statewide_votes = sum(c["votes"] for c in snap_candidates.values()) current_vs_round1 = ( current_statewide_votes / round1_total_votes if round1_total_votes > 0 else 0.0 ) evidence_by_votes = min(1.0, math.sqrt(max(0.0, current_vs_round1) / 0.20)) evidence_by_geography = min( 1.0, math.sqrt(max(0.0, reporting_geo_progress) / 0.25) ) swing_shrink = max(0.0, min(1.0, 0.5 * evidence_by_votes + 0.5 * evidence_by_geography)) live_swing = {n: raw_swing[n] * swing_shrink for n in runoff_names} # Current county vote footprint, used only as a gradual turnout-weight update. current_county_totals = {} current_county_sum = 0 for cid, live in live_counties.items(): votes = _candidate_votes_in_region(live, runoff_names) total = sum(votes.values()) if total > 0: current_county_totals[cid] = total current_county_sum += total current_weights = { cid: total / current_county_sum for cid, total in current_county_totals.items() } if current_county_sum > 0 else {} # Keep early turnout geography anchored to Round 1; converge toward observed # county vote distribution as reporting geography matures. turnout_blend = max(0.0, min(1.0, reporting_geo_progress)) all_county_ids = set(baseline_weights) | set(current_weights) projected_state_shares = {n: 0.0 for n in runoff_names} total_weight = 0.0 counties_projected = 0 for cid in all_county_ids: base_weight = baseline_weights.get(cid, 0.0) current_weight = current_weights.get(cid, 0.0) county_weight = ( (1.0 - turnout_blend) * base_weight + turnout_blend * current_weight ) if county_weight <= 0: continue base_share = county_baseline_shares.get(cid, baseline_state_shares) # Apply the statewide observed swing to the Round 1 county baseline. adjusted = { n: base_share.get(n, baseline_state_shares.get(n, 0.0)) + live_swing[n] for n in runoff_names } adjusted = _normalized(adjusted, runoff_names) live = live_counties.get(cid) if live: live_votes = _candidate_votes_in_region(live, runoff_names) live_total = sum(live_votes.values()) try: reporting_fraction = min( 1.0, max(0.0, float(live.get("percent_reporting") or 0.0) / 100.0) ) except (TypeError, ValueError): reporting_fraction = 0.0 if live_total > 0: live_share = _normalized(live_votes, runoff_names) # Local result gradually replaces the historical expectation. local_blend = math.sqrt(reporting_fraction) county_share = { n: ( (1.0 - local_blend) * adjusted[n] + local_blend * live_share[n] ) for n in runoff_names } county_share = _normalized(county_share, runoff_names) else: county_share = adjusted else: county_share = adjusted counties_projected += 1 total_weight += county_weight for n in runoff_names: projected_state_shares[n] += county_weight * county_share[n] if total_weight <= 0: return None projected_state_shares = _normalized(projected_state_shares, runoff_names) result = _score_projection( base_url, snapshot_name, snapshot, final, projected_state_shares, "V2R_SAME_CYCLE_RUNOFF_BASELINE", ( "V2R uses the completed Round 1 primary as the pre-runoff geographic " "baseline. County turnout weights come from Round 1; runoff candidate " "shares start from their normalized Round 1 county performance, are " "adjusted by a shrunk live statewide swing, and blend toward live county " "results as reporting increases." ), { "v2r_baseline_url": baseline_url, "v2r_round1_is_projection_input": True, "v2r_runoff_final_is_scoring_only": True, "v2r_limitation": ( "V2R assumes the runoff county turnout footprint broadly resembles " "Round 1 and does not yet model county-specific turnout changes or " "vote-method composition." ), }, ) result["runoff_baseline"] = { "available": True, "baseline_url": baseline_url, "round1_election_name": baseline.get("election_name"), "round1_total_votes": round1_total_votes, "runoff_candidates": runoff_names, "baseline_statewide_normalized_shares": baseline_state_shares, "baseline_counties": len(baseline_counties), "usable_candidate_baseline_counties": usable_baseline_counties, "reporting_counties_used_for_swing": reporting_counties_used, "counties_projected": counties_projected, "reporting_geographic_progress": reporting_geo_progress, "current_votes_as_share_of_round1": current_vs_round1, "swing_shrink": swing_shrink, "raw_swing_points": {n: 100.0 * raw_swing[n] for n in runoff_names}, "applied_swing_points": {n: 100.0 * live_swing[n] for n in runoff_names}, "turnout_weight_blend": turnout_blend, } return result def normalize_reference_url(url: Optional[str]) -> Optional[str]: if not url: return None url = str(url).strip() if not url: return None return url if url.endswith(".json") else slug_base(url) + "latest.json" def weighted_median(values: List[tuple], default: float) -> float: clean = [] for value, weight in values: try: value = float(value) weight = float(weight) except (TypeError, ValueError): continue if weight > 0: clean.append((value, weight)) if not clean: return default clean.sort(key=lambda x: x[0]) half = sum(w for _, w in clean) / 2.0 running = 0.0 for value, weight in clean: running += weight if running >= half: return value return clean[-1][0] def all_region_votes(region: Optional[Dict[str, Any]]) -> Dict[str, int]: if not isinstance(region, dict): return {} rows = region.get("candidates") if not isinstance(rows, list): return {} out = {} for c in rows: if not isinstance(c, dict): continue name = str(c.get("name") or "").strip() if name: out[name] = int(c.get("votes") or 0) return out def live_only_v3(base_url: str, snapshot_name: str) -> tuple: """ Single-round fallback when no historical reference is supplied. It deliberately cannot wander far from V0 early. """ base_url = slug_base(base_url) snapshot = fetch_json(base_url + snapshot_name) v0 = shares_from_votes(candidate_map(snapshot)) v1_result = project_v1(base_url, snapshot_name) v1 = {r["name"]: r["projected_share"] for r in v1_result.get("projection") or []} names = sorted(set(v0) | set(v1)) num = 0.0 den = 0.0 for region in county_regions(snapshot): votes = sum(all_region_votes(region).values()) if votes <= 0: continue try: p = min(1.0, max(0.0, float(region.get("percent_reporting") or 0) / 100.0)) except (TypeError, ValueError): p = 0.0 num += votes * p den += votes maturity = num / den if den else 0.0 disagreement = max( (abs(v1.get(n, 0.0) - v0.get(n, 0.0)) for n in names), default=0.0, ) # V1 gets at most 35% weight, and less when V0/V1 strongly disagree. penalty = max(0.20, 1.0 - min(1.0, disagreement / 0.10)) v1_weight = min(0.35, 0.35 * (maturity ** 0.75) * penalty) shares = { n: (1.0 - v1_weight) * v0.get(n, 0.0) + v1_weight * v1.get(n, 0.0) for n in names } return _normalized(shares, names), { "mode": "live_only", "reference_available": False, "reporting_maturity": maturity, "v1_blend_weight": v1_weight, "v0_v1_disagreement_points": 100.0 * disagreement, } def transfer_coefficients( baseline_counties: Dict[str, Dict[str, Any]], live_counties: Dict[str, Dict[str, Any]], baseline_weights: Dict[str, float], current_names: List[str], baseline_current_shares: Dict[str, Dict[str, float]], eliminated_names: List[str], global_swing: Dict[str, float], ) -> Dict[str, Dict[str, float]]: """ Learn whether counties that backed eliminated candidates in the reference are breaking differently in the live election. This is intentionally strongly shrunk/capped because early county samples are noisy. """ coeffs = {n: {} for n in current_names} if not eliminated_names: return coeffs # statewide eliminated-candidate shares in reference statewide = {e: 0.0 for e in eliminated_names} total_ref = 0.0 for cid, region in baseline_counties.items(): w = baseline_weights.get(cid, 0.0) votes = all_region_votes(region) total = sum(votes.values()) if w <= 0 or total <= 0: continue total_ref += w for e in eliminated_names: statewide[e] += w * votes.get(e, 0) / total if total_ref > 0: statewide = {e: statewide[e] / total_ref for e in eliminated_names} for name in current_names: for elim in eliminated_names: num = 0.0 den = 0.0 weight_sum = 0.0 for cid, base_share in baseline_current_shares.items(): live = live_counties.get(cid) base = baseline_counties.get(cid) if not live or not base: continue live_votes = _candidate_votes_in_region(live, current_names) live_total = sum(live_votes.values()) if live_total <= 0: continue try: p = min(1.0, max(0.0, float(live.get("percent_reporting") or 0) / 100.0)) except (TypeError, ValueError): p = 0.0 if p <= 0: continue base_all = all_region_votes(base) base_total = sum(base_all.values()) if base_total <= 0: continue x = base_all.get(elim, 0) / base_total - statewide.get(elim, 0.0) live_share = live_votes.get(name, 0) / live_total residual = live_share - base_share.get(name, 0.0) - global_swing.get(name, 0.0) w = baseline_weights.get(cid, 0.0) * math.sqrt(p) num += w * x * residual den += w * x * x weight_sum += w # ridge + hard cap ridge = 0.005 * max(weight_sum, 0.01) beta = num / (den + ridge) if den + ridge > 0 else 0.0 coeffs[name][elim] = max(-0.50, min(0.50, beta)) return coeffs def project_v3( base_url: str, snapshot_name: str, reference_url: Optional[str] = None, ) -> Dict[str, Any]: """ V3 Adaptive Remaining Vote. Works on single rounds and runoffs. Reference selection: 1) explicit reference URL if supplied; 2) runoff automatically uses Round 1 final; 3) otherwise live-only conservative mode. With a reference, V3 explicitly estimates county votes REMAINING, leaves already-counted votes fixed, and predicts only the outstanding vote. """ base_url = slug_base(base_url) snapshot = fetch_json(base_url + snapshot_name) final = fetch_json(base_url + "latest.json") snap_candidates = candidate_map(snapshot) current_names = list(snap_candidates.keys()) explicit = normalize_reference_url(reference_url) auto_round1 = round1_baseline_url(base_url) ref_url = explicit or auto_round1 if not ref_url: projected, info = live_only_v3(base_url, snapshot_name) result = _score_projection( base_url, snapshot_name, snapshot, final, projected, "V3_ADAPTIVE_REMAINING_VOTE", "V3 live-only mode conservatively blends V0 with V1 as county reporting matures.", { "v3_reference_available": False, "v3_mode": "live_only", "v3_limitation": "No historical/reference final supplied; turnout geography is not historically anchored.", }, ) result["v3"] = info result["v3"]["reference_url"] = None result["v3"]["estimated_remaining_votes"] = None return result baseline = load_reference_for_snapshot(ref_url, snapshot) if not isinstance(baseline, dict): projected, info = live_only_v3( base_url, snapshot_name ) result = _score_projection( base_url, snapshot_name, snapshot, final, projected, "V3_ADAPTIVE_REMAINING_VOTE", ( "V3 fell back to live-only mode because the requested " "historical reference was unavailable." ), { "v3_reference_available": False, "v3_mode": "unavailable_reference_fallback", "v3_reference_requested": ref_url, }, ) result["v3"] = info result["v3"]["reference_url"] = ref_url result["v3"]["estimated_remaining_votes"] = None return result baseline_counties = {_region_identity(r): r for r in county_regions(baseline)} live_counties = {_region_identity(r): r for r in county_regions(snapshot)} baseline_totals = {} baseline_total = 0 for cid, region in baseline_counties.items(): total = sum(all_region_votes(region).values()) if total > 0: baseline_totals[cid] = total baseline_total += total if baseline_total <= 0: projected, info = live_only_v3(base_url, snapshot_name) result = _score_projection( base_url, snapshot_name, snapshot, final, projected, "V3_ADAPTIVE_REMAINING_VOTE", "V3 fell back to live-only mode because the reference had no usable county totals.", {"v3_reference_available": False, "v3_mode": "bad_reference_fallback"}, ) result["v3"] = info result["v3"]["reference_url"] = ref_url result["v3"]["estimated_remaining_votes"] = None return result baseline_weights = {cid: total / baseline_total for cid, total in baseline_totals.items()} # Estimate turnout ratio versus reference from reporting counties. ratio_samples = [] geo_progress = 0.0 snapshot_county_votes = {n: 0 for n in current_names} for cid, base_total in baseline_totals.items(): live = live_counties.get(cid) if not live: continue live_all = all_region_votes(live) live_total = sum(live_all.values()) for n in current_names: snapshot_county_votes[n] += live_all.get(n, 0) try: p = min(1.0, max(0.0, float(live.get("percent_reporting") or 0) / 100.0)) except (TypeError, ValueError): p = 0.0 geo_progress += baseline_weights.get(cid, 0.0) * p if live_total > 0 and p > 0: # Don't let a 1% source reading imply a ridiculous 100x turnout. effective_p = max(0.05, p) implied = live_total / (base_total * effective_p) implied = max(0.10, min(3.0, implied)) ratio_samples.append((implied, baseline_weights.get(cid, 0.0) * max(0.02, p * p))) current_state_total = sum(c["votes"] for c in snap_candidates.values()) default_ratio = max(0.10, min(2.0, current_state_total / baseline_total if baseline_total else 1.0)) turnout_ratio = weighted_median(ratio_samples, default_ratio) baseline_state = candidate_map(baseline) matched_names = [ n for n in current_names if int((baseline_state.get(n) or {}).get("votes") or 0) > 0 ] candidate_baseline = len(matched_names) >= 2 baseline_current_shares = {} global_swing = {n: 0.0 for n in current_names} raw_global_swing = {n: 0.0 for n in current_names} swing_strength = 0.0 if candidate_baseline: for cid, region in baseline_counties.items(): votes = _candidate_votes_in_region(region, current_names) if sum(votes.values()) > 0: baseline_current_shares[cid] = _normalized(votes, current_names) nums = {n: 0.0 for n in current_names} den = 0.0 for cid, base_share in baseline_current_shares.items(): live = live_counties.get(cid) if not live: continue live_votes = _candidate_votes_in_region(live, current_names) live_total = sum(live_votes.values()) if live_total <= 0: continue try: p = min(1.0, max(0.0, float(live.get("percent_reporting") or 0) / 100.0)) except (TypeError, ValueError): p = 0.0 if p <= 0: continue live_share = _normalized(live_votes, current_names) w = baseline_weights.get(cid, 0.0) * math.sqrt(p) if w <= 0: continue den += w for n in current_names: nums[n] += w * max(-0.30, min(0.30, live_share[n] - base_share[n])) if den > 0: raw_global_swing = {n: nums[n] / den for n in current_names} swing_strength = min(1.0, math.sqrt(max(0.0, geo_progress) / 0.30)) global_swing = {n: raw_global_swing[n] * swing_strength for n in current_names} eliminated = [ n for n, c in baseline_state.items() if n not in current_names and int(c.get("votes") or 0) > 0 ] transfers = ( transfer_coefficients( baseline_counties, live_counties, baseline_weights, current_names, baseline_current_shares, eliminated, global_swing ) if candidate_baseline and eliminated else {n: {} for n in current_names} ) # Statewide reference shares for centered eliminated-candidate predictors. baseline_all_votes = {n: int(c.get("votes") or 0) for n, c in baseline_state.items()} baseline_all_total = sum(baseline_all_votes.values()) elim_state = { e: baseline_all_votes.get(e, 0) / baseline_all_total if baseline_all_total else 0.0 for e in eliminated } statewide_live = shares_from_votes(snap_candidates) projected_votes = {n: 0.0 for n in current_names} estimated_final_votes = 0.0 estimated_remaining_votes = 0.0 county_rows = [] for cid in set(baseline_totals) | set(live_counties): base_region = baseline_counties.get(cid) live = live_counties.get(cid) base_total = baseline_totals.get(cid, 0) live_votes = _candidate_votes_in_region(live, current_names) if live else {n: 0 for n in current_names} live_total = sum(live_votes.values()) try: p = min(1.0, max(0.0, float(live.get("percent_reporting") or 0) / 100.0)) if live else 0.0 except (TypeError, ValueError): p = 0.0 historical_expected = base_total * turnout_ratio if base_total > 0 else 0.0 if live_total > 0 and p > 0: local_expected = live_total / max(0.05, p) local_weight = min(0.90, p * p) expected_final = ( (1.0 - local_weight) * historical_expected + local_weight * local_expected if historical_expected > 0 else local_expected ) else: expected_final = historical_expected expected_final = max(float(live_total), float(expected_final)) remaining = max(0.0, expected_final - live_total) # Predict composition of REMAINING vote. if candidate_baseline: prior = baseline_current_shares.get(cid, statewide_live) pred = {n: prior.get(n, statewide_live.get(n, 0.0)) + global_swing.get(n, 0.0) for n in current_names} if base_region and eliminated: base_all = all_region_votes(base_region) base_all_total = sum(base_all.values()) if base_all_total > 0: for n in current_names: adj = 0.0 for e in eliminated: x = base_all.get(e, 0) / base_all_total - elim_state.get(e, 0.0) adj += transfers.get(n, {}).get(e, 0.0) * x pred[n] += max(-0.12, min(0.12, adj)) pred = _normalized(pred, current_names) if live_total > 0: live_share = _normalized(live_votes, current_names) local_w = min(0.90, math.sqrt(max(0.0, p))) pred = _normalized({ n: (1.0 - local_w) * pred[n] + local_w * live_share[n] for n in current_names }, current_names) else: # Single-round prior with different candidate names: # history supplies turnout geography, current election supplies candidate shares. if live_total > 0: live_share = _normalized(live_votes, current_names) local_w = min(0.80, math.sqrt(max(0.0, p))) pred = _normalized({ n: (1.0 - local_w) * statewide_live.get(n, 0.0) + local_w * live_share[n] for n in current_names }, current_names) else: pred = statewide_live for n in current_names: projected_votes[n] += live_votes.get(n, 0) + remaining * pred.get(n, 0.0) county_rows.append({ "county": (live or base_region or {}).get("name") or cid, "cid": cid, "fips": (live or base_region or {}).get("fips") or (live or base_region or {}).get("county_fips"), "percent_reporting": 100.0 * p, "current_votes": live_total, "expected_final_votes": expected_final, "estimated_remaining_votes": remaining, "projected_remaining_shares": {n: 100.0 * pred.get(n, 0.0) for n in current_names}, }) estimated_final_votes += expected_final estimated_remaining_votes += remaining # Add any statewide votes not represented in county rows. for n in current_names: residual = max(0, int((snap_candidates.get(n) or {}).get("votes") or 0) - snapshot_county_votes.get(n, 0)) projected_votes[n] += residual estimated_final_votes += residual projected = _normalized(projected_votes, current_names) mode = ( "runoff_transfer" if auto_round1 and ref_url == auto_round1 and candidate_baseline else ("historical_candidate_and_turnout" if candidate_baseline else "historical_turnout_only") ) result = _score_projection( base_url, snapshot_name, snapshot, final, projected, "V3_ADAPTIVE_REMAINING_VOTE", ( "V3 estimates remaining votes county-by-county from a prior turnout footprint, " "keeps counted votes fixed, and predicts only outstanding votes. When candidate " "names match the reference it also learns live geographic swing and eliminated-candidate transfers." ), { "v3_reference_available": True, "v3_reference_url": ref_url, "v3_reference_is_auto_round1": bool(auto_round1 and ref_url == auto_round1), "v3_mode": mode, "v3_candidate_baseline_available": candidate_baseline, "v3_limitation": ( "Vote-method composition is not modeled yet. A single-round reference with different " "candidate names improves turnout geography but cannot supply candidate-specific geography." ), }, ) result["v3"] = { "mode": mode, "reference_available": True, "reference_url": ref_url, "reference_election_name": baseline.get("election_name"), "reference_is_auto_round1": bool(auto_round1 and ref_url == auto_round1), "turnout_ratio_vs_reference": turnout_ratio, "geographic_progress": geo_progress, "estimated_final_votes": estimated_final_votes, "estimated_remaining_votes": estimated_remaining_votes, "candidate_baseline_available": candidate_baseline, "matched_current_candidates": matched_names, "eliminated_candidates": eliminated, "raw_global_swing_points": {n: 100.0 * raw_global_swing.get(n, 0.0) for n in current_names}, "applied_global_swing_points": {n: 100.0 * global_swing.get(n, 0.0) for n in current_names}, "swing_strength": swing_strength, "transfer_coefficients": transfers, "reference_library": baseline.get("reference_library") if isinstance(baseline, dict) else None, "counties": county_rows, } return result def parse_generated_media_market_js(text: str) -> Dict[str, Any]: """ Parse AFI generated JS files like: const SOME_MEDIA_MARKETS = { "MARKET NAME": { name: "MARKET NAME", fips: ["45001", "45003"] } }; This does NOT execute JavaScript. It only extracts quoted market keys and their fips arrays. """ markets: Dict[str, List[Dict[str, str]]] = {} block_pattern = re.compile( r'(?P["\'])(?P.+?)(?P=quote)\s*:\s*\{(?P.*?)\}\s*,?', re.DOTALL, ) fips_pattern = re.compile(r'\bfips\s*:\s*\[(?P.*?)\]', re.DOTALL) value_pattern = re.compile(r'["\'](?P\d{3,6})["\']') for match in block_pattern.finditer(text): market = str(match.group("market") or "").strip() body = match.group("body") or "" fips_match = fips_pattern.search(body) if not market or not fips_match: continue values = [ m.group("fips") for m in value_pattern.finditer(fips_match.group("body") or "") ] if values: markets[market] = [{"fips": value} for value in values] if not markets: raise ValueError( "No media-market FIPS blocks were found in the JS file. " "Expected generated objects containing fips: [\".....\"] arrays." ) return {"markets": markets} def fetch_text(url: str) -> str: req = urllib.request.Request( url, headers={"User-Agent": USER_AGENT, "Accept": "text/plain,*/*"}, ) with urllib.request.urlopen(req, timeout=20) as r: return r.read().decode("utf-8", errors="replace") def load_json_source(source: Optional[str]) -> Optional[Any]: """ Load JSON or an AFI generated media-market .js file from HTTP(S) or disk. """ if not source: return None source = str(source).strip() if not source: return None is_js = source.lower().split("?", 1)[0].endswith(".js") if source.startswith("http://") or source.startswith("https://"): if is_js: return parse_generated_media_market_js(fetch_text(source)) return fetch_json(source) path = Path(source) if not path.exists(): raise FileNotFoundError(f"Media/reference source does not exist: {source}") if path.suffix.lower() == ".js": return parse_generated_media_market_js( path.read_text(encoding="utf-8", errors="replace") ) with path.open("r", encoding="utf-8") as f: return json.load(f) def load_reference_source(source: Optional[str]) -> Optional[Dict[str, Any]]: """Load a reference election from HTTP(S) or a local normalized JSON file.""" if not source: return None source = str(source).strip() if not source: return None if source.startswith("http://") or source.startswith("https://"): return fetch_json(source) path = Path(source) if path.is_dir(): path = path / "latest.json" if not path.is_file(): raise FileNotFoundError(f"Reference election JSON does not exist: {source}") with path.open("r", encoding="utf-8") as f: data = json.load(f) if not isinstance(data, dict): raise ValueError(f"Reference election JSON is not an object: {source}") return data def _reference_year(reference: Dict[str, Any]) -> Optional[int]: values = [ reference.get("last_updated"), reference.get("election_name"), (reference.get("import_summary") or {}).get("election_date") if isinstance(reference.get("import_summary"), dict) else None, ] for value in values: match = re.search(r"\b(?:19|20)\d{2}\b", str(value or "")) if match: return int(match.group(0)) return None def _reference_party(reference: Dict[str, Any]) -> str: summary = reference.get("import_summary") or {} source = reference.get("source") or {} selection = source.get("selection") or {} value = summary.get("party") or selection.get("party") or source.get("party_guess") or "" value = str(value).strip().casefold() if value.startswith("dem"): return "democratic" if value.startswith("rep") or value.startswith("gop"): return "republican" return re.sub(r"[^a-z0-9]+", "_", value).strip("_") def _office_tokens(value: Any) -> set: stop = { "the", "and", "for", "of", "term", "position", "election", "primary", "general", "special", "year", "years", "state", "united", "states", "office", "district", } return { token for token in re.findall(r"[a-z0-9]+", str(value or "").casefold()) if len(token) >= 3 and token not in stop } def _reference_office_similarity( snapshot: Dict[str, Any], reference: Dict[str, Any], ) -> float: current = snapshot.get("election_name") or snapshot.get("office") or snapshot.get("election_scope") or "" summary = reference.get("import_summary") or {} historical = summary.get("office") or reference.get("election_name") or reference.get("election_scope") or "" a = _office_tokens(current) b = _office_tokens(historical) if not a or not b: return 0.70 families = [ {"senator", "senate"}, {"governor", "gubernatorial"}, {"president", "presidential"}, {"attorney"}, {"secretary"}, ] family_bonus = 0.0 for family in families: if (a & family) and (b & family): family_bonus = 0.25 break jaccard = len(a & b) / max(1, len(a | b)) return max(0.45, min(1.0, 0.55 + 0.45 * jaccard + family_bonus)) def _reference_total_votes(reference: Dict[str, Any]) -> int: return sum( sum(all_region_votes(region).values()) for region in county_regions(reference) ) def _compose_reference_library( manifest: Dict[str, Any], snapshot: Dict[str, Any], ) -> Dict[str, Any]: entries = manifest.get("references") or [] if not isinstance(entries, list) or not entries: raise ValueError("Historical reference library contains no references.") current_state = str(snapshot.get("province") or "").upper() current_party = str(manifest.get("party") or "").strip().casefold() now_year = datetime.utcnow().year loaded = [] for entry in entries: source = ( entry if isinstance(entry, str) else (entry.get("path") or entry.get("source")) if isinstance(entry, dict) else None ) if not source: continue try: ref = load_reference_source(str(source)) except Exception: continue if not isinstance(ref, dict): continue ref_state = str(ref.get("province") or "").upper() if current_state and ref_state and current_state != ref_state: continue year = _reference_year(ref) age = max(0, now_year - year) if year else 4 recency = max(0.25, math.exp(-0.13 * age)) office_similarity = _reference_office_similarity(snapshot, ref) ref_party = _reference_party(ref) party_match = ( 1.0 if not current_party or not ref_party or current_party == ref_party else 0.35 ) fit = reference_fit_scores( snapshot, ref, list(candidate_map(snapshot).keys()), ) turnout_fit = float(fit.get("turnout_fit") or 0.35) raw_weight = ( recency * office_similarity * party_match * (0.35 + 0.65 * turnout_fit) ) total_votes = _reference_total_votes(ref) if total_votes <= 0 or raw_weight <= 0: continue loaded.append({ "source": str(source), "reference": ref, "year": year, "recency": recency, "office_similarity": office_similarity, "party_match": party_match, "turnout_fit": turnout_fit, "raw_weight": raw_weight, "total_votes": total_votes, }) if not loaded: raise ValueError("No usable historical references remained after library filtering.") weight_sum = sum(item["raw_weight"] for item in loaded) for item in loaded: item["weight"] = item["raw_weight"] / weight_sum if weight_sum else 0.0 composite_total = max( 1.0, sum(item["weight"] * item["total_votes"] for item in loaded), ) county_shares: Dict[str, float] = {} county_meta: Dict[str, Dict[str, Any]] = {} for item in loaded: ref = item["reference"] total = float(item["total_votes"]) if total <= 0: continue for region in county_regions(ref): cid = _region_identity(region) votes = sum(all_region_votes(region).values()) if not cid or votes <= 0: continue county_shares[cid] = county_shares.get(cid, 0.0) + item["weight"] * (votes / total) county_meta.setdefault(cid, { "name": region.get("name") or cid, "fips": region.get("fips") or region.get("county_fips"), "type": region.get("type") or "County", }) footprint_total = sum(county_shares.values()) if footprint_total <= 0: raise ValueError("Historical library produced no usable county turnout footprint.") county_shares = {cid: share / footprint_total for cid, share in county_shares.items()} region_results = {} statewide_total = 0 for cid, share in county_shares.items(): votes = max(1, int(round(composite_total * share))) statewide_total += votes meta = county_meta.get(cid) or {} region_results[cid] = { "name": meta.get("name") or cid, "type": meta.get("type") or "County", "fips": meta.get("fips"), "percent_reporting": 100.0, "candidates": [{ "name": "Historical Total", "votes": votes, "percent": 100.0, "major_candidate": True, }], } diagnostics = [] for item in sorted(loaded, key=lambda x: x["weight"], reverse=True): ref = item["reference"] diagnostics.append({ "source": item["source"], "election_name": ref.get("election_name"), "year": item["year"], "party": _reference_party(ref) or None, "total_votes": item["total_votes"], "counties": len(county_regions(ref)), "recency_score": item["recency"], "office_similarity": item["office_similarity"], "live_turnout_fit": item["turnout_fit"], "weight": item["weight"], }) return { "schema_version": 1, "source_type": "historical_reference_library_composite", "election_name": f"{current_state} historical reference library ({len(loaded)} races)", "province": current_state, "election_type": "Historical Composite", "election_scope": "State", "last_updated": None, "percent_reporting": 100.0, "candidates": [{ "name": "Historical Total", "votes": statewide_total, "percent": 100.0, "major_candidate": True, }], "region_results": region_results, "reference_library": { "manifest_state": manifest.get("state"), "manifest_party": manifest.get("party"), "references_requested": len(entries), "references_used": len(loaded), "composite_total_votes": statewide_total, "weights": diagnostics, "weighting_method": ( "recency × office similarity × party match × live county-turnout fit, " "normalized across usable historical races" ), }, } def load_reference_for_snapshot( source: Optional[str], snapshot: Dict[str, Any], ) -> Optional[Dict[str, Any]]: """ Load one historical reference for a live/backtest snapshot. A stale remote reference should disable the historical layer for that projection, not abort the entire Needle run. Local-file errors still raise normally because they usually indicate a broken library manifest that the operator should repair. """ try: reference = load_reference_source(source) except urllib.error.HTTPError as exc: if int(getattr(exc, "code", 0) or 0) in (404, 410): return None raise except urllib.error.URLError: # Remote historical data is optional. If its host is temporarily # unreachable, continue without it rather than losing the whole race. if str(source or "").startswith(("http://", "https://")): return None raise if ( isinstance(reference, dict) and reference.get("source_type") == "historical_reference_library" ): return _compose_reference_library(reference, snapshot) return reference def discover_media_market_source( state: Optional[str], explicit_source: Optional[str] = None, ) -> Optional[str]: """ Locate a state media-market file in engine/media_markets automatically. Preferred naming: sc_media_markets.js sc_media_markets.json Also accepts a few equivalent STATE/media_markets naming variants. """ if explicit_source: return str(explicit_source).strip() or None state_code = str(state or "").strip().lower() if not state_code: return None directory = MEDIA_MARKET_DIR if not directory.is_dir(): return None preferred = [ f"{state_code}_media_markets.js", f"{state_code}_media_markets.json", f"{state_code}-media-markets.js", f"{state_code}-media-markets.json", f"media_markets_{state_code}.js", f"media_markets_{state_code}.json", f"media-markets-{state_code}.js", f"media-markets-{state_code}.json", ] # Exact preferred names first. for name in preferred: candidate = directory / name if candidate.is_file(): return str(candidate) # Case-insensitive fallback. preferred_lower = {name.lower() for name in preferred} for candidate in directory.iterdir(): if candidate.is_file() and candidate.name.lower() in preferred_lower: return str(candidate) return None def normalize_ma_municipality_name(value: Any) -> str: """ Normalize Massachusetts municipality names across VoteFlow, PD43+, and Census TIGER naming. TIGER uses a few legal basenames such as "Barnstable Town", "North Attleborough Town", and "Southbridge Town". VoteFlow/PD43+ normally use the shorter municipality name, so a trailing legal "Town"/"City" marker is removed before matching. """ s = unescape(str(value or "")).strip().casefold() s = s.replace("&", "and") s = re.sub(r"\s+", " ", s) s = re.sub(r"\s+(?:township|town|city)$", "", s) s = re.sub(r"[^a-z0-9]+", "_", s).strip("_") return s class _AFISimpleHTMLTableParser(HTMLParser): """Minimal table parser used only for the static Census TIGER lookup.""" def __init__(self) -> None: super().__init__() self.rows: List[List[str]] = [] self._row: Optional[List[str]] = None self._cell_parts: Optional[List[str]] = None def handle_starttag(self, tag: str, attrs: List[Any]) -> None: tag = tag.casefold() if tag == "tr": self._row = [] elif tag in {"td", "th"} and self._row is not None: self._cell_parts = [] def handle_data(self, data: str) -> None: if self._cell_parts is not None: self._cell_parts.append(data) def handle_endtag(self, tag: str) -> None: tag = tag.casefold() if tag in {"td", "th"} and self._row is not None and self._cell_parts is not None: value = unescape("".join(self._cell_parts)) value = re.sub(r"\s+", " ", value).strip() self._row.append(value) self._cell_parts = None elif tag == "tr" and self._row is not None: if self._row: self.rows.append(self._row) self._row = None self._cell_parts = None def _parse_ma_tiger_municipality_counties(html_text: str) -> Dict[str, str]: """ Parse the Census TIGER Massachusetts county-subdivision table into: normalized municipality name -> five-digit county FIPS. We only need COUNTY + BASENAME. "County subdivisions not defined" rows are intentionally ignored. """ parser = _AFISimpleHTMLTableParser() parser.feed(html_text) header: Optional[List[str]] = None header_index: Dict[str, int] = {} bridge: Dict[str, str] = {} for row in parser.rows: upper = [str(cell or "").strip().upper() for cell in row] if header is None and "COUNTY" in upper and "BASENAME" in upper: header = row header_index = { str(name or "").strip().upper(): idx for idx, name in enumerate(row) } continue if header is None: continue county_idx = header_index.get("COUNTY") basename_idx = header_index.get("BASENAME") state_idx = header_index.get("STATE") if county_idx is None or basename_idx is None: continue if max(county_idx, basename_idx, state_idx or 0) >= len(row): continue if state_idx is not None: state_value = re.sub(r"\D", "", str(row[state_idx] or "")) if state_value and state_value != "25": continue county = re.sub(r"\D", "", str(row[county_idx] or "")) basename = str(row[basename_idx] or "").strip() if len(county) != 3 or not basename: continue if "county subdivisions not defined" in basename.casefold(): continue municipality = normalize_ma_municipality_name(basename) if not municipality: continue bridge[municipality] = "25" + county # Massachusetts has 351 municipalities. Allow a little tolerance if Census # changes the presentation, but reject a clearly partial scrape. if len(bridge) < 340: raise ValueError( f"Massachusetts municipality/county bridge parsed only " f"{len(bridge)} municipalities; expected roughly 351." ) return bridge def _read_ma_municipality_county_cache(path: Path) -> Optional[Dict[str, str]]: try: if not path.is_file(): return None with path.open("r", encoding="utf-8") as source: payload = json.load(source) municipalities = ( payload.get("municipalities") if isinstance(payload, dict) and isinstance(payload.get("municipalities"), dict) else payload if isinstance(payload, dict) else None ) if not isinstance(municipalities, dict): return None cleaned = { normalize_ma_municipality_name(name): re.sub(r"\D", "", str(fips)) for name, fips in municipalities.items() if normalize_ma_municipality_name(name) and len(re.sub(r"\D", "", str(fips))) == 5 } return cleaned if len(cleaned) >= 340 else None except Exception: return None def _write_ma_municipality_county_cache( bridge: Dict[str, str], path: Path, ) -> bool: payload = { "schema_version": 1, "state": "MA", "source": MA_TIGER_COUSUB_URL, "generated_at_utc": datetime.now(timezone.utc).isoformat(), "municipalities": dict(sorted(bridge.items())), } try: save_json(path, payload) return True except Exception: return False def load_ma_municipality_county_bridge() -> Dict[str, str]: """ Load the MA City/Town -> county-FIPS bridge. Order: 1. in-process memory 2. persistent engine/media_markets cache 3. /tmp fallback cache 4. Census TIGER fetch, then persist to whichever cache is writable A failed bridge never breaks the forecast. It only means the MA media-market specialist is unavailable for that run. """ global _MA_MUNICIPAL_COUNTY_CACHE global _MA_MUNICIPAL_COUNTY_CACHE_SOURCE global _MA_MUNICIPAL_COUNTY_LOAD_ATTEMPTED if _MA_MUNICIPAL_COUNTY_CACHE is not None: return _MA_MUNICIPAL_COUNTY_CACHE if _MA_MUNICIPAL_COUNTY_LOAD_ATTEMPTED: return {} _MA_MUNICIPAL_COUNTY_LOAD_ATTEMPTED = True for cache_path in ( MA_MUNICIPAL_COUNTY_CACHE_PATH, MA_MUNICIPAL_COUNTY_FALLBACK_CACHE_PATH, ): cached = _read_ma_municipality_county_cache(cache_path) if cached: _MA_MUNICIPAL_COUNTY_CACHE = cached _MA_MUNICIPAL_COUNTY_CACHE_SOURCE = str(cache_path) return cached try: html_text = fetch_text(MA_TIGER_COUSUB_URL) bridge = _parse_ma_tiger_municipality_counties(html_text) except Exception: _MA_MUNICIPAL_COUNTY_CACHE = {} _MA_MUNICIPAL_COUNTY_CACHE_SOURCE = None return {} wrote_to: Optional[str] = None for cache_path in ( MA_MUNICIPAL_COUNTY_CACHE_PATH, MA_MUNICIPAL_COUNTY_FALLBACK_CACHE_PATH, ): if _write_ma_municipality_county_cache(bridge, cache_path): wrote_to = str(cache_path) break _MA_MUNICIPAL_COUNTY_CACHE = bridge _MA_MUNICIPAL_COUNTY_CACHE_SOURCE = wrote_to or MA_TIGER_COUSUB_URL return bridge def ma_county_fips_for_region(region: Dict[str, Any]) -> Optional[str]: region_type = str(region.get("type") or "").strip().casefold() if region_type not in { "town", "city", "municipality", "city/town", "city or town", }: return None municipality = normalize_ma_municipality_name( region.get("name") or region.get("_key") ) if not municipality: return None return load_ma_municipality_county_bridge().get(municipality) def ma_municipality_bridge_diagnostics() -> Dict[str, Any]: bridge = load_ma_municipality_county_bridge() return { "available": len(bridge) >= 340, "municipalities": len(bridge), "source": _MA_MUNICIPAL_COUNTY_CACHE_SOURCE, "preferred_cache": str(MA_MUNICIPAL_COUNTY_CACHE_PATH), "fallback_cache": str(MA_MUNICIPAL_COUNTY_FALLBACK_CACHE_PATH), } def normalize_county_name(value: Any) -> str: s = str(value or "").strip().casefold() s = s.replace("&", "and") s = re.sub(r"\b(county|parish|borough|census area|municipality)\b", "", s) s = re.sub(r"[^a-z0-9]+", "_", s).strip("_") return s def load_media_market_map(source: Optional[str], state: Optional[str] = None) -> Dict[str, Any]: """ Flexible media-market JSON / AFI generated-JS reader. Accepted examples: {"Greenville":"Greenville-Spartanburg", "Charleston":"Charleston"} {"counties":{"Greenville":"Greenville-Spartanburg"}} {"markets":{"Greenville-Spartanburg":["Greenville","Spartanburg"]}} [{"county":"Greenville","market":"Greenville-Spartanburg","fips":"45045"}] {"SC":{"Greenville":"Greenville-Spartanburg"}} AFI generated JS with market fips arrays is supported natively. FIPS keys are preferred when supplied; county-name matching is the fallback. """ if not source: return { "available": False, "source": None, "state": str(state or "").upper(), "by_fips": {}, "by_name": {}, "markets": {}, "entries": 0, } raw = load_json_source(source) state_upper = str(state or "").upper() by_fips: Dict[str, str] = {} by_name: Dict[str, str] = {} markets: Dict[str, List[str]] = {} def add(county=None, market=None, fips=None, row_state=None): if row_state and state_upper and str(row_state).upper() != state_upper: return market_name = str(market or "").strip() if not market_name: return if fips is not None: fs = re.sub(r"\D", "", str(fips)) if fs: by_fips[fs] = market_name if len(fs) >= 3: by_fips[fs[-3:]] = market_name county_name = normalize_county_name(county) if county_name: by_name[county_name] = market_name markets.setdefault(market_name, []) if county_name not in markets[market_name]: markets[market_name].append(county_name) def parse(obj, inherited_market=None, inherited_state=None): if isinstance(obj, list): for item in obj: parse(item, inherited_market, inherited_state) return if not isinstance(obj, dict): return # State wrapper: {"SC": {...}} if state_upper and state_upper in obj and isinstance(obj[state_upper], (dict, list)): parse(obj[state_upper], inherited_market, state_upper) return # Common explicit containers. if "counties" in obj: counties = obj["counties"] if isinstance(counties, dict): for county, market in counties.items(): if isinstance(market, str): add(county=county, market=market, row_state=inherited_state) elif isinstance(market, dict): add( county=county, market=market.get("market") or market.get("media_market") or market.get("dma"), fips=market.get("fips") or market.get("county_fips"), row_state=market.get("state") or inherited_state, ) elif inherited_market: for county in counties if isinstance(counties, list) else []: if isinstance(county, str): add(county=county, market=inherited_market, row_state=inherited_state) elif isinstance(county, dict): add( county=county.get("county") or county.get("name"), market=inherited_market, fips=county.get("fips") or county.get("county_fips"), row_state=county.get("state") or inherited_state, ) if "markets" in obj and isinstance(obj["markets"], dict): for market_name, counties in obj["markets"].items(): if isinstance(counties, list): for county in counties: if isinstance(county, str): add(county=county, market=market_name, row_state=inherited_state) elif isinstance(county, dict): add( county=county.get("county") or county.get("name"), market=market_name, fips=county.get("fips") or county.get("county_fips"), row_state=county.get("state") or inherited_state, ) elif isinstance(counties, dict): parse(counties, market_name, inherited_state) # Row-style object. market = ( obj.get("market") or obj.get("media_market") or obj.get("mediaMarket") or obj.get("dma") or obj.get("dma_name") ) county = obj.get("county") or obj.get("county_name") fips = obj.get("fips") or obj.get("county_fips") or obj.get("county_geoid") if market and (county or fips): add( county=county, market=market, fips=fips, row_state=obj.get("state") or inherited_state, ) # A market row such as {"name":"X","counties":[...]}. if inherited_market is None and obj.get("name") and isinstance(obj.get("counties"), list): parse(obj.get("counties"), str(obj.get("name")), obj.get("state") or inherited_state) # Flat county -> market mapping. reserved = { "counties","markets","market","media_market","mediaMarket","dma","dma_name", "county","county_name","fips","county_fips","county_geoid","state","name" } scalar_pairs = [ (k, v) for k, v in obj.items() if k not in reserved and isinstance(v, str) ] if scalar_pairs: for county_key, market_value in scalar_pairs: add(county=county_key, market=market_value, row_state=inherited_state) # Generic nested traversal. for k, v in obj.items(): if k in reserved or k in ("counties", "markets"): continue if isinstance(v, (dict, list)): parse(v, inherited_market, inherited_state) parse(raw) return { "available": bool(by_fips or by_name), "source": source, "state": state_upper, "by_fips": by_fips, "by_name": by_name, "markets": markets, "entries": len(set(list(by_name.keys()) + list(by_fips.keys()))), } def market_for_region(region: Dict[str, Any], mapping: Dict[str, Any]) -> Optional[str]: if not mapping.get("available"): return None fips = re.sub(r"\D", "", str(region.get("fips") or region.get("county_fips") or "")) if fips: if fips in mapping["by_fips"]: return mapping["by_fips"][fips] if len(fips) >= 3 and fips[-3:] in mapping["by_fips"]: return mapping["by_fips"][fips[-3:]] name = normalize_county_name(region.get("name") or region.get("_key")) direct = mapping["by_name"].get(name) if direct: return direct # Massachusetts model geography is municipal, while AFI's DMA map remains # county/FIPS based. Resolve City/Town -> county FIPS, then reuse the same # county media-market mapping as every other state. if str(mapping.get("state") or "").upper() == "MA": county_fips = ma_county_fips_for_region(region) if county_fips: if county_fips in mapping["by_fips"]: return mapping["by_fips"][county_fips] if county_fips[-3:] in mapping["by_fips"]: return mapping["by_fips"][county_fips[-3:]] return None def result_shares(result: Dict[str, Any]) -> Dict[str, float]: return { str(row.get("name")): float(row.get("projected_share") or 0.0) for row in result.get("projection") or [] if row.get("name") } def statewide_reporting_maturity(snapshot: Dict[str, Any]) -> float: num = 0.0 den = 0.0 for region in county_regions(snapshot): total = sum(all_region_votes(region).values()) if total <= 0: continue try: p = min(1.0, max(0.0, float(region.get("percent_reporting") or 0) / 100.0)) except (TypeError, ValueError): p = 0.0 num += total * p den += total return num / den if den > 0 else 0.0 def reference_fit_scores( snapshot: Dict[str, Any], reference: Optional[Dict[str, Any]], current_names: List[str], ) -> Dict[str, float]: """ Projection-time fit diagnostics only. No tested-race final is used. V4.6 separates: raw fit = how coherent the observed counties look; evidence = how much of the reference electorate has actually reported; usable fit = raw fit shrunk toward a conservative prior when evidence is thin. This prevents a handful of early counties from looking "99% certain". """ if not reference: return { "turnout_fit": 0.35, "turnout_fit_raw": 0.35, "turnout_fit_evidence": 0.0, "candidate_fit": 0.25, "candidate_fit_raw": 0.25, "candidate_fit_evidence": 0.0, "candidate_fit_available": False, } ref_counties = {_region_identity(r): r for r in county_regions(reference)} live_counties = {_region_identity(r): r for r in county_regions(snapshot)} ref_totals = { cid: sum(all_region_votes(region).values()) for cid, region in ref_counties.items() } ref_total_state = sum(v for v in ref_totals.values() if v > 0) # Turnout fit: dispersion of implied turnout ratios in counties with returns. ratios = [] reporting_footprint = 0.0 reporting_progress = 0.0 reporting_counties = 0 for cid, ref_region in ref_counties.items(): live = live_counties.get(cid) ref_total = ref_totals.get(cid, 0) if not live or ref_total <= 0: continue live_total = sum(all_region_votes(live).values()) try: p = min(1.0, max(0.0, float(live.get("percent_reporting") or 0) / 100.0)) except (TypeError, ValueError): p = 0.0 electorate_weight = ref_total / ref_total_state if ref_total_state > 0 else 0.0 if p > 0: reporting_footprint += electorate_weight reporting_progress += electorate_weight * p if live_total <= 0 or p < 0.05: continue reporting_counties += 1 ratio = live_total / (ref_total * max(0.05, p)) weight = math.sqrt(p) * math.sqrt(max(1.0, live_total)) * max(0.01, electorate_weight) ratios.append((ratio, weight)) if ratios: med = weighted_median(ratios, 1.0) mad_num = sum(w * abs(v - med) for v, w in ratios) mad_den = sum(w for _, w in ratios) rel_mad = mad_num / mad_den / max(0.10, abs(med)) if mad_den else 1.0 turnout_fit_raw = math.exp(-2.2 * rel_mad) else: turnout_fit_raw = 0.35 footprint_evidence = min(1.0, reporting_footprint / 0.70) progress_evidence = min(1.0, math.sqrt(max(0.0, reporting_progress) / 0.30)) county_evidence = min(1.0, math.sqrt(reporting_counties / 24.0)) turnout_evidence = ( 0.35 * footprint_evidence + 0.45 * progress_evidence + 0.20 * county_evidence ) turnout_evidence = max(0.0, min(1.0, turnout_evidence)) # Thin evidence shrinks toward the old conservative default rather than # accepting a near-perfect raw fit. turnout_fit = 0.35 + turnout_evidence * (turnout_fit_raw - 0.35) # Candidate fit if at least two current candidates existed in reference. ref_state = candidate_map(reference) matched = [ n for n in current_names if int((ref_state.get(n) or {}).get("votes") or 0) > 0 ] if len(matched) < 2: return { "turnout_fit": max(0.05, min(1.0, turnout_fit)), "turnout_fit_raw": max(0.05, min(1.0, turnout_fit_raw)), "turnout_fit_evidence": turnout_evidence, "candidate_fit": 0.30, "candidate_fit_raw": 0.30, "candidate_fit_evidence": 0.0, "candidate_fit_available": False, } county_pairs = [] pair_progress = 0.0 pair_footprint = 0.0 for cid, ref_region in ref_counties.items(): live = live_counties.get(cid) if not live: continue rv = _candidate_votes_in_region(ref_region, current_names) lv = _candidate_votes_in_region(live, current_names) if sum(rv.values()) <= 0 or sum(lv.values()) <= 0: continue try: p = min(1.0, max(0.0, float(live.get("percent_reporting") or 0) / 100.0)) except (TypeError, ValueError): p = 0.0 if p <= 0: continue rs = _normalized(rv, current_names) ls = _normalized(lv, current_names) electorate_weight = ( ref_totals.get(cid, 0) / ref_total_state if ref_total_state > 0 else 0.0 ) pair_footprint += electorate_weight pair_progress += electorate_weight * p county_pairs.append( (rs, ls, math.sqrt(p) * math.sqrt(max(1.0, sum(lv.values())))) ) if len(county_pairs) < 2: candidate_fit_raw = 0.40 else: swing = {n: 0.0 for n in current_names} den = sum(w for _, _, w in county_pairs) for rs, ls, w in county_pairs: for n in current_names: swing[n] += w * (ls[n] - rs[n]) if den: swing = {n: swing[n] / den for n in current_names} err_num = 0.0 err_den = 0.0 for rs, ls, w in county_pairs: predicted = _normalized( {n: rs[n] + swing[n] for n in current_names}, current_names, ) mae = sum(abs(predicted[n] - ls[n]) for n in current_names) / len(current_names) err_num += w * mae err_den += w mae = err_num / err_den if err_den else 0.15 candidate_fit_raw = math.exp(-8.0 * mae) candidate_evidence = ( 0.40 * min(1.0, pair_footprint / 0.70) + 0.40 * min(1.0, math.sqrt(max(0.0, pair_progress) / 0.30)) + 0.20 * min(1.0, math.sqrt(len(county_pairs) / 20.0)) ) candidate_evidence = max(0.0, min(1.0, candidate_evidence)) candidate_fit = 0.30 + candidate_evidence * (candidate_fit_raw - 0.30) return { "turnout_fit": max(0.05, min(1.0, turnout_fit)), "turnout_fit_raw": max(0.05, min(1.0, turnout_fit_raw)), "turnout_fit_evidence": turnout_evidence, "candidate_fit": max(0.05, min(1.0, candidate_fit)), "candidate_fit_raw": max(0.05, min(1.0, candidate_fit_raw)), "candidate_fit_evidence": candidate_evidence, "candidate_fit_available": True, } def _historical_reference_entries( reference_url: Optional[str], snapshot: Dict[str, Any], ) -> List[Dict[str, Any]]: """ Return raw historical elections plus the library weight that applies at the current snapshot. Candidate rows are retained here; the turnout composite intentionally strips them elsewhere. """ source = normalize_reference_url(reference_url) if not source: return [] try: root = load_reference_source(source) except urllib.error.HTTPError as exc: if int(getattr(exc, "code", 0) or 0) in (404, 410): return [] raise except urllib.error.URLError: if source.startswith(("http://", "https://")): return [] raise if not isinstance(root, dict): return [] if root.get("source_type") == "historical_reference_library": composite = _compose_reference_library(root, snapshot) weight_by_source = { str(row.get("source")): float(row.get("weight") or 0.0) for row in (composite.get("reference_library") or {}).get("weights") or [] } entries = [] for item in root.get("references") or []: path = ( item if isinstance(item, str) else item.get("path") or item.get("source") if isinstance(item, dict) else None ) if not path: continue try: ref = load_reference_source(str(path)) except Exception: continue if not isinstance(ref, dict): continue entries.append({ "source": str(path), "reference": ref, "library_weight": weight_by_source.get(str(path), 0.0), }) return entries # A single normalized historical election can still supply patterns. if county_regions(root) and len(candidate_map(root)) >= 2: return [{ "source": source, "reference": root, "library_weight": 1.0, }] return [] def _historical_candidate_patterns(entries: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """ Convert every meaningful historical candidate into a centered county over/under-performance vector. This learns geography without assigning ideological labels to candidates. """ patterns = [] for entry in entries: ref = entry["reference"] state = candidate_map(ref) total_state = sum(int(c.get("votes") or 0) for c in state.values()) if total_state <= 0: continue counties = county_regions(ref) county_totals = { _region_identity(region): sum(all_region_votes(region).values()) for region in counties } for candidate, state_row in state.items(): state_votes = int(state_row.get("votes") or 0) state_share = state_votes / total_state if total_state else 0.0 if state_share < 0.015 or state_share > 0.985: continue residuals = {} weighted_sq = 0.0 weighted_den = 0.0 for region in counties: cid = _region_identity(region) votes = all_region_votes(region) total = county_totals.get(cid, 0) if total <= 0: continue county_share = votes.get(candidate, 0) / total residual = county_share - state_share residuals[cid] = residual w = total / total_state weighted_sq += w * residual * residual weighted_den += w if len(residuals) < 8: continue std = math.sqrt(weighted_sq / weighted_den) if weighted_den else 0.0 # Near-flat candidates contain little geographic information. if std < 0.008: continue patterns.append({ "source": entry["source"], "election_name": ref.get("election_name"), "candidate": candidate, "state_share": state_share, "residuals": residuals, "std": std, "reference_weight": max(0.02, float(entry.get("library_weight") or 0.0)), }) return patterns def _histgeo_training_rows( snapshot: Dict[str, Any], names: List[str], ) -> List[Dict[str, Any]]: rows = [] for region in county_regions(snapshot): cid = _region_identity(region) votes = _candidate_votes_in_region(region, names) total = sum(votes.values()) if total < 50: continue try: p = min(1.0, max(0.0, float(region.get("percent_reporting") or 0.0) / 100.0)) except (TypeError, ValueError): p = 0.0 if p < 0.05: continue shares = _normalized(votes, names) weight = math.sqrt(max(1.0, total)) * math.sqrt(max(0.02, p)) rows.append({ "cid": cid, "name": region.get("name") or cid, "shares": shares, "weight": weight, "votes": total, "p": p, }) return rows def _fit_one_histgeo_candidate( candidate: str, train_rows: List[Dict[str, Any]], patterns: List[Dict[str, Any]], max_patterns: int = 5, ) -> Dict[str, Any]: if len(train_rows) < 3: return { "candidate": candidate, "available": False, "intercept": 0.0, "patterns": [], "signal_strength": 0.0, } den = sum(row["weight"] for row in train_rows) intercept = ( sum(row["weight"] * row["shares"].get(candidate, 0.0) for row in train_rows) / den if den else 0.0 ) scored = [] for pattern in patterns: common = [ row for row in train_rows if row["cid"] in pattern["residuals"] ] if len(common) < 3: continue wsum = sum(row["weight"] for row in common) if wsum <= 0: continue xmean = sum( row["weight"] * pattern["residuals"][row["cid"]] for row in common ) / wsum ymean = sum( row["weight"] * (row["shares"].get(candidate, 0.0) - intercept) for row in common ) / wsum cov = 0.0 varx = 0.0 vary = 0.0 for row in common: w = row["weight"] x = pattern["residuals"][row["cid"]] - xmean y = (row["shares"].get(candidate, 0.0) - intercept) - ymean cov += w * x * y varx += w * x * x vary += w * y * y if varx <= 1e-9 or vary <= 1e-9: continue corr = cov / math.sqrt(varx * vary) corr = max(-1.0, min(1.0, corr)) # Ridge-like shrinkage on the slope, especially with small samples. ridge = 0.20 * varx + wsum * 0.00015 beta = cov / (varx + ridge) beta = max(-3.0, min(3.0, beta)) sample_factor = min(1.0, math.sqrt(len(common) / 12.0)) quality = ( abs(corr) * sample_factor * math.sqrt(max(0.02, pattern["reference_weight"])) * min(1.0, pattern["std"] / 0.035) ) if quality < 0.03: continue scored.append({ "source": pattern["source"], "election_name": pattern["election_name"], "historical_candidate": pattern["candidate"], "corr": corr, "beta": beta, "quality": quality, "xmean": xmean, "residuals": pattern["residuals"], "common_counties": len(common), }) scored.sort(key=lambda row: row["quality"], reverse=True) selected = scored[:max_patterns] signal_strength = ( sum(row["quality"] for row in selected) / max(1, len(selected)) if selected else 0.0 ) return { "candidate": candidate, "available": bool(selected), "intercept": intercept, "patterns": selected, "signal_strength": min(1.0, signal_strength), } def _histgeo_predict_candidate_share( fit: Dict[str, Any], cid: str, ) -> float: intercept = float(fit.get("intercept") or 0.0) selected = fit.get("patterns") or [] num = 0.0 den = 0.0 for row in selected: if cid not in row["residuals"]: continue w = float(row["quality"] or 0.0) if w <= 0: continue contribution = float(row["beta"]) * ( float(row["residuals"][cid]) - float(row["xmean"]) ) contribution = max(-0.25, min(0.25, contribution)) num += w * contribution den += w adjustment = num / den if den > 0 else 0.0 return max(0.001, min(0.999, intercept + adjustment)) def _histgeo_cross_validate( training_rows: List[Dict[str, Any]], patterns: List[Dict[str, Any]], names: List[str], ) -> Dict[str, Any]: """ Deterministic two-fold county holdout. A county is never used to fit the pattern that predicts its own live share. """ if len(training_rows) < 8: return { "available": False, "mae": None, "score": 0.20, "counties": len(training_rows), } err_num = 0.0 err_den = 0.0 predicted_counties = 0 def fold_id(cid: str) -> int: return sum(ord(ch) for ch in cid) % 2 for fold in (0, 1): train = [row for row in training_rows if fold_id(row["cid"]) != fold] valid = [row for row in training_rows if fold_id(row["cid"]) == fold] if len(train) < 4 or not valid: continue fits = { name: _fit_one_histgeo_candidate(name, train, patterns) for name in names } for row in valid: raw = { name: _histgeo_predict_candidate_share(fits[name], row["cid"]) for name in names } pred = _normalized(raw, names) mae = sum( abs(pred.get(name, 0.0) - row["shares"].get(name, 0.0)) for name in names ) / max(1, len(names)) w = row["weight"] err_num += w * mae err_den += w predicted_counties += 1 if err_den <= 0 or predicted_counties < 4: return { "available": False, "mae": None, "score": 0.20, "counties": predicted_counties, } mae = err_num / err_den # 2-point candidate-share MAE is strong; 8+ is weak. score = math.exp(-mae / 0.055) return { "available": True, "mae": mae, "score": max(0.05, min(1.0, score)), "counties": predicted_counties, } def project_historical_candidate_geo( base_url: str, snapshot_name: str, reference_url: Optional[str], v3_result: Optional[Dict[str, Any]] = None, ) -> Optional[Dict[str, Any]]: """ V4.6 Historical Coalition Geography. Historical candidate identities are NOT mapped to current candidates. Instead, every old candidate supplies a centered county over/under- performance pattern. Current candidates are fit to combinations of those patterns using only live counties available at this snapshot. A deterministic held-out-county score controls how much this model can be trusted by the ensemble. """ ref_url = normalize_reference_url(reference_url) if not ref_url: return None base_url = slug_base(base_url) snapshot = fetch_json(base_url + snapshot_name) final = fetch_json(base_url + "latest.json") current = candidate_map(snapshot) names = list(current.keys()) if len(names) < 2: return None entries = _historical_reference_entries(ref_url, snapshot) patterns = _historical_candidate_patterns(entries) if len(patterns) < 2: return None training_rows = _histgeo_training_rows(snapshot, names) if len(training_rows) < 3: return None fits = { name: _fit_one_histgeo_candidate(name, training_rows, patterns) for name in names } validation = _histgeo_cross_validate(training_rows, patterns, names) maturity = statewide_reporting_maturity(snapshot) county_evidence = min(1.0, math.sqrt(len(training_rows) / 28.0)) maturity_evidence = min(1.0, math.sqrt(max(0.0, maturity) / 0.30)) reporting_evidence = math.sqrt(max(0.0, county_evidence * maturity_evidence)) mean_signal = sum( float(fits[name].get("signal_strength") or 0.0) for name in names ) / max(1, len(names)) validation_score = float(validation.get("score") or 0.0) confidence = ( reporting_evidence * validation_score * (0.45 + 0.55 * min(1.0, mean_signal)) ) confidence = max(0.0, min(1.0, confidence)) composite = load_reference_for_snapshot(ref_url, snapshot) if not isinstance(composite, dict): return None ref_counties = {_region_identity(r): r for r in county_regions(composite)} live_counties = {_region_identity(r): r for r in county_regions(snapshot)} if v3_result is None: v3_result = project_v3(base_url, snapshot_name, ref_url) v3d = (v3_result or {}).get("v3") or {} turnout_ratio = float(v3d.get("turnout_ratio_vs_reference") or 1.0) statewide_live = shares_from_votes(current) projected_votes = {name: 0.0 for name in names} estimated_remaining = 0.0 county_audit = [] for cid in set(ref_counties) | set(live_counties): ref_region = ref_counties.get(cid) live_region = live_counties.get(cid) ref_total = sum(all_region_votes(ref_region).values()) if ref_region else 0 live_votes = ( _candidate_votes_in_region(live_region, names) if live_region else {name: 0 for name in names} ) live_total = sum(live_votes.values()) try: p = ( min(1.0, max(0.0, float(live_region.get("percent_reporting") or 0.0) / 100.0)) if live_region else 0.0 ) except (TypeError, ValueError): p = 0.0 historical_expected = ref_total * turnout_ratio if ref_total > 0 else 0.0 if live_total > 0 and p > 0: local_expected = live_total / max(0.05, p) local_turnout_weight = min(0.90, p * p) expected_final = ( (1.0 - local_turnout_weight) * historical_expected + local_turnout_weight * local_expected if historical_expected > 0 else local_expected ) else: expected_final = historical_expected expected_final = max(float(live_total), float(expected_final)) remaining = max(0.0, expected_final - live_total) estimated_remaining += remaining pattern_raw = { name: _histgeo_predict_candidate_share(fits[name], cid) for name in names } pattern_pred = _normalized(pattern_raw, names) # If cross-validation/evidence is weak, retreat toward the current # statewide shares rather than forcing a historical analogy. pattern_pred = _normalized({ name: confidence * pattern_pred.get(name, 0.0) + (1.0 - confidence) * statewide_live.get(name, 0.0) for name in names }, names) if live_total > 0: local_live = _normalized(live_votes, names) local_weight = min(0.92, math.sqrt(max(0.0, p))) pattern_pred = _normalized({ name: (1.0 - local_weight) * pattern_pred[name] + local_weight * local_live.get(name, 0.0) for name in names }, names) for name in names: projected_votes[name] += live_votes.get(name, 0) + remaining * pattern_pred.get(name, 0.0) ranked = sorted(names, key=lambda name: pattern_pred.get(name, 0.0), reverse=True) county_audit.append({ "county": ( (live_region or {}).get("name") or (ref_region or {}).get("name") or cid ), "cid": cid, "reporting": 100.0 * p, "current_votes": live_total, "estimated_remaining_votes": remaining, "projected_remaining_leader": ranked[0] if ranked else None, "projected_remaining_shares": { name: 100.0 * pattern_pred.get(name, 0.0) for name in names }, }) # Preserve statewide votes not present in county rows. county_current = {name: 0 for name in names} for region in county_regions(snapshot): votes = _candidate_votes_in_region(region, names) for name in names: county_current[name] += votes.get(name, 0) for name in names: residual = max( 0, int((current.get(name) or {}).get("votes") or 0) - county_current.get(name, 0), ) projected_votes[name] += residual projected = _normalized(projected_votes, names) result = _score_projection( base_url, snapshot_name, snapshot, final, projected, "HISTORICAL_CANDIDATE_GEOGRAPHY", ( "Historical candidate geography fits current candidates to centered " "county over/under-performance patterns from prior primaries. Candidate " "identities and ideological labels are not transferred." ), { "historical_candidate_geo": True, "uses_final_results_for_projection": False, }, ) fit_summary = {} for name, fit in fits.items(): fit_summary[name] = { "signal_strength": fit.get("signal_strength"), "top_patterns": [ { "election_name": row.get("election_name"), "historical_candidate": row.get("historical_candidate"), "correlation": row.get("corr"), "beta": row.get("beta"), "quality": row.get("quality"), "common_counties": row.get("common_counties"), } for row in (fit.get("patterns") or [])[:5] ], } county_audit.sort( key=lambda row: float(row.get("estimated_remaining_votes") or 0.0), reverse=True, ) result["historical_candidate_geo"] = { "available": True, "reference_url": ref_url, "historical_elections": len(entries), "historical_candidate_patterns": len(patterns), "training_counties": len(training_rows), "reporting_evidence": reporting_evidence, "mean_pattern_signal": mean_signal, "cross_validation_available": bool(validation.get("available")), "cross_validation_counties": validation.get("counties"), "cross_validation_mae_points": ( 100.0 * float(validation["mae"]) if validation.get("mae") is not None else None ), "cross_validation_score": validation_score, "confidence": confidence, "max_ensemble_influence": min( 0.38, 0.04 + 0.42 * reporting_evidence * validation_score, ), "estimated_remaining_votes": estimated_remaining, "candidate_fits": fit_summary, "counties": county_audit, "method_note": ( "Patterns are centered within historical races, so the model learns " "geographic coalition structure rather than historical statewide winners." ), } return result def project_market_model( base_url: str, snapshot_name: str, reference_url: Optional[str], market_map_source: Optional[str], ) -> Optional[Dict[str, Any]]: """ Media-market hierarchical model. Geographic units remain atomic. Most states use counties; Massachusetts uses City/Town rows. Media markets provide a shared swing layer above those units, with MA municipalities bridged to the county-FIPS DMA map. """ base_url = slug_base(base_url) snapshot = fetch_json(base_url + snapshot_name) final = fetch_json(base_url + "latest.json") state = snapshot.get("province") or final.get("province") market_map_source = discover_media_market_source(state, market_map_source) if not market_map_source: return None mapping = load_media_market_map(market_map_source, state) if not mapping.get("available"): return None snap_candidates = candidate_map(snapshot) names = list(snap_candidates.keys()) statewide_live = shares_from_votes(snap_candidates) explicit_ref = normalize_reference_url(reference_url) auto_ref = round1_baseline_url(base_url) ref_url = explicit_ref or auto_ref reference = load_reference_for_snapshot(ref_url, snapshot) if ref_url else None live_counties = {_region_identity(r): r for r in county_regions(snapshot)} ref_counties = ( {_region_identity(r): r for r in county_regions(reference)} if reference else {} ) ref_state = candidate_map(reference) if reference else {} matched_names = [ n for n in names if int((ref_state.get(n) or {}).get("votes") or 0) > 0 ] if reference else [] candidate_baseline = len(matched_names) >= 2 # Expected final county turnout. ref_totals = {} ref_total_state = 0 if reference: for cid, region in ref_counties.items(): total = sum(all_region_votes(region).values()) if total > 0: ref_totals[cid] = total ref_total_state += total ratio_samples = [] if ref_total_state > 0: for cid, ref_total in ref_totals.items(): live = live_counties.get(cid) if not live: continue live_total = sum(_candidate_votes_in_region(live, names).values()) if live_total <= 0: continue try: p = min(1.0, max(0.0, float(live.get("percent_reporting") or 0) / 100.0)) except (TypeError, ValueError): p = 0.0 if p <= 0: continue implied = live_total / (ref_total * max(0.05, p)) implied = max(0.10, min(3.00, implied)) ratio_samples.append((implied, max(1.0, live_total) * p * p)) current_state_votes = sum(c["votes"] for c in snap_candidates.values()) default_ratio = current_state_votes / ref_total_state if ref_total_state > 0 else 1.0 turnout_ratio = weighted_median(ratio_samples, max(0.10, min(2.0, default_ratio))) county_expected_final = {} county_live_votes = {} county_reporting = {} for cid, live in live_counties.items(): lv = _candidate_votes_in_region(live, names) live_total = sum(lv.values()) county_live_votes[cid] = lv try: p = min(1.0, max(0.0, float(live.get("percent_reporting") or 0) / 100.0)) except (TypeError, ValueError): p = 0.0 county_reporting[cid] = p hist_expected = ref_totals.get(cid, 0) * turnout_ratio if ref_total_state > 0 else 0.0 if live_total > 0 and p > 0: local_expected = live_total / max(0.05, p) local_weight = min(0.85, p * p) expected = ( (1.0 - local_weight) * hist_expected + local_weight * local_expected if hist_expected > 0 else local_expected ) else: expected = hist_expected if expected <= 0 and live_total > 0: expected = live_total / max(0.05, p) if p > 0 else live_total county_expected_final[cid] = max(float(live_total), float(expected)) # Include reference-only geographic units so their markets still have expected weight. for cid, ref_total in ref_totals.items(): county_expected_final.setdefault(cid, ref_total * turnout_ratio) county_live_votes.setdefault(cid, {n: 0 for n in names}) county_reporting.setdefault(cid, 0.0) # Candidate county prior if candidate names match reference. county_prior = {} global_swing = {n: 0.0 for n in names} if candidate_baseline: swing_num = {n: 0.0 for n in names} swing_den = 0.0 for cid, ref_region in ref_counties.items(): rv = _candidate_votes_in_region(ref_region, names) if sum(rv.values()) <= 0: continue prior = _normalized(rv, names) county_prior[cid] = prior lv = county_live_votes.get(cid, {n: 0 for n in names}) lvt = sum(lv.values()) p = county_reporting.get(cid, 0.0) if lvt <= 0 or p <= 0: continue ls = _normalized(lv, names) w = max(1.0, lvt) * math.sqrt(p) swing_den += w for n in names: swing_num[n] += w * max(-0.30, min(0.30, ls[n] - prior[n])) if swing_den: global_swing = {n: swing_num[n] / swing_den for n in names} # Build market groups from all geographic-unit IDs we know. market_counties: Dict[str, List[str]] = {} unmapped_current_votes = 0 mapped_current_votes = 0 all_ids = set(county_expected_final) | set(live_counties) for cid in all_ids: region = live_counties.get(cid) or ref_counties.get(cid) if not region: continue market = market_for_region(region, mapping) live_total = sum(county_live_votes.get(cid, {}).values()) if market: market_counties.setdefault(market, []).append(cid) mapped_current_votes += live_total else: unmapped_current_votes += live_total if not market_counties: return None # Learn market-specific residuals from reporting geographic units. market_rows = [] market_residuals: Dict[str, Dict[str, float]] = {} market_progress: Dict[str, float] = {} for market, cids in market_counties.items(): expected_total = sum(county_expected_final.get(cid, 0.0) for cid in cids) current_total = sum(sum(county_live_votes.get(cid, {}).values()) for cid in cids) progress = min(1.0, current_total / expected_total) if expected_total > 0 else 0.0 market_progress[market] = progress live_votes = {n: 0 for n in names} prior_votes = {n: 0.0 for n in names} prior_weight = 0.0 for cid in cids: lv = county_live_votes.get(cid, {}) for n in names: live_votes[n] += int(lv.get(n, 0)) expected = county_expected_final.get(cid, 0.0) if candidate_baseline and cid in county_prior and expected > 0: prior_weight += expected for n in names: prior_votes[n] += expected * county_prior[cid][n] live_total = sum(live_votes.values()) live_share = _normalized(live_votes, names) if live_total > 0 else statewide_live if candidate_baseline and prior_weight > 0: prior_share = _normalized(prior_votes, names) residual = { n: live_share[n] - prior_share[n] - global_swing.get(n, 0.0) for n in names } else: prior_share = statewide_live residual = {n: live_share[n] - statewide_live.get(n, 0.0) for n in names} residual_strength = min(1.0, math.sqrt(max(0.0, progress) / 0.25)) market_residuals[market] = { n: max(-0.15, min(0.15, residual[n] * residual_strength)) for n in names } ranked_live = sorted(names, key=lambda n: live_share.get(n, 0.0), reverse=True) live_leader = ranked_live[0] if ranked_live else None live_runner = ranked_live[1] if len(ranked_live) > 1 else None live_margin = ( 100.0 * (live_share[live_leader] - live_share[live_runner]) if live_leader and live_runner else None ) market_rows.append({ "market": market, "counties": len(cids), "current_votes": current_total, "expected_final_votes": expected_total, "estimated_remaining_votes": max(0.0, expected_total - current_total), "progress": progress, "live_leader": live_leader, "live_margin_points": live_margin, "residual_points": {n: 100.0 * market_residuals[market][n] for n in names}, }) # Project each geographic unit's remaining vote with the market layer. projected_votes = {n: 0.0 for n in names} county_rows = [] projected_county_votes: Dict[str, Dict[str, float]] = {} for cid in all_ids: live = live_counties.get(cid) ref_region = ref_counties.get(cid) region = live or ref_region if not region: continue lv = county_live_votes.get(cid, {n: 0 for n in names}) live_total = sum(lv.values()) expected = max(float(live_total), float(county_expected_final.get(cid, live_total))) remaining = max(0.0, expected - live_total) p = county_reporting.get(cid, 0.0) market = market_for_region(region, mapping) if candidate_baseline: prior = county_prior.get(cid, statewide_live) pred = { n: prior.get(n, statewide_live.get(n, 0.0)) + global_swing.get(n, 0.0) + (market_residuals.get(market, {}) or {}).get(n, 0.0) for n in names } pred = _normalized(pred, names) else: pred = statewide_live if market: # Market live average moves unreported counties together. market_cids = market_counties.get(market, []) mv = {n: 0 for n in names} for mcid in market_cids: for n in names: mv[n] += county_live_votes.get(mcid, {}).get(n, 0) if sum(mv.values()) > 0: market_share = _normalized(mv, names) mw = min(0.75, math.sqrt(market_progress.get(market, 0.0))) pred = _normalized({ n: (1.0 - mw) * statewide_live.get(n, 0.0) + mw * market_share[n] for n in names }, names) # County's own live return increasingly replaces the market/prior signal. if live_total > 0: live_share = _normalized(lv, names) local_w = min(0.90, math.sqrt(max(0.0, p))) pred = _normalized({ n: (1.0 - local_w) * pred[n] + local_w * live_share[n] for n in names }, names) county_projected = { n: float(lv.get(n, 0)) + remaining * pred.get(n, 0.0) for n in names } projected_county_votes[cid] = county_projected for n in names: projected_votes[n] += county_projected[n] live_share = _normalized(lv, names) if live_total > 0 else {n: 0.0 for n in names} pred_rank = sorted(names, key=lambda n: pred.get(n, 0.0), reverse=True) live_rank = sorted(names, key=lambda n: live_share.get(n, 0.0), reverse=True) county_rows.append({ "county": region.get("name") or region.get("_key") or cid, "region_type": region.get("type") or "County", "fips": region.get("fips") or region.get("county_fips"), "market_bridge_county_fips": ( ma_county_fips_for_region(region) if str(state or "").upper() == "MA" else None ), "market": market, "percent_reporting": 100.0 * p, "current_votes": live_total, "expected_final_votes": expected, "estimated_remaining_votes": remaining, "live_leader": live_rank[0] if live_total > 0 and live_rank else None, "projected_remaining_leader": pred_rank[0] if pred_rank else None, "projected_remaining_shares": {n: 100.0 * pred.get(n, 0.0) for n in names}, }) # Add statewide votes that did not reconcile into counties. county_current_by_name = {n: 0 for n in names} for cid in live_counties: lv = county_live_votes.get(cid, {}) for n in names: county_current_by_name[n] += int(lv.get(n, 0)) for n in names: residual = max(0, int((snap_candidates.get(n) or {}).get("votes") or 0) - county_current_by_name[n]) projected_votes[n] += residual projected_shares = _normalized(projected_votes, names) result = _score_projection( base_url, snapshot_name, snapshot, final, projected_shares, "V4_MEDIA_MARKET_COMPONENT", ( "Media-market component keeps local geographic units atomic and shares " "a market-level swing between units assigned to the same media market. " "Massachusetts City/Town rows are assigned through a municipality-to-" "county-FIPS bridge." ), { "media_market_source": market_map_source, "media_market_reference_url": ref_url, "media_market_candidate_baseline": candidate_baseline, "uses_final_results_for_projection": False, }, ) projected_rank = sorted(names, key=lambda n: projected_shares.get(n, 0.0), reverse=True) overall_leader = projected_rank[0] if projected_rank else None overall_runner = projected_rank[1] if len(projected_rank) > 1 else None # Annotate county contributions to the projected top-two margin. if overall_leader and overall_runner: for row in county_rows: cid_match = None target_name = normalize_county_name(row["county"]) for cid in projected_county_votes: region = live_counties.get(cid) or ref_counties.get(cid) if region and normalize_county_name(region.get("name") or region.get("_key")) == target_name: cid_match = cid break pv = projected_county_votes.get(cid_match, {}) row["projected_top_two_margin_contribution_votes"] = ( float(pv.get(overall_leader, 0.0)) - float(pv.get(overall_runner, 0.0)) ) else: for row in county_rows: row["projected_top_two_margin_contribution_votes"] = 0.0 county_rows.sort( key=lambda r: ( abs(float(r.get("projected_top_two_margin_contribution_votes") or 0.0)), float(r.get("estimated_remaining_votes") or 0.0), ), reverse=True, ) market_rows.sort(key=lambda r: (float(r.get("effective_influence") or 0.0), float(r.get("estimated_remaining_votes") or 0.0)), reverse=True) mapped_total = mapped_current_votes + unmapped_current_votes mapped_vote_coverage = ( mapped_current_votes / mapped_total if mapped_total > 0 else 0.0 ) expected_market_state_total = sum( float(r.get("expected_final_votes") or 0.0) for r in market_rows ) # V4.1: Every market receives an explicit effective influence: # electorate weight × reporting confidence × internal reliability. # # Tiny markets can still be informative locally, but they cannot cancel a # giant market merely because each is "one market". weighted_market_progress_num = 0.0 weighted_market_progress_den = 0.0 reliability_num = 0.0 reliability_den = 0.0 for r in market_rows: expected_votes = float(r.get("expected_final_votes") or 0.0) progress = max(0.0, min(1.0, float(r.get("progress") or 0.0))) electorate_weight = ( expected_votes / expected_market_state_total if expected_market_state_total > 0 else 0.0 ) residual_values = list((r.get("residual_points") or {}).values()) residual_mae = ( sum(abs(float(v)) for v in residual_values) / len(residual_values) if residual_values else 0.0 ) # Reporting confidence grows sub-linearly. A market at 25% reporting # has meaningful evidence without pretending it is complete. reporting_confidence = math.sqrt(progress) # Market reliability penalizes very large unexplained residuals, but # does not punish normal market-specific swing. # # <= 4 pts residual ~ near full reliability # 10 pts residual ~ ~60% # 20 pts residual ~ ~20% reliability = math.exp(-max(0.0, residual_mae - 4.0) / 9.0) reliability = max(0.05, min(1.0, reliability)) effective_influence = ( electorate_weight * reporting_confidence * reliability ) r["electorate_weight"] = electorate_weight r["reporting_confidence"] = reporting_confidence r["residual_mae_points"] = residual_mae r["reliability"] = reliability r["effective_influence"] = effective_influence weighted_market_progress_num += expected_votes * progress weighted_market_progress_den += expected_votes reliability_weight = electorate_weight * max(0.02, reporting_confidence) reliability_num += reliability_weight * reliability reliability_den += reliability_weight weighted_market_progress = ( weighted_market_progress_num / weighted_market_progress_den if weighted_market_progress_den > 0 else 0.0 ) # This is now electorate/reporting weighted, not one-market-one-vote. market_coherence = ( reliability_num / reliability_den if reliability_den > 0 else 0.0 ) total_effective_influence = sum( float(r.get("effective_influence") or 0.0) for r in market_rows ) max_effective_market = max( market_rows, key=lambda r: float(r.get("effective_influence") or 0.0), default=None, ) result["media_markets"] = { "available": True, "source": market_map_source, "reference_url": ref_url, "atomic_geography": ( "municipality" if str(state or "").upper() == "MA" else "county" ), "ma_municipality_county_bridge": ( ma_municipality_bridge_diagnostics() if str(state or "").upper() == "MA" else None ), "mapped_current_vote_coverage": mapped_vote_coverage, "mapped_current_votes": mapped_current_votes, "unmapped_current_votes": unmapped_current_votes, "markets_used": len(market_rows), "weighted_market_progress": weighted_market_progress, "market_coherence": market_coherence, "total_effective_influence": total_effective_influence, "largest_effective_market": ( max_effective_market.get("market") if max_effective_market else None ), "largest_effective_market_influence": ( float(max_effective_market.get("effective_influence") or 0.0) if max_effective_market else 0.0 ), "candidate_baseline_available": candidate_baseline, "global_swing_points": {n: 100.0 * global_swing.get(n, 0.0) for n in names}, "markets": market_rows, "counties": county_rows, } return result def snapshot_history_window( base_url: str, snapshot_name: str, max_intervals: int = 3, ) -> List[Dict[str, Any]]: """ Return only snapshots available AT OR BEFORE snapshot_name. max_intervals=3 means current + up to three prior dumps, producing up to three recent vote intervals. Future snapshots are never loaded. """ base_url = slug_base(base_url) metadata = fetch_json(base_url + "snapshot_metadata.json") items = [ item for item in (metadata.get("snapshots") or []) if item.get("filename") ] def order_key(item): try: return (0, int(item.get("snapshot"))) except (TypeError, ValueError): return (1, str(item.get("timestamp") or ""), str(item.get("filename") or "")) items.sort(key=order_key) idx = next( (i for i, item in enumerate(items) if item.get("filename") == snapshot_name), None, ) if idx is None: return [] chosen = items[max(0, idx - max_intervals): idx + 1] out = [] for item in chosen: row = dict(item) row["data"] = fetch_json(base_url + str(item["filename"])) out.append(row) return out def temporal_influence_cap(counted_fraction: float) -> float: """Maximum V4.5 ensemble weight allowed for recent-return velocity.""" x = max(0.0, min(1.0, float(counted_fraction or 0.0))) knots = [ (0.00, 0.03), (0.03, 0.05), (0.10, 0.15), (0.25, 0.20), (0.50, 0.30), (0.75, 0.35), (1.00, 0.45), ] for (x0, y0), (x1, y1) in zip(knots, knots[1:]): if x <= x1: if x1 == x0: return y1 t = (x - x0) / (x1 - x0) return y0 + t * (y1 - y0) return knots[-1][1] def _temporal_recency_weights(n: int) -> List[float]: if n <= 0: return [] if n == 1: return [1.0] if n == 2: return [0.375, 0.625] # Oldest -> newest. Discussion target: 20 / 30 / 50. base = [0.20, 0.30, 0.50] return base[-n:] def _positive_interval( earlier: Dict[str, int], later: Dict[str, int], names: List[str], ) -> Dict[str, Any]: raw = {n: int(later.get(n, 0)) - int(earlier.get(n, 0)) for n in names} positive = {n: max(0, raw[n]) for n in names} added = sum(positive.values()) negative_corrections = sum(abs(min(0, raw[n])) for n in names) net_added = sum(raw.values()) correction_ratio = negative_corrections / max(1.0, float(added)) correction_reliability = max(0.20, 1.0 - min(0.80, 2.0 * correction_ratio)) shares = _normalized(positive, names) if added > 0 else {n: 0.0 for n in names} ranked = sorted(names, key=lambda n: shares.get(n, 0.0), reverse=True) leader = ranked[0] if added > 0 and ranked else None runner = ranked[1] if added > 0 and len(ranked) > 1 else None margin = ( 100.0 * (shares[leader] - shares[runner]) if leader and runner else None ) return { "raw_deltas": raw, "positive_deltas": positive, "new_votes": added, "net_new_votes": net_added, "negative_correction_votes": negative_corrections, "correction_reliability": correction_reliability, "shares": shares, "leader": leader, "margin_points": margin, } def combine_temporal_intervals( intervals: List[Dict[str, Any]], names: List[str], current_votes: int, ) -> Dict[str, Any]: valid = [x for x in intervals if int(x.get("new_votes") or 0) > 0] if not valid: return { "available": False, "shares": {n: 0.0 for n in names}, "confidence": 0.0, "stability": 0.0, "leader_consistency": 0.0, "recent_votes": 0, "recent_vote_share_of_current": 0.0, "momentum_slope_points_per_dump": 0.0, "trend_label": "No usable recent vote intervals", } recency = _temporal_recency_weights(len(valid)) max_votes = max(int(x.get("new_votes") or 0) for x in valid) raw_weights = [] for rw, interval in zip(recency, valid): volume = int(interval.get("new_votes") or 0) volume_factor = math.sqrt(volume / max_votes) if max_votes > 0 else 0.0 correction = float(interval.get("correction_reliability") or 0.0) raw_weights.append(rw * volume_factor * correction) total_w = sum(raw_weights) if total_w <= 0: raw_weights = recency total_w = sum(raw_weights) weights = [w / total_w for w in raw_weights] shares = {n: 0.0 for n in names} for w, interval in zip(weights, valid): for n in names: shares[n] += w * float((interval.get("shares") or {}).get(n, 0.0)) shares = _normalized(shares, names) # Dispersion around the recent-return center. dispersion = 0.0 for w, interval in zip(weights, valid): s = interval.get("shares") or {} mae = sum(abs(float(s.get(n, 0.0)) - shares.get(n, 0.0)) for n in names) / max(1, len(names)) dispersion += w * mae stability = math.exp(-dispersion / 0.055) ranked = sorted(names, key=lambda n: shares.get(n, 0.0), reverse=True) trend_leader = ranked[0] if ranked else None leader_consistency = sum( w for w, interval in zip(weights, valid) if interval.get("leader") == trend_leader ) recent_votes = sum(int(x.get("new_votes") or 0) for x in valid) recent_share = recent_votes / max(1.0, float(current_votes)) volume_confidence = min(1.0, math.sqrt(max(0.0, recent_share) / 0.25)) interval_confidence = min(1.0, len(valid) / 3.0) correction_confidence = sum( w * float(interval.get("correction_reliability") or 0.0) for w, interval in zip(weights, valid) ) confidence = ( interval_confidence * volume_confidence * stability * (0.55 + 0.45 * leader_consistency) * correction_confidence ) confidence = max(0.0, min(1.0, confidence)) # Momentum slope for the weighted trend leader, measured against its best # opponent in each successive vote batch. margins_for_leader = [] if trend_leader: for interval in valid: s = interval.get("shares") or {} opponent = max((float(s.get(n, 0.0)) for n in names if n != trend_leader), default=0.0) margins_for_leader.append(100.0 * (float(s.get(trend_leader, 0.0)) - opponent)) slope = 0.0 if len(margins_for_leader) >= 2: xs = list(range(len(margins_for_leader))) xbar = sum(xs) / len(xs) ybar = sum(margins_for_leader) / len(margins_for_leader) denom = sum((x - xbar) ** 2 for x in xs) if denom > 0: slope = sum((x - xbar) * (y - ybar) for x, y in zip(xs, margins_for_leader)) / denom if not trend_leader: trend_label = "No clear recent-return leader" elif slope >= 1.0: trend_label = f"{trend_leader} strengthening in recent dumps" elif slope <= -1.0: trend_label = f"{trend_leader} recent batch advantage is weakening" else: trend_label = f"{trend_leader} recent-return advantage is broadly stable" return { "available": True, "shares": shares, "weights": weights, "confidence": confidence, "stability": stability, "dispersion_points": 100.0 * dispersion, "leader": trend_leader, "leader_consistency": leader_consistency, "recent_votes": recent_votes, "recent_vote_share_of_current": recent_share, "volume_confidence": volume_confidence, "interval_confidence": interval_confidence, "correction_confidence": correction_confidence, "momentum_slope_points_per_dump": slope, "trend_label": trend_label, } def aggregate_snapshot_by_market( snapshot: Dict[str, Any], mapping: Dict[str, Any], names: List[str], ) -> Dict[str, Dict[str, int]]: out: Dict[str, Dict[str, int]] = {} for region in county_regions(snapshot): market = market_for_region(region, mapping) if not market: continue out.setdefault(market, {n: 0 for n in names}) votes = _candidate_votes_in_region(region, names) for n in names: out[market][n] += int(votes.get(n, 0)) return out def project_temporal_model( base_url: str, snapshot_name: str, v3_result: Dict[str, Any], market_result: Optional[Dict[str, Any]] = None, market_map_source: Optional[str] = None, ) -> Optional[Dict[str, Any]]: """ V4.5 recent-return velocity component. Uses ONLY prior/current snapshots. It never loads a later numbered dump. Up to three intervals are analyzed statewide and, when available, by media market. Already-counted votes remain fixed; only estimated remaining votes receive the temporal share estimate. """ base_url = slug_base(base_url) history = snapshot_history_window(base_url, snapshot_name, max_intervals=3) if len(history) < 2: return None current_snapshot = history[-1]["data"] final = fetch_json(base_url + "latest.json") # scoring only current_candidates = candidate_map(current_snapshot) names = list(current_candidates.keys()) current_vote_total = sum(c["votes"] for c in current_candidates.values()) intervals = [] for earlier, later in zip(history, history[1:]): ev = {n: int((candidate_map(earlier["data"]).get(n) or {}).get("votes") or 0) for n in names} lv = {n: int((candidate_map(later["data"]).get(n) or {}).get("votes") or 0) for n in names} interval = _positive_interval(ev, lv, names) interval.update({ "from_snapshot": earlier.get("snapshot"), "from_filename": earlier.get("filename"), "from_timestamp": earlier.get("timestamp"), "to_snapshot": later.get("snapshot"), "to_filename": later.get("filename"), "to_timestamp": later.get("timestamp"), }) intervals.append(interval) combined = combine_temporal_intervals(intervals, names, current_vote_total) if not combined.get("available"): return None v3d = v3_result.get("v3") or {} estimated_final = v3d.get("estimated_final_votes") estimated_remaining = v3d.get("estimated_remaining_votes") if estimated_final is None or float(estimated_final or 0.0) <= current_vote_total: # If V3 has no historical turnout estimate, use reporting maturity only # as a conservative fallback and cap expansion to avoid absurd totals. maturity = statewide_reporting_maturity(current_snapshot) effective = max(0.12, maturity) estimated_final = min(current_vote_total * 7.0, current_vote_total / effective) estimated_final = max(float(current_vote_total), float(estimated_final)) estimated_remaining = max(0.0, estimated_final - current_vote_total) else: estimated_final = float(estimated_final) estimated_remaining = max(0.0, float(estimated_remaining or (estimated_final - current_vote_total))) estimated_counted_fraction = min(1.0, current_vote_total / max(1.0, estimated_final)) max_influence = temporal_influence_cap(estimated_counted_fraction) # Default statewide temporal forecast. temporal_remaining_shares = dict(combined["shares"]) projected_votes = { n: float(current_candidates[n]["votes"]) + estimated_remaining * temporal_remaining_shares.get(n, 0.0) for n in names } market_temporal_rows = [] market_coverage_remaining = 0.0 if market_result and market_map_source: state = current_snapshot.get("province") or final.get("province") mapping = load_media_market_map(market_map_source, state) if mapping.get("available"): market_snapshots = [aggregate_snapshot_by_market(h["data"], mapping, names) for h in history] market_interval_map: Dict[str, List[Dict[str, Any]]] = {} all_markets = set() for agg in market_snapshots: all_markets.update(agg.keys()) for market_name in all_markets: rows = [] for i in range(len(market_snapshots) - 1): earlier_votes = market_snapshots[i].get(market_name, {n: 0 for n in names}) later_votes = market_snapshots[i + 1].get(market_name, {n: 0 for n in names}) row = _positive_interval(earlier_votes, later_votes, names) row.update({ "from_snapshot": history[i].get("snapshot"), "to_snapshot": history[i + 1].get("snapshot"), }) rows.append(row) market_interval_map[market_name] = rows market_current = { m.get("market"): m for m in ((market_result.get("media_markets") or {}).get("markets") or []) if m.get("market") } mapped_remaining = sum( max(0.0, float(m.get("estimated_remaining_votes") or 0.0)) for m in market_current.values() ) remaining_scale = ( min(1.0, estimated_remaining / mapped_remaining) if mapped_remaining > 0 else 0.0 ) # Rebuild the temporal projection from current votes + market-level # remaining vote. Unmapped remaining falls back to statewide trend. projected_votes = {n: float(current_candidates[n]["votes"]) for n in names} allocated_remaining = 0.0 for market_name, market_info in market_current.items(): raw_remaining = max(0.0, float(market_info.get("estimated_remaining_votes") or 0.0)) market_remaining = raw_remaining * remaining_scale if market_remaining <= 0: continue market_rows = market_interval_map.get(market_name, []) market_current_votes = int(market_info.get("current_votes") or 0) market_combined = combine_temporal_intervals( market_rows, names, max(1, market_current_votes), ) local_conf = float(market_combined.get("confidence") or 0.0) local_conf *= float(market_info.get("reliability") or 1.0) local_conf = max(0.0, min(1.0, local_conf)) if market_combined.get("available"): local_shares = _normalized({ n: ( local_conf * float((market_combined.get("shares") or {}).get(n, 0.0)) + (1.0 - local_conf) * temporal_remaining_shares.get(n, 0.0) ) for n in names }, names) else: local_shares = temporal_remaining_shares for n in names: projected_votes[n] += market_remaining * local_shares.get(n, 0.0) allocated_remaining += market_remaining ranked = sorted(names, key=lambda n: local_shares.get(n, 0.0), reverse=True) leader = ranked[0] if ranked else None runner = ranked[1] if len(ranked) > 1 else None margin = ( 100.0 * (local_shares[leader] - local_shares[runner]) if leader and runner else None ) market_temporal_rows.append({ "market": market_name, "recent_votes": int(market_combined.get("recent_votes") or 0), "temporal_confidence": local_conf, "trend_label": market_combined.get("trend_label"), "trend_leader": leader, "trend_margin_points": margin, "projected_remaining_shares": {n: 100.0 * local_shares.get(n, 0.0) for n in names}, "estimated_remaining_votes": market_remaining, "electorate_weight": market_info.get("electorate_weight"), "reporting_confidence": market_info.get("reporting_confidence"), "reliability": market_info.get("reliability"), }) unmapped_remaining = max(0.0, estimated_remaining - allocated_remaining) for n in names: projected_votes[n] += unmapped_remaining * temporal_remaining_shares.get(n, 0.0) market_coverage_remaining = ( allocated_remaining / estimated_remaining if estimated_remaining > 0 else 0.0 ) projected_shares = _normalized(projected_votes, names) result = _score_projection( base_url, snapshot_name, current_snapshot, final, projected_shares, "V4_5_TEMPORAL_RETURNS", ( "V4.5 temporal component uses up to the three most recent vote intervals, " "weighted by recency, vote volume, correction quality, and consistency. " "With media markets, recent returns are also estimated market-by-market." ), { "uses_final_results_for_projection": False, "temporal_looks_forward": False, "temporal_max_intervals": 3, }, ) market_temporal_rows.sort( key=lambda r: ( float(r.get("estimated_remaining_votes") or 0.0) * float(r.get("temporal_confidence") or 0.0) ), reverse=True, ) result["temporal"] = { "available": True, "history_snapshots": [ { "snapshot": h.get("snapshot"), "filename": h.get("filename"), "timestamp": h.get("timestamp"), "votes": h.get("votes"), } for h in history ], "intervals": intervals, "recent_return_shares": {n: 100.0 * combined["shares"].get(n, 0.0) for n in names}, "recent_return_leader": combined.get("leader"), "trend_label": combined.get("trend_label"), "momentum_slope_points_per_dump": combined.get("momentum_slope_points_per_dump"), "stability": combined.get("stability"), "dispersion_points": combined.get("dispersion_points"), "leader_consistency": combined.get("leader_consistency"), "recent_votes": combined.get("recent_votes"), "recent_vote_share_of_current": combined.get("recent_vote_share_of_current"), "confidence": combined.get("confidence"), "estimated_final_votes": estimated_final, "estimated_remaining_votes": estimated_remaining, "estimated_counted_fraction": estimated_counted_fraction, "max_ensemble_influence": max_influence, "market_remaining_coverage": market_coverage_remaining, "media_markets": market_temporal_rows, } return result def _runoff_feature_rows( baseline: Dict[str, Any], snapshot: Dict[str, Any], current_names: List[str], max_eliminated: int = 8, ) -> Dict[str, Any]: baseline_state = candidate_map(baseline) baseline_counties = {_region_identity(r): r for r in county_regions(baseline)} live_counties = {_region_identity(r): r for r in county_regions(snapshot)} eliminated_ranked = sorted( [ (name, int(row.get("votes") or 0)) for name, row in baseline_state.items() if name not in current_names and int(row.get("votes") or 0) > 0 ], key=lambda item: item[1], reverse=True, )[:max_eliminated] eliminated = [name for name, _ in eliminated_ranked] if not eliminated: return { "available": False, "eliminated": [], "county_features": {}, "training_rows": [], "baseline_counties": baseline_counties, "live_counties": live_counties, } statewide_votes = {name: int(row.get("votes") or 0) for name, row in baseline_state.items()} statewide_total = sum(statewide_votes.values()) elim_state = { name: statewide_votes.get(name, 0) / statewide_total if statewide_total else 0.0 for name in eliminated } county_features = {} training_rows = [] round1_total = 0 live_training_votes = 0 for cid, region in baseline_counties.items(): all_votes = all_region_votes(region) total = sum(all_votes.values()) if total <= 0: continue round1_total += total pair_votes = {n: all_votes.get(n, 0) for n in current_names} if sum(pair_votes.values()) <= 0: continue pair_share = _normalized(pair_votes, current_names) features = { f"elim::{e}": all_votes.get(e, 0) / total - elim_state.get(e, 0.0) for e in eliminated } features["log_round1_size"] = math.log(max(1.0, total)) county_features[cid] = { "features": features, "pair_share": pair_share, "round1_total": total, "name": region.get("name") or cid, } live = live_counties.get(cid) if not live: continue live_votes = _candidate_votes_in_region(live, current_names) live_total = sum(live_votes.values()) p = _safe_reporting_fraction(live) if live_total < 80 or p < 0.03: continue live_training_votes += live_total live_share = _normalized(live_votes, current_names) row = { "cid": cid, "weight": math.sqrt(live_total) * math.sqrt(max(0.03, p)), "features": dict(features), "live_share": live_share, "pair_share": pair_share, "p": p, "live_total": live_total, "round1_total": total, } for n in current_names: row[f"residual::{n}"] = live_share[n] - pair_share[n] if p >= 0.07: implied_retention = live_total / max(1.0, p * total) row["log_retention"] = math.log(max(0.15, min(2.0, implied_retention))) training_rows.append(row) # Center size feature so ridge coefficients remain stable. size_mean = _weighted_mean([ ((row.get("features") or {}).get("log_round1_size", 0.0), row.get("weight", 1.0)) for row in training_rows ]) for info in county_features.values(): info["features"]["log_round1_size"] -= size_mean for row in training_rows: row["features"]["log_round1_size"] -= size_mean return { "available": bool(training_rows), "eliminated": eliminated, "feature_names": [f"elim::{e}" for e in eliminated] + ["log_round1_size"], "county_features": county_features, "training_rows": training_rows, "baseline_counties": baseline_counties, "live_counties": live_counties, "round1_total": round1_total, "live_training_votes": live_training_votes, "size_mean": size_mean, } def _fit_runoff_transfer_models( rows: List[Dict[str, Any]], feature_names: List[str], current_names: List[str], ) -> Dict[str, Any]: candidate_fits = { n: _weighted_ridge_fit( rows, feature_names, f"residual::{n}", ridge=0.08, ) for n in current_names } turnout_rows = [row for row in rows if "log_retention" in row] turnout_fit = _weighted_ridge_fit( turnout_rows, feature_names, "log_retention", ridge=0.14, ) return { "candidate_fits": candidate_fits, "turnout_fit": turnout_fit, "turnout_rows": len(turnout_rows), } def _runoff_transfer_predict( info: Dict[str, Any], fits: Dict[str, Any], cid: str, current_names: List[str], ) -> Optional[Dict[str, Any]]: county = (info.get("county_features") or {}).get(cid) if not county: return None features = county["features"] raw = {} for n in current_names: fit = (fits.get("candidate_fits") or {}).get(n) or {} adjustment = _ridge_predict(fit, features) if fit.get("available") else 0.0 adjustment = max(-0.22, min(0.22, adjustment)) raw[n] = county["pair_share"].get(n, 0.0) + adjustment shares = _normalized(raw, current_names) turnout_fit = fits.get("turnout_fit") or {} if turnout_fit.get("available"): retention = math.exp(max(-1.8, min(0.60, _ridge_predict(turnout_fit, features)))) else: retention = None return { "shares": shares, "retention": retention, "round1_total": county["round1_total"], "county": county["name"], } def _cross_validate_runoff_transfer( info: Dict[str, Any], current_names: List[str], ) -> Dict[str, Any]: """ V6 runoff specialist exam. The specialist must pass three separate tests: 1. held-out county finalist-share accuracy, 2. held-out turnout-retention accuracy, 3. aggregate held-out margin accuracy after turnout weighting. This prevents a model from receiving large statewide weight merely because it learned the direction of transfers while mis-sizing the electorate. """ rows = info.get("training_rows") or [] feature_names = info.get("feature_names") or [] if len(rows) < 8: return { "available": False, "counties": len(rows), "mae": None, "share_mae": None, "turnout_log_mae": None, "aggregate_margin_mae_points": None, "score": 0.12, } share_err_num = 0.0 share_err_den = 0.0 turnout_err_num = 0.0 turnout_err_den = 0.0 fold_margin_errors = [] predicted = 0 for fold in (0, 1): train = [ row for row in rows if sum(ord(ch) for ch in row["cid"]) % 2 != fold ] valid = [ row for row in rows if sum(ord(ch) for ch in row["cid"]) % 2 == fold ] if len(train) < 4 or not valid: continue fits = _fit_runoff_transfer_models(train, feature_names, current_names) retention_train = [ (math.exp(row["log_retention"]), row["weight"]) for row in train if "log_retention" in row ] fallback_retention = max( 0.20, min(1.40, weighted_median(retention_train, 0.62)), ) pred_agg = {n: 0.0 for n in current_names} actual_agg = {n: 0.0 for n in current_names} for row in valid: pred_info = _runoff_transfer_predict( info, fits, row["cid"], current_names ) if not pred_info: continue share_mae = sum( abs( pred_info["shares"].get(n, 0.0) - row["live_share"].get(n, 0.0) ) for n in current_names ) / max(1, len(current_names)) w = row["weight"] share_err_num += w * share_mae share_err_den += w predicted_retention = pred_info.get("retention") if predicted_retention is None: predicted_retention = fallback_retention predicted_retention = max( 0.18, min(1.60, float(predicted_retention)) ) if "log_retention" in row: turnout_log_error = abs( math.log(predicted_retention) - float(row["log_retention"]) ) turnout_err_num += w * turnout_log_error turnout_err_den += w predicted_final = ( float(row["round1_total"]) * predicted_retention ) actual_final_est = ( float(row["live_total"]) / max(0.05, float(row.get("p") or 0.0)) ) # Very low reporting fractions produce unstable turnout estimates. actual_final_est = min( float(row["round1_total"]) * 2.25, max( float(row["live_total"]), actual_final_est, ), ) for n in current_names: pred_agg[n] += ( predicted_final * pred_info["shares"].get(n, 0.0) ) actual_agg[n] += ( actual_final_est * row["live_share"].get(n, 0.0) ) predicted += 1 if sum(pred_agg.values()) > 0 and sum(actual_agg.values()) > 0: pred_shares = _normalized(pred_agg, current_names) actual_shares = _normalized(actual_agg, current_names) pred_rank = sorted( current_names, key=lambda n: pred_shares.get(n, 0.0), reverse=True, ) actual_rank = sorted( current_names, key=lambda n: actual_shares.get(n, 0.0), reverse=True, ) if len(pred_rank) >= 2 and len(actual_rank) >= 2: # Score the margin of the actual held-out leader so direction # errors are penalized, not hidden by absolute margins. actual_leader = actual_rank[0] actual_opp = max( ( actual_shares.get(n, 0.0) for n in current_names if n != actual_leader ), default=0.0, ) pred_opp = max( ( pred_shares.get(n, 0.0) for n in current_names if n != actual_leader ), default=0.0, ) actual_margin = ( actual_shares.get(actual_leader, 0.0) - actual_opp ) pred_margin = ( pred_shares.get(actual_leader, 0.0) - pred_opp ) fold_margin_errors.append( abs(pred_margin - actual_margin) ) if share_err_den <= 0 or predicted < 4: return { "available": False, "counties": predicted, "mae": None, "share_mae": None, "turnout_log_mae": None, "aggregate_margin_mae_points": None, "score": 0.12, } share_mae = share_err_num / share_err_den turnout_log_mae = ( turnout_err_num / turnout_err_den if turnout_err_den > 0 else 0.40 ) aggregate_margin_mae = ( sum(fold_margin_errors) / len(fold_margin_errors) if fold_margin_errors else 0.10 ) share_score = math.exp(-share_mae / 0.035) turnout_score = math.exp(-turnout_log_mae / 0.22) margin_score = math.exp(-aggregate_margin_mae / 0.030) # The full turnout-weighted margin exam gets the largest weight. combined_score = ( 0.30 * share_score + 0.25 * turnout_score + 0.45 * margin_score ) return { "available": True, "counties": predicted, "mae": share_mae, "share_mae": share_mae, "turnout_log_mae": turnout_log_mae, "aggregate_margin_mae_points": 100.0 * aggregate_margin_mae, "share_score": max(0.0, min(1.0, share_score)), "turnout_score": max(0.0, min(1.0, turnout_score)), "aggregate_margin_score": max( 0.0, min(1.0, margin_score) ), "score": max(0.02, min(1.0, combined_score)), } def project_runoff_transfer_expert( base_url: str, snapshot_name: str, ) -> Optional[Dict[str, Any]]: """ V5 learned runoff-transfer expert. It learns, without ideological labels, how Round-1 eliminated-candidate geography predicts runoff finalist movement and turnout retention. """ base_url = slug_base(base_url) baseline_url = round1_baseline_url(base_url) if not baseline_url: return None snapshot = fetch_json(base_url + snapshot_name) final = fetch_json(base_url + "latest.json") current = candidate_map(snapshot) names = list(current.keys()) if len(names) < 2: return None baseline = fetch_json(baseline_url) info = _runoff_feature_rows(baseline, snapshot, names) if not info.get("available"): return None rows = info["training_rows"] feature_names = info["feature_names"] fits = _fit_runoff_transfer_models(rows, feature_names, names) cv = _cross_validate_runoff_transfer(info, names) live_training_votes = float(info.get("live_training_votes") or 0.0) round1_total = float(info.get("round1_total") or 0.0) county_evidence = min(1.0, math.sqrt(len(rows) / 24.0)) vote_evidence = min( 1.0, math.sqrt(live_training_votes / max(1.0, 0.22 * round1_total)), ) training_evidence = math.sqrt(max(0.0, county_evidence * vote_evidence)) cv_score = float(cv.get("score") or 0.20) confidence = max( 0.0, min(1.0, training_evidence * (0.30 + 0.70 * cv_score)), ) # Global turnout-retention fallback learned from reporting counties. retention_samples = [] for row in rows: if "log_retention" in row: retention_samples.append( (math.exp(row["log_retention"]), row["weight"]) ) global_retention = weighted_median(retention_samples, 0.62) global_retention = max(0.20, min(1.40, global_retention)) live_counties = info["live_counties"] projected_votes = {n: 0.0 for n in names} county_rows = [] estimated_remaining = 0.0 for cid, county in (info.get("county_features") or {}).items(): pred_info = _runoff_transfer_predict(info, fits, cid, names) if not pred_info: continue live = live_counties.get(cid) live_votes = _candidate_votes_in_region(live, names) if live else {n: 0 for n in names} live_total = sum(live_votes.values()) p = _safe_reporting_fraction(live) retention = pred_info.get("retention") if retention is None: retention = global_retention retention = ( confidence * retention + (1.0 - confidence) * global_retention ) retention = max(0.18, min(1.60, retention)) historical_expected = pred_info["round1_total"] * retention if live_total > 0 and p > 0: local_expected = live_total / max(0.05, p) local_turnout_weight = min(0.92, p ** 2.2) expected_final = ( (1.0 - local_turnout_weight) * historical_expected + local_turnout_weight * local_expected ) else: expected_final = historical_expected expected_final = max(float(live_total), float(expected_final)) remaining = max(0.0, expected_final - live_total) estimated_remaining += remaining remaining_shares = dict(pred_info["shares"]) if live_total > 0: live_share = _normalized(live_votes, names) local_blend = min(0.88, p ** 0.60) remaining_shares = _normalized({ n: (1.0 - local_blend) * remaining_shares[n] + local_blend * live_share.get(n, 0.0) for n in names }, names) for n in names: projected_votes[n] += live_votes.get(n, 0) + remaining * remaining_shares[n] county_rows.append({ "cid": cid, "county": pred_info["county"], "reporting": 100.0 * p, "current_votes": live_total, "round1_votes": pred_info["round1_total"], "predicted_turnout_retention": retention, "estimated_final_votes": expected_final, "estimated_remaining_votes": remaining, "prediction_informative": True, "projected_remaining_shares": { n: 100.0 * remaining_shares[n] for n in names }, }) # Preserve statewide votes not represented in county rows. county_current = {n: 0 for n in names} for region in county_regions(snapshot): votes = _candidate_votes_in_region(region, names) for n in names: county_current[n] += votes.get(n, 0) for n in names: projected_votes[n] += max( 0, int(current[n].get("votes") or 0) - county_current[n], ) projected = _normalized(projected_votes, names) result = _score_projection( base_url, snapshot_name, snapshot, final, projected, "V6_LEARNED_RUNOFF_TRANSFER", ( "V6 learns finalist movement from the geographic footprint of eliminated " "Round-1 candidates with ridge regularization and held-out county validation. " "A separate live turnout-retention regression estimates the runoff electorate." ), { "uses_final_results_for_projection": False, "round1_baseline_url": baseline_url, "specialist": "runoff_transfer", }, ) county_rows.sort( key=lambda row: float(row.get("estimated_remaining_votes") or 0.0), reverse=True, ) result["runoff_transfer"] = { "available": True, "baseline_url": baseline_url, "eliminated_candidates": info.get("eliminated") or [], "feature_names": feature_names, "training_counties": len(rows), "live_training_votes": int(live_training_votes), "training_evidence": training_evidence, "cross_validation_available": bool(cv.get("available")), "cross_validation_counties": cv.get("counties"), "cross_validation_mae_points": ( 100.0 * float(cv["share_mae"]) if cv.get("share_mae") is not None else None ), "cross_validation_turnout_log_mae": cv.get("turnout_log_mae"), "cross_validation_aggregate_margin_mae_points": cv.get( "aggregate_margin_mae_points" ), "cross_validation_share_score": cv.get("share_score"), "cross_validation_turnout_score": cv.get("turnout_score"), "cross_validation_aggregate_margin_score": cv.get( "aggregate_margin_score" ), "cross_validation_score": cv_score, "confidence": confidence, "global_turnout_retention": global_retention, "max_ensemble_influence": min(0.58, 0.12 + 0.58 * confidence), "estimated_remaining_votes": estimated_remaining, "candidate_coefficients": { n: (fits["candidate_fits"].get(n) or {}).get("coefficients") for n in names }, "turnout_coefficients": (fits.get("turnout_fit") or {}).get("coefficients"), "counties": county_rows, "method_note": ( "No ideological equivalence is hard-coded. Eliminated-candidate county " "shares become predictors only if they improve live held-out runoff fit." ), } return result def _latent_factor_maps(patterns: List[Dict[str, Any]], max_factors: int = 4) -> List[Dict[str, Any]]: """Approximate PCA in county space using power iteration on historical patterns.""" ranked = sorted( patterns, key=lambda p: float(p.get("std") or 0.0) * math.sqrt(float(p.get("reference_weight") or 0.02)), reverse=True, )[:40] county_ids = sorted({ cid for p in ranked for cid in (p.get("residuals") or {}).keys() }) if len(county_ids) < 8 or len(ranked) < 3: return [] vectors = [] for p in ranked: std = max(0.008, float(p.get("std") or 0.0)) scale = math.sqrt(max(0.02, float(p.get("reference_weight") or 0.02))) vec = { cid: scale * float((p.get("residuals") or {}).get(cid, 0.0)) / std for cid in county_ids } norm = math.sqrt(sum(v * v for v in vec.values())) if norm > 1e-9: vectors.append({cid: v / norm for cid, v in vec.items()}) if len(vectors) < 3: return [] factors = [] for factor_idx in range(min(max_factors, len(vectors))): seed_vec = dict(vectors[factor_idx % len(vectors)]) v = seed_vec for _ in range(18): new = {cid: 0.0 for cid in county_ids} for x in vectors: dot = sum(x[cid] * v.get(cid, 0.0) for cid in county_ids) for cid in county_ids: new[cid] += dot * x[cid] for prior in factors: pv = prior["values"] dot = sum(new[cid] * pv.get(cid, 0.0) for cid in county_ids) for cid in county_ids: new[cid] -= dot * pv.get(cid, 0.0) norm = math.sqrt(sum(value * value for value in new.values())) if norm <= 1e-10: break v = {cid: value / norm for cid, value in new.items()} norm = math.sqrt(sum(value * value for value in v.values())) if norm <= 1e-8: continue # Scale factors to election-share-like units. values = {cid: 0.045 * value / max(1e-9, max(abs(x) for x in v.values())) for cid, value in v.items()} factors.append({"name": f"factor_{len(factors)+1}", "values": values}) return factors def _fit_latent_candidate( candidate: str, training_rows: List[Dict[str, Any]], factors: List[Dict[str, Any]], ) -> Dict[str, Any]: feature_names = [f["name"] for f in factors] rows = [] for row in training_rows: features = { f["name"]: float((f.get("values") or {}).get(row["cid"], 0.0)) for f in factors } rows.append({ "weight": row["weight"], "features": features, "target": row["shares"].get(candidate, 0.0), }) return _weighted_ridge_fit(rows, feature_names, "target", ridge=0.10) def _latent_cv( training_rows: List[Dict[str, Any]], factors: List[Dict[str, Any]], names: List[str], ) -> Dict[str, Any]: if len(training_rows) < 10: return {"available": False, "mae": None, "score": 0.20, "counties": len(training_rows)} err_num = 0.0 err_den = 0.0 count = 0 for fold in (0, 1): train = [r for r in training_rows if sum(ord(ch) for ch in r["cid"]) % 2 != fold] valid = [r for r in training_rows if sum(ord(ch) for ch in r["cid"]) % 2 == fold] if len(train) < 5: continue fits = {n: _fit_latent_candidate(n, train, factors) for n in names} for row in valid: raw = {} for n in names: features = { f["name"]: float((f.get("values") or {}).get(row["cid"], 0.0)) for f in factors } raw[n] = max(0.001, _ridge_predict(fits[n], features)) pred = _normalized(raw, names) mae = sum(abs(pred[n] - row["shares"].get(n, 0.0)) for n in names) / len(names) err_num += row["weight"] * mae err_den += row["weight"] count += 1 if err_den <= 0 or count < 5: return {"available": False, "mae": None, "score": 0.20, "counties": count} mae = err_num / err_den return { "available": True, "mae": mae, "score": max(0.03, min(1.0, math.exp(-mae / 0.050))), "counties": count, } def project_latent_coalition_factors( base_url: str, snapshot_name: str, reference_url: Optional[str], v3_result: Optional[Dict[str, Any]] = None, ) -> Optional[Dict[str, Any]]: """V5 latent historical coalition-factor expert.""" ref_url = normalize_reference_url(reference_url) if not ref_url: return None base_url = slug_base(base_url) snapshot = fetch_json(base_url + snapshot_name) final = fetch_json(base_url + "latest.json") current = candidate_map(snapshot) names = list(current.keys()) entries = _historical_reference_entries(ref_url, snapshot) patterns = _historical_candidate_patterns(entries) factors = _latent_factor_maps(patterns, max_factors=4) training_rows = _histgeo_training_rows(snapshot, names) if len(factors) < 2 or len(training_rows) < 5: return None fits = {n: _fit_latent_candidate(n, training_rows, factors) for n in names} cv = _latent_cv(training_rows, factors, names) maturity = statewide_reporting_maturity(snapshot) training_evidence = min(1.0, math.sqrt(len(training_rows) / 28.0)) * min( 1.0, math.sqrt(max(0.0, maturity) / 0.30) ) cv_score = float(cv.get("score") or 0.20) confidence = max(0.0, min(1.0, training_evidence * (0.35 + 0.65 * cv_score))) if v3_result is None: v3_result = project_v3(base_url, snapshot_name, ref_url) v3_rows = { str(row.get("cid") or ""): row for row in ((v3_result.get("v3") or {}).get("counties") or []) if row.get("cid") } live_counties = {_region_identity(r): r for r in county_regions(snapshot)} statewide_live = shares_from_votes(current) projected_votes = {n: 0.0 for n in names} county_rows = [] estimated_remaining = 0.0 all_ids = set(v3_rows) | set(live_counties) for cid in all_ids: live = live_counties.get(cid) live_votes = _candidate_votes_in_region(live, names) if live else {n: 0 for n in names} live_total = sum(live_votes.values()) p = _safe_reporting_fraction(live) remaining = max(0.0, float((v3_rows.get(cid) or {}).get("estimated_remaining_votes") or 0.0)) estimated_remaining += remaining features = { f["name"]: float((f.get("values") or {}).get(cid, 0.0)) for f in factors } raw = { n: max(0.001, _ridge_predict(fits[n], features)) for n in names } pred = _normalized(raw, names) pred = _normalized({ n: confidence * pred[n] + (1.0 - confidence) * statewide_live.get(n, 0.0) for n in names }, names) if live_total > 0: live_share = _normalized(live_votes, names) local_blend = min(0.90, math.sqrt(max(0.0, p))) pred = _normalized({ n: (1.0 - local_blend) * pred[n] + local_blend * live_share[n] for n in names }, names) for n in names: projected_votes[n] += live_votes.get(n, 0) + remaining * pred[n] county_rows.append({ "cid": cid, "county": (live or {}).get("name") or cid, "reporting": 100.0 * p, "current_votes": live_total, "estimated_remaining_votes": remaining, "prediction_informative": bool(factors), "projected_remaining_shares": {n: 100.0 * pred[n] for n in names}, }) # Preserve statewide residual. county_current = {n: 0 for n in names} for region in county_regions(snapshot): votes = _candidate_votes_in_region(region, names) for n in names: county_current[n] += votes.get(n, 0) for n in names: projected_votes[n] += max(0, int(current[n].get("votes") or 0) - county_current[n]) result = _score_projection( base_url, snapshot_name, snapshot, final, _normalized(projected_votes, names), "V6_LATENT_COALITION_FACTORS", ( "V6 compresses many historical candidate county maps into orthogonal " "latent coalition factors, then ridge-fits current candidates to those " "factors with held-out county validation." ), {"uses_final_results_for_projection": False, "specialist": "latent_coalition"}, ) result["latent_coalition"] = { "available": True, "historical_patterns": len(patterns), "factors": len(factors), "training_counties": len(training_rows), "training_evidence": training_evidence, "cross_validation_available": bool(cv.get("available")), "cross_validation_counties": cv.get("counties"), "cross_validation_mae_points": 100.0 * cv["mae"] if cv.get("mae") is not None else None, "cross_validation_score": cv_score, "confidence": confidence, "max_ensemble_influence": min(0.32, 0.04 + 0.36 * confidence), "estimated_remaining_votes": estimated_remaining, "factor_names": [f["name"] for f in factors], "counties": county_rows, } return result def project_batch_bias_model( base_url: str, snapshot_name: str, v3_result: Dict[str, Any], ) -> Optional[Dict[str, Any]]: """ V5 within-county reporting-order expert. It learns whether recent batches inside the same county are persistently different from that county's running result, and uses the sign/stability of those residuals to adjust only the remaining vote. """ base_url = slug_base(base_url) history = snapshot_history_window(base_url, snapshot_name, max_intervals=5) if len(history) < 3: return None snapshot = history[-1]["data"] final = fetch_json(base_url + "latest.json") current = candidate_map(snapshot) names = list(current.keys()) current_counties = {_region_identity(r): r for r in county_regions(snapshot)} v3_rows = { str(row.get("cid") or ""): row for row in ((v3_result.get("v3") or {}).get("counties") or []) if row.get("cid") } if not current_counties: return None residual_history: Dict[str, List[Dict[str, Any]]] = {} validation_err_num = 0.0 validation_err_den = 0.0 for i in range(len(history) - 1): before = history[i]["data"] after = history[i + 1]["data"] before_counties = {_region_identity(r): r for r in county_regions(before)} inc = _actual_increment(before, after, names) recency = (i + 1) / max(1, len(history) - 1) for item in inc["changed"]: cid = item["cid"] before_region = before_counties.get(cid) if not before_region: continue before_votes = _candidate_votes_in_region(before_region, names) before_total = sum(before_votes.values()) if before_total < 60 or item["new_votes"] < 40: continue running = _normalized(before_votes, names) batch = _normalized(item["actual_votes"], names) residual = {n: batch[n] - running[n] for n in names} residual_history.setdefault(cid, []).append({ "residual": residual, "batch_share": batch, "running_share": running, "new_votes": item["new_votes"], "recency": recency, }) projected_votes = {n: 0.0 for n in names} county_rows = [] informative_votes = 0.0 confidence_values = [] for cid, region in current_counties.items(): live_votes = _candidate_votes_in_region(region, names) live_total = sum(live_votes.values()) p = _safe_reporting_fraction(region) remaining = max(0.0, float((v3_rows.get(cid) or {}).get("estimated_remaining_votes") or 0.0)) base_raw = (v3_rows.get(cid) or {}).get("projected_remaining_shares") or {} base = ( _normalized({n: float(base_raw.get(n) or 0.0) / 100.0 for n in names}, names) if base_raw else (_normalized(live_votes, names) if live_total > 0 else shares_from_votes(current)) ) rows = residual_history.get(cid) or [] informative = len(rows) >= 2 and sum(r["new_votes"] for r in rows) >= 150 pred = dict(base) local_conf = 0.0 persistence = 0.0 mean_resid = {n: 0.0 for n in names} if informative: den = sum(math.sqrt(r["new_votes"]) * r["recency"] for r in rows) if den > 0: for n in names: mean_resid[n] = sum( math.sqrt(r["new_votes"]) * r["recency"] * r["residual"][n] for r in rows ) / den sims = [ _cosine_similarity(rows[j-1]["residual"], rows[j]["residual"], names) for j in range(1, len(rows)) ] persistence = sum(sims) / len(sims) if sims else 0.0 persistence = max(-0.60, min(0.85, persistence)) volume_factor = min(1.0, math.sqrt(sum(r["new_votes"] for r in rows) / 2500.0)) interval_factor = min(1.0, math.sqrt(len(rows) / 3.0)) local_conf = volume_factor * interval_factor * (0.45 + 0.55 * abs(persistence)) adjustment_scale = 0.55 * persistence * local_conf pred = _normalized({ n: base[n] + max(-0.12, min(0.12, adjustment_scale * mean_resid[n])) for n in names }, names) informative_votes += live_total + remaining confidence_values.append(local_conf) # Internal one-step validation: use all but the final residual to # forecast the last batch direction. if len(rows) >= 3: train = rows[:-1] test = rows[-1] tden = sum(math.sqrt(r["new_votes"]) * r["recency"] for r in train) if tden > 0: train_mean = { n: sum(math.sqrt(r["new_votes"]) * r["recency"] * r["residual"][n] for r in train) / tden for n in names } sim_prev = _cosine_similarity(train[-2]["residual"], train[-1]["residual"], names) if len(train) >= 2 else 0.0 predicted_batch = _normalized({ n: test["running_share"][n] + 0.50 * sim_prev * train_mean[n] for n in names }, names) mae = sum(abs(predicted_batch[n] - test["batch_share"][n]) for n in names) / len(names) w = math.sqrt(test["new_votes"]) validation_err_num += w * mae validation_err_den += w for n in names: projected_votes[n] += live_votes.get(n, 0) + remaining * pred[n] county_rows.append({ "cid": cid, "county": region.get("name") or cid, "reporting": 100.0 * p, "current_votes": live_total, "estimated_remaining_votes": remaining, "prediction_informative": informative, "local_confidence": local_conf, "persistence": persistence, "projected_remaining_shares": {n: 100.0 * pred[n] for n in names}, }) if not confidence_values: return None confidence = min( 1.0, (sum(confidence_values) / len(confidence_values)) * min(1.0, math.sqrt(len(confidence_values) / 16.0)), ) internal_mae = validation_err_num / validation_err_den if validation_err_den > 0 else None internal_score = ( max(0.03, min(1.0, math.exp(-internal_mae / 0.055))) if internal_mae is not None else 0.20 ) result = _score_projection( base_url, snapshot_name, snapshot, final, _normalized(projected_votes, names), "V6_WITHIN_COUNTY_BATCH_BIAS", ( "V6 models persistent or reversing within-county batch residuals. " "It adjusts V3's remaining-vote county expectation rather than treating " "a statewide reporting wave as permanent candidate momentum." ), {"uses_final_results_for_projection": False, "specialist": "batch_bias"}, ) result["batch_bias"] = { "available": True, "informative_counties": len(confidence_values), "informative_electorate_votes": informative_votes, "confidence": confidence, "internal_validation_available": internal_mae is not None, "internal_validation_mae_points": 100.0 * internal_mae if internal_mae is not None else None, "internal_validation_score": internal_score, "max_ensemble_influence": min(0.22, 0.03 + 0.28 * confidence), "counties": county_rows, } return result def precinct_regions(data: Dict[str, Any]) -> List[Dict[str, Any]]: regions = data.get("region_results") if not isinstance(regions, dict): return [] out = [] for key, region in regions.items(): if not isinstance(region, dict): continue rtype = str(region.get("type") or "").strip().casefold() if rtype not in {"precinct", "ward", "vote center", "voting precinct"}: continue row = dict(region) row["_key"] = key out.append(row) return out def _precinct_parent_key(region: Dict[str, Any]) -> Optional[str]: fips = str( region.get("county_fips") or region.get("parent_fips") or region.get("countyFips") or "" ).strip() if fips: return "fips:" + fips county = str( region.get("county") or region.get("county_name") or region.get("parent") or "" ).strip().casefold() if county: return "name:" + re.sub(r"[^a-z0-9]+", "_", county).strip("_") return None def project_precinct_hierarchy_model( base_url: str, snapshot_name: str, ) -> Optional[Dict[str, Any]]: """Optional V5 precinct→county hierarchy. It activates only when precinct rows exist.""" base_url = slug_base(base_url) snapshot = fetch_json(base_url + snapshot_name) final = fetch_json(base_url + "latest.json") current = candidate_map(snapshot) names = list(current.keys()) precincts = precinct_regions(snapshot) if len(precincts) < 5: return None county_live = {_region_identity(r): r for r in county_regions(snapshot)} statewide = shares_from_votes(current) projected_votes = {n: 0.0 for n in names} county_accum: Dict[str, Dict[str, Any]] = {} covered_votes = 0 used = 0 for region in precincts: votes = _candidate_votes_in_region(region, names) total = sum(votes.values()) p = _safe_reporting_fraction(region) if total <= 0: continue parent = _precinct_parent_key(region) parent_region = county_live.get(parent) if parent else None parent_votes = _candidate_votes_in_region(parent_region, names) if parent_region else {} parent_share = _normalized(parent_votes, names) if sum(parent_votes.values()) > 0 else statewide local_share = _normalized(votes, names) reliability = min(1.0, math.sqrt(total / 300.0)) * (0.45 + 0.55 * math.sqrt(max(0.0, p))) share = _normalized({ n: reliability * local_share[n] + (1.0 - reliability) * parent_share.get(n, 0.0) for n in names }, names) expansion = min(20.0, 1.0 / max(0.05, p)) if p > 0 else 1.0 expected_final = max(float(total), total * expansion) for n in names: projected_votes[n] += expected_final * share[n] covered_votes += total used += 1 if parent: acc = county_accum.setdefault(parent, { "remaining": 0.0, "votes": {n: 0.0 for n in names}, "current": 0, }) rem = max(0.0, expected_final - total) acc["remaining"] += rem acc["current"] += total for n in names: acc["votes"][n] += rem * share[n] if used < 5: return None total_current = sum(int(c.get("votes") or 0) for c in current.values()) coverage = min(1.0, covered_votes / max(1.0, total_current)) confidence = coverage * min(1.0, math.sqrt(used / 40.0)) county_rows = [] for cid, acc in county_accum.items(): shares = _normalized(acc["votes"], names) county_rows.append({ "cid": cid, "county": (county_live.get(cid) or {}).get("name") or cid, "current_votes": acc["current"], "estimated_remaining_votes": acc["remaining"], "prediction_informative": acc["remaining"] > 0, "projected_remaining_shares": {n: 100.0 * shares[n] for n in names}, }) result = _score_projection( base_url, snapshot_name, snapshot, final, _normalized(projected_votes, names), "V6_PRECINCT_HIERARCHY", "V6 uses precinct results when available, shrinking sparse precincts toward their county before aggregating statewide.", {"uses_final_results_for_projection": False, "specialist": "precinct_hierarchy"}, ) result["precinct_hierarchy"] = { "available": True, "precincts_used": used, "current_vote_coverage": coverage, "confidence": confidence, "max_ensemble_influence": min(0.36, 0.05 + 0.40 * confidence), "counties": county_rows, } return result def _profile_quantile(values: List[float], q: float) -> Optional[float]: if not values: return None vals = sorted(float(v) for v in values) pos = max(0.0, min(1.0, q)) * (len(vals) - 1) lo, hi = int(math.floor(pos)), int(math.ceil(pos)) if lo == hi: return vals[lo] return vals[lo] + (vals[hi] - vals[lo]) * (pos - lo) def build_historical_reporting_profiles(races: List[Any]) -> Dict[str, Dict[str, Any]]: """ Build state-level reporting-path priors from completed archived races. These profiles are intended for *other* races. Each saved profile records its source race URLs, and live replay automatically refuses to use a profile containing the race currently being backtested. """ accum: Dict[str, Dict[str, Any]] = {} for raw_item in races: item = {"race_url": raw_item} if isinstance(raw_item, str) else raw_item if not isinstance(item, dict) or not item.get("race_url"): continue base_url = slug_base(str(item["race_url"])) try: final = fetch_json(base_url + "latest.json") metadata = fetch_json(base_url + "snapshot_metadata.json") except Exception: continue state = str(final.get("province") or "").upper().strip() if not state: continue final_votes = sum(int(c.get("votes") or 0) for c in candidate_map(final).values()) if final_votes <= 0: continue names = list(candidate_map(final).keys()) items = [x for x in (metadata.get("snapshots") or []) if x.get("filename")] def order_key(x): try: return (0, int(x.get("snapshot"))) except (TypeError, ValueError): return (1, str(x.get("timestamp") or ""), str(x.get("filename") or "")) items.sort(key=order_key) if len(items) < 2: continue resolved = discover_media_market_source(state, item.get("market_map")) mapping = None if resolved: try: mapping = load_media_market_map(resolved, state) except Exception: mapping = None row = accum.setdefault(state, { "dump_fractions": [], "cadence_seconds": [], "changed_county_fractions": [], "market_votes": {}, "source_race_urls": [], "race_count": 0, "interval_count": 0, }) row["source_race_urls"].append(base_url) row["race_count"] += 1 total_counties = max(1, len(county_regions(final))) for before_meta, after_meta in zip(items, items[1:]): try: before = fetch_json(base_url + str(before_meta["filename"])) after = fetch_json(base_url + str(after_meta["filename"])) except Exception: continue inc = _actual_increment(before, after, names) if inc["new_votes"] <= 0: continue row["interval_count"] += 1 row["dump_fractions"].append(inc["new_votes"] / final_votes) row["changed_county_fractions"].append(len(inc["changed"]) / total_counties) try: a = datetime.fromisoformat(str(before_meta.get("timestamp") or "").replace("Z", "+00:00")) b = datetime.fromisoformat(str(after_meta.get("timestamp") or "").replace("Z", "+00:00")) seconds = max(0.0, (b - a).total_seconds()) if seconds > 0: row["cadence_seconds"].append(seconds) except Exception: pass if mapping and mapping.get("available"): after_regions = {_region_identity(r): r for r in county_regions(after)} for changed in inc["changed"]: region = after_regions.get(changed["cid"]) market = market_for_region(region or {}, mapping) if market: row["market_votes"][market] = row["market_votes"].get(market, 0.0) + changed["new_votes"] profiles = {} for state, row in accum.items(): fractions = [x for x in row["dump_fractions"] if x > 0] logs = [math.log(x) for x in fractions] log_sigma = 0.45 if len(logs) >= 2: mean = sum(logs) / len(logs) var = sum((x - mean) ** 2 for x in logs) / (len(logs) - 1) log_sigma = max(0.18, min(1.10, math.sqrt(max(0.0, var)))) mv = row["market_votes"] mv_total = sum(mv.values()) profiles[state] = { "schema_version": 1, "source_type": "v6_historical_reporting_profile", "state": state, "generated_at_utc": datetime.utcnow().isoformat(timespec="seconds") + "Z", "source_race_urls": sorted(set(row["source_race_urls"])), "race_count": row["race_count"], "interval_count": row["interval_count"], "median_dump_fraction_of_final_vote": _profile_quantile(fractions, 0.50), "dump_fraction_log_sigma": log_sigma, "median_cadence_seconds": _profile_quantile(row["cadence_seconds"], 0.50), "median_changed_county_fraction": _profile_quantile(row["changed_county_fractions"], 0.50), "market_activity_share": { market: votes / mv_total for market, votes in mv.items() } if mv_total > 0 else {}, "method_note": ( "Built only from completed archived races. V6 refuses to use this " "profile when replaying any race listed in source_race_urls." ), } return profiles def _load_historical_reporting_profile(state: str, base_url: str) -> Optional[Dict[str, Any]]: state_slug = str(state or "").lower().strip() if not state_slug: return None path = REPORTING_PROFILE_DIR / f"{state_slug}_reporting_profile.json" if not path.is_file(): return None try: profile = json.loads(path.read_text(encoding="utf-8")) except Exception: return None sources = {slug_base(str(x)) for x in (profile.get("source_race_urls") or [])} # Backtest leakage guard. if slug_base(base_url) in sources: return None return profile def learn_reporting_path_profile( base_url: str, snapshot_name: str, market_map_source: Optional[str] = None, max_intervals: int = 12, estimated_final_votes: Optional[float] = None, ) -> Dict[str, Any]: """ Learn reporting-path behavior from already-observed dumps, with an optional state historical prior built from other completed archived races. """ base_url = slug_base(base_url) history = snapshot_history_window(base_url, snapshot_name, max_intervals=max_intervals) if not history: return {"available": False} current = history[-1]["data"] names = list(candidate_map(current).keys()) current_total = sum(int(c.get("votes") or 0) for c in candidate_map(current).values()) state = current.get("province") historical = _load_historical_reporting_profile(str(state or ""), base_url) mapping = None resolved = discover_media_market_source(state, market_map_source) if resolved: try: mapping = load_media_market_map(resolved, state) except Exception: mapping = None sizes = [] county_activity: Dict[str, int] = {} market_activity: Dict[str, float] = {} intervals = [] for before_meta, after_meta in zip(history, history[1:]): inc = _actual_increment(before_meta["data"], after_meta["data"], names) if inc["new_votes"] <= 0: continue sizes.append(inc["new_votes"]) after_regions = {_region_identity(r): r for r in county_regions(after_meta["data"])} for item in inc["changed"]: county_activity[item["cid"]] = county_activity.get(item["cid"], 0) + 1 if mapping and mapping.get("available"): region = after_regions.get(item["cid"]) market = market_for_region(region or {}, mapping) if market: market_activity[market] = market_activity.get(market, 0.0) + item["new_votes"] cadence_seconds = None try: a = datetime.fromisoformat(str(before_meta.get("timestamp") or "").replace("Z", "+00:00")) b = datetime.fromisoformat(str(after_meta.get("timestamp") or "").replace("Z", "+00:00")) cadence_seconds = max(0.0, (b - a).total_seconds()) except Exception: pass intervals.append({ "new_votes": inc["new_votes"], "changed_counties": len(inc["changed"]), "cadence_seconds": cadence_seconds, }) live_evidence = min(1.0, math.sqrt(len(intervals) / 5.0)) if intervals else 0.0 live_median = _profile_quantile(sizes, 0.50) hist_median = None if historical and historical.get("median_dump_fraction_of_final_vote") is not None: est_final = float(estimated_final_votes or 0.0) if est_final <= current_total: maturity = max(0.03, statewide_reporting_maturity(current)) est_final = current_total / maturity if current_total > 0 else 0.0 if est_final > 0: hist_median = float(historical["median_dump_fraction_of_final_vote"]) * est_final if live_median is not None and hist_median is not None: median_size = live_evidence * live_median + (1.0 - live_evidence) * hist_median elif live_median is not None: median_size = live_median elif hist_median is not None: median_size = hist_median else: median_size = max(500.0, 0.05 * max(1.0, current_total)) logs = [math.log(max(1.0, x)) for x in sizes] live_sigma = None if len(logs) >= 2: log_mean = sum(logs) / len(logs) log_var = sum((x - log_mean) ** 2 for x in logs) / (len(logs) - 1) live_sigma = max(0.18, min(0.80, math.sqrt(max(0.0, log_var)))) hist_sigma = float(historical.get("dump_fraction_log_sigma") or 0.45) if historical else None if live_sigma is not None and hist_sigma is not None: dump_sigma = live_evidence * live_sigma + (1.0 - live_evidence) * hist_sigma else: dump_sigma = live_sigma if live_sigma is not None else (hist_sigma if hist_sigma is not None else 0.45) cadence_values = [x["cadence_seconds"] for x in intervals if x.get("cadence_seconds") is not None] live_cadence = _profile_quantile(cadence_values, 0.50) hist_cadence = historical.get("median_cadence_seconds") if historical else None if live_cadence is not None and hist_cadence is not None: cadence = live_evidence * float(live_cadence) + (1.0 - live_evidence) * float(hist_cadence) else: cadence = live_cadence if live_cadence is not None else hist_cadence # Blend live and historical market activity as relative weights. live_mv_total = sum(market_activity.values()) live_mv_share = {k: v / live_mv_total for k, v in market_activity.items()} if live_mv_total > 0 else {} hist_mv_share = historical.get("market_activity_share") or {} if historical else {} markets = set(live_mv_share) | set(hist_mv_share) blended_market_share = {} for market in markets: blended_market_share[market] = ( live_evidence * float(live_mv_share.get(market, 0.0)) + (1.0 - live_evidence) * float(hist_mv_share.get(market, 0.0)) ) bsum = sum(blended_market_share.values()) if bsum > 0: blended_market_share = {k: v / bsum for k, v in blended_market_share.items()} market_activity_votes = { k: v * max(1.0, float(sum(sizes) if sizes else median_size)) for k, v in blended_market_share.items() } return { "available": bool(intervals or historical), "intervals": len(intervals), "live_evidence": live_evidence, "median_dump_votes": median_size, "dump_log_sigma": max(0.18, min(0.90, float(dump_sigma))), "median_cadence_seconds": cadence, "county_recurrence": county_activity, "market_activity_votes": market_activity_votes, "historical_profile_used": bool(historical), "historical_profile_races": historical.get("race_count") if historical else 0, "historical_profile_intervals": historical.get("interval_count") if historical else 0, "historical_profile_source_races": historical.get("source_race_urls") if historical else [], "method_note": ( "V6 blends current-race prior dumps with a state historical reporting " "profile from other archived races when available. The current race " "is automatically excluded from its own historical profile." ), } def component_snapshot_bundle( base_url: str, snapshot_name: str, reference_url: Optional[str] = None, market_map_source: Optional[str] = None, ) -> Dict[str, Any]: """Build/cached component projections for one snapshot without invoking the ensemble.""" base_url = slug_base(base_url) key = (base_url, snapshot_name, normalize_reference_url(reference_url) or "", market_map_source or "") if key in _COMPONENT_CACHE: return _COMPONENT_CACHE[key] snapshot = fetch_json(base_url + snapshot_name) final = fetch_json(base_url + "latest.json") names = list(candidate_map(snapshot).keys()) state = snapshot.get("province") or final.get("province") resolved_market_map_source = discover_media_market_source(state, market_map_source) v0 = project_v0(base_url, snapshot_name) v1 = project_v1(base_url, snapshot_name) v3 = project_v3(base_url, snapshot_name, reference_url) histgeo = None try: histgeo = project_historical_candidate_geo(base_url, snapshot_name, reference_url, v3_result=v3) except Exception: histgeo = None latent = None try: latent = project_latent_coalition_factors(base_url, snapshot_name, reference_url, v3_result=v3) except Exception: latent = None v2r = None runoff_transfer = None if round1_baseline_url(base_url): try: v2r = project_v2r(base_url, snapshot_name) except Exception: v2r = None try: runoff_transfer = project_runoff_transfer_expert(base_url, snapshot_name) except Exception: runoff_transfer = None market = None if resolved_market_map_source: try: market = project_market_model(base_url, snapshot_name, reference_url, resolved_market_map_source) except Exception: market = None temporal = None try: temporal = project_temporal_model( base_url, snapshot_name, v3, market_result=market, market_map_source=resolved_market_map_source, ) except Exception: temporal = None batchbias = None try: batchbias = project_batch_bias_model(base_url, snapshot_name, v3) except Exception: batchbias = None precinct = None try: precinct = project_precinct_hierarchy_model(base_url, snapshot_name) except Exception: precinct = None models = {"v0": v0, "v1": v1, "v3": v3} if v2r: models["v2r"] = v2r if market: models["market"] = market if temporal: models["temporal"] = temporal if histgeo: models["histgeo"] = histgeo if latent: models["latent"] = latent if batchbias: models["batchbias"] = batchbias if runoff_transfer: models["runoff_transfer"] = runoff_transfer if precinct: models["precinct"] = precinct bundle = { "snapshot": snapshot, "final": final, "names": names, "state": state, "resolved_market_map_source": resolved_market_map_source, "models": models, } _COMPONENT_CACHE[key] = bundle return bundle def _row_share_map(rows: List[Dict[str, Any]], names: List[str], share_key: str = "projected_remaining_shares") -> Dict[str, Dict[str, float]]: out = {} for row in rows or []: cid = str(row.get("cid") or row.get("fips") or "").strip() if not cid: continue raw = row.get(share_key) or {} shares = {n: float(raw.get(n) or 0.0) / 100.0 for n in names} if sum(shares.values()) > 0: out[cid] = _normalized(shares, names) return out def _component_local_prediction( model_key: str, model_result: Dict[str, Any], prior_snapshot: Dict[str, Any], names: List[str], item: Dict[str, Any], market_mapping: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ V5 local prediction with provenance. Crucially, an equal-share normalization fallback is never labeled informative. Completion validation only scores genuine ex-ante signals. """ cid = item["cid"] statewide = result_shares(model_result) prior_candidates = candidate_map(prior_snapshot) raw_statewide = shares_from_votes(prior_candidates) statewide_votes = sum(int(c.get("votes") or 0) for c in prior_candidates.values()) prior_counties = {_region_identity(r): r for r in county_regions(prior_snapshot)} region = prior_counties.get(cid) region_votes = _candidate_votes_in_region(region, names) if region else {n: 0 for n in names} region_total = sum(region_votes.values()) p = _safe_reporting_fraction(region) if model_key == "v0": informative = statewide_votes >= max(100, 20 * len(names)) return { "shares": _normalized(raw_statewide, names), "informative": informative, "source": "statewide live scoreboard", "effective_sample_votes": statewide_votes, "geography_level": "statewide", } if model_key == "v1": if region_total >= 50 and p > 0: return { "shares": _normalized(region_votes, names), "informative": True, "source": "county live share", "effective_sample_votes": region_total, "geography_level": "county", } return { "shares": _normalized(raw_statewide, names), "informative": False, "source": "fallback only: county has no usable live share", "effective_sample_votes": region_total, "geography_level": "fallback", } if model_key == "v2r": baseline = model_result.get("runoff_baseline") or {} baseline_shares = baseline.get("baseline_statewide_normalized_shares") or {} if baseline_shares: shares = _normalized({n: float(baseline_shares.get(n) or 0.0) for n in names}, names) return { "shares": shares, "informative": True, "source": "same-cycle Round 1 finalist baseline", "effective_sample_votes": int(baseline.get("round1_total_votes") or 0), "geography_level": "runoff prior", } rows = _model_county_rows(model_key, model_result) row_map = { str(row.get("cid") or row.get("fips") or ""): row for row in rows if row.get("cid") or row.get("fips") } row = row_map.get(cid) if row: raw = row.get("projected_remaining_shares") or {} values = {n: float(raw.get(n) or 0.0) / 100.0 for n in names} if sum(values.values()) > 0: informative = bool(row.get("prediction_informative", True)) effective = max( int(row.get("current_votes") or 0), int(row.get("round1_votes") or 0), int(row.get("reference_votes") or 0), ) return { "shares": _normalized(values, names), "informative": informative, "source": f"{model_key} county remaining-vote prediction", "effective_sample_votes": effective, "geography_level": "county", } if model_key == "temporal": temporal = model_result.get("temporal") or {} temporal_market = {} for mrow in temporal.get("media_markets") or []: raw = mrow.get("projected_remaining_shares") or {} vals = {n: float(raw.get(n) or 0.0) / 100.0 for n in names} if mrow.get("market") and sum(vals.values()) > 0: temporal_market[str(mrow["market"])] = _normalized(vals, names) market_name = None if market_mapping and market_mapping.get("available"): market_name = market_for_region(region or {"fips": cid, "name": item.get("county")}, market_mapping) if market_name and str(market_name) in temporal_market: return { "shares": temporal_market[str(market_name)], "informative": int(temporal.get("recent_votes") or 0) >= 100, "source": "recent-return media-market path", "effective_sample_votes": int(temporal.get("recent_votes") or 0), "geography_level": "media market", } recent_raw = temporal.get("recent_return_shares") or {} vals = {n: float(recent_raw.get(n) or 0.0) / 100.0 for n in names} if sum(vals.values()) > 0: return { "shares": _normalized(vals, names), "informative": int(temporal.get("recent_votes") or 0) >= 100, "source": "recent-return statewide path", "effective_sample_votes": int(temporal.get("recent_votes") or 0), "geography_level": "statewide path", } return { "shares": _normalized(statewide, names), "informative": False, "source": "non-informative statewide fallback", "effective_sample_votes": 0, "geography_level": "fallback", } def _component_local_expectation( model_key: str, model_result: Dict[str, Any], prior_snapshot: Dict[str, Any], names: List[str], changed: List[Dict[str, Any]], market_mapping: Optional[Dict[str, Any]] = None, ) -> Dict[str, float]: """Compatibility wrapper used by the Path Referee.""" statewide = result_shares(model_result) expected_votes = {n: 0.0 for n in names} total = 0.0 for item in changed: volume = float(item.get("new_votes") or 0.0) if volume <= 0: continue pred = _component_local_prediction( model_key, model_result, prior_snapshot, names, item, market_mapping ) shares = pred["shares"] for n in names: expected_votes[n] += volume * shares.get(n, 0.0) total += volume return _normalized(expected_votes, names) if total > 0 else _normalized(statewide, names) def _actual_increment( earlier: Dict[str, Any], later: Dict[str, Any], names: List[str], ) -> Dict[str, Any]: early = {_region_identity(r): r for r in county_regions(earlier)} late = {_region_identity(r): r for r in county_regions(later)} changed = [] actual_votes = {n: 0 for n in names} correction_votes = 0 for cid in set(early) | set(late): before = _candidate_votes_in_region(early.get(cid), names) if early.get(cid) else {n: 0 for n in names} after = _candidate_votes_in_region(late.get(cid), names) if late.get(cid) else {n: 0 for n in names} positive = {} raw_total = 0 for n in names: delta = int(after.get(n, 0)) - int(before.get(n, 0)) if delta < 0: correction_votes += -delta positive[n] = max(0, delta) actual_votes[n] += positive[n] raw_total += positive[n] if raw_total > 0: region = late.get(cid) or early.get(cid) or {} changed.append({ "cid": cid, "county": region.get("name") or cid, "new_votes": raw_total, "actual_votes": positive, }) total = sum(actual_votes.values()) return { "changed": changed, "actual_votes": actual_votes, "actual_shares": _normalized(actual_votes, names) if total > 0 else {n: 0.0 for n in names}, "new_votes": total, "correction_votes": correction_votes, } def rolling_component_referee( base_url: str, snapshot_name: str, reference_url: Optional[str] = None, market_map_source: Optional[str] = None, max_intervals: int = 4, ) -> Dict[str, Any]: """ V5 short-horizon Path Referee. Each model is graded only on geography for which it had a genuine prediction. This score is for reporting-path simulation and does not directly reweight the final-result ensemble. """ base_url = slug_base(base_url) cache_key = ( base_url, snapshot_name, normalize_reference_url(reference_url) or "", market_map_source or "", int(max_intervals), ) if cache_key in _PATH_REFEREE_CACHE: return _PATH_REFEREE_CACHE[cache_key] history = snapshot_history_window(base_url, snapshot_name, max_intervals=max_intervals) if len(history) < 2: return {"available": False, "intervals": [], "models": {}, "evidence": 0.0} current = history[-1]["data"] names = list(candidate_map(current).keys()) current_votes = sum(int(c.get("votes") or 0) for c in candidate_map(current).values()) state = current.get("province") resolved = discover_media_market_source(state, market_map_source) mapping = load_media_market_map(resolved, state) if resolved else None interval_rows = [] model_obs: Dict[str, List[Dict[str, Any]]] = {} recencies = _temporal_recency_weights(len(history) - 1) if len(recencies) != len(history) - 1: raw = [0.16, 0.20, 0.27, 0.37][-(len(history)-1):] s = sum(raw) recencies = [x/s for x in raw] for i in range(len(history) - 1): before_meta, after_meta = history[i], history[i + 1] before, after = before_meta["data"], after_meta["data"] actual = _actual_increment(before, after, names) if actual["new_votes"] <= 0: continue bundle = component_snapshot_bundle( base_url, str(before_meta["filename"]), reference_url, resolved, ) predictions = {} for key, model in bundle["models"].items(): exp_votes = {n: 0.0 for n in names} actual_votes = {n: 0.0 for n in names} informative_votes = 0.0 informative_counties = 0 for item in actual["changed"]: pred = _component_local_prediction( key, model, before, names, item, mapping ) if not pred.get("informative"): continue volume = float(item.get("new_votes") or 0.0) if volume <= 0: continue informative_votes += volume informative_counties += 1 for n in names: exp_votes[n] += volume * pred["shares"].get(n, 0.0) actual_votes[n] += float((item.get("actual_votes") or {}).get(n, 0.0)) if informative_votes <= 0: predictions[key] = { "available": False, "informative_vote_coverage": 0.0, } continue expected = _normalized(exp_votes, names) realized = _normalized(actual_votes, names) mae = sum( abs(expected.get(n, 0.0) - realized.get(n, 0.0)) for n in names ) / max(1, len(names)) coverage = informative_votes / max(1.0, actual["new_votes"]) predictions[key] = { "available": True, "expected_shares": {n: 100.0 * expected.get(n, 0.0) for n in names}, "actual_informative_shares": {n: 100.0 * realized.get(n, 0.0) for n in names}, "mae_points": 100.0 * mae, "informative_votes": informative_votes, "informative_counties": informative_counties, "informative_vote_coverage": coverage, } model_obs.setdefault(key, []).append({ "mae": mae, "new_votes": informative_votes, "total_votes": actual["new_votes"], "coverage": coverage, "recency": recencies[i] if i < len(recencies) else 1.0, "changed_counties": informative_counties, }) interval_rows.append({ "from_snapshot": before_meta.get("snapshot"), "from_filename": before_meta.get("filename"), "to_snapshot": after_meta.get("snapshot"), "to_filename": after_meta.get("filename"), "new_votes": actual["new_votes"], "changed_counties": len(actual["changed"]), "correction_votes": actual["correction_votes"], "actual_shares": {n: 100.0 * actual["actual_shares"].get(n, 0.0) for n in names}, "predictions": predictions, }) total_recent_votes = sum(row["new_votes"] for row in interval_rows) unique_intervals = len(interval_rows) total_changed = sum(row["changed_counties"] for row in interval_rows) volume_evidence = min( 1.0, math.sqrt(total_recent_votes / max(1.0, 0.18 * current_votes)), ) if current_votes > 0 else 0.0 interval_evidence = min(1.0, math.sqrt(unique_intervals / 3.0)) geography_evidence = min(1.0, math.sqrt(total_changed / 28.0)) base_evidence = max( 0.0, min( 1.0, volume_evidence * interval_evidence * (0.55 + 0.45 * geography_evidence), ), ) models = {} coverages = [] for key, obs in model_obs.items(): if not obs: continue max_volume = max(x["new_votes"] for x in obs) num = den = 0.0 covered = possible = 0.0 for x in obs: volume_weight = math.sqrt(x["new_votes"] / max(1.0, max_volume)) w = x["recency"] * volume_weight num += w * x["mae"] den += w covered += x["new_votes"] possible += x["total_votes"] mae = num / den if den else 0.10 coverage = min(1.0, covered / max(1.0, possible)) coverages.append(coverage) model_evidence = base_evidence * math.sqrt(coverage) * min(1.0, math.sqrt(len(obs) / 3.0)) raw_score = math.exp(-mae / 0.055) usable_score = 0.55 + model_evidence * (raw_score - 0.55) models[key] = { "mae_points": 100.0 * mae, "raw_score": max(0.0, min(1.0, raw_score)), "usable_score": max(0.0, min(1.0, usable_score)), "intervals_scored": len(obs), "informative_vote_coverage": coverage, "evidence": model_evidence, } evidence = base_evidence * math.sqrt( sum(coverages) / len(coverages) if coverages else 0.0 ) if models: center = sum(v["usable_score"] for v in models.values()) / len(models) for key, row in models.items(): multiplier = math.exp( 1.75 * float(row.get("evidence") or 0.0) * (row["usable_score"] - center) ) row["strength_multiplier"] = max(0.58, min(1.65, multiplier)) best = max(models, key=lambda k: models[k]["usable_score"]) if models else None result = { "available": bool(models), "max_intervals": max_intervals, "intervals": interval_rows, "recent_votes": total_recent_votes, "current_votes": current_votes, "base_evidence_before_prediction_coverage": base_evidence, "evidence": max(0.0, min(1.0, evidence)), "models": models, "best_recent_predictor": best, "method_note": ( "V6 Path Referee scores only genuine ex-ante predictions on the exact " "geography that reported next. This score drives path simulation, not " "the final-result mixture." ), } _PATH_REFEREE_CACHE[cache_key] = result return result def _material_county_maturation( earlier: Dict[str, Any], later: Dict[str, Any], names: List[str], ) -> Dict[str, Any]: """ Identify counties that materially matured between two already-observed snapshots. This is deliberately a longer-horizon validation target than one next dump. """ early = {_region_identity(r): r for r in county_regions(earlier)} late = {_region_identity(r): r for r in county_regions(later)} rows = [] total_new = 0 total_corrections = 0 for cid in set(early) | set(late): before_region = early.get(cid) after_region = late.get(cid) before_votes = ( _candidate_votes_in_region(before_region, names) if before_region else {n: 0 for n in names} ) after_votes = ( _candidate_votes_in_region(after_region, names) if after_region else {n: 0 for n in names} ) before_total = sum(before_votes.values()) positive = {} new_votes = 0 corrections = 0 for name in names: delta = int(after_votes.get(name, 0)) - int(before_votes.get(name, 0)) if delta < 0: corrections += -delta positive[name] = max(0, delta) new_votes += positive[name] if new_votes <= 0: total_corrections += corrections continue try: p_before = min( 1.0, max( 0.0, float((before_region or {}).get("percent_reporting") or 0.0) / 100.0, ), ) except (TypeError, ValueError): p_before = 0.0 try: p_after = min( 1.0, max( 0.0, float((after_region or {}).get("percent_reporting") or 0.0) / 100.0, ), ) except (TypeError, ValueError): p_after = 0.0 reporting_advance = max(0.0, p_after - p_before) relative_growth = new_votes / max(500.0, float(before_total)) material = ( new_votes >= 2500 or reporting_advance >= 0.10 or (new_votes >= 500 and relative_growth >= 0.18) ) if not material: total_corrections += corrections continue actual_shares = _normalized(positive, names) materiality = max( min(1.0, reporting_advance / 0.18), min(1.0, math.sqrt(new_votes / 5000.0)), min(1.0, relative_growth / 0.40), ) region = after_region or before_region or {} rows.append({ "cid": cid, "county": region.get("name") or cid, "new_votes": new_votes, "before_votes": before_total, "reporting_before": p_before, "reporting_after": p_after, "reporting_advance": reporting_advance, "relative_growth": relative_growth, "materiality": materiality, "actual_votes": positive, "actual_shares": actual_shares, }) total_new += new_votes total_corrections += corrections return { "counties": rows, "new_votes": total_new, "correction_votes": total_corrections, } def completion_component_referee( base_url: str, snapshot_name: str, reference_url: Optional[str] = None, market_map_source: Optional[str] = None, max_windows: int = 4, ) -> Dict[str, Any]: """ V5 final-completion referee. Models are scored only where they had a genuine ex-ante prediction. Equal-share/statewide fallbacks created from missing local information do not count as validation evidence. """ base_url = slug_base(base_url) cache_key = ( base_url, snapshot_name, normalize_reference_url(reference_url) or "", market_map_source or "", int(max_windows), ) if cache_key in _COMPLETION_REFEREE_CACHE: return _COMPLETION_REFEREE_CACHE[cache_key] metadata = fetch_json(base_url + "snapshot_metadata.json") items = [item for item in (metadata.get("snapshots") or []) if item.get("filename")] def order_key(item): try: return (0, int(item.get("snapshot"))) except (TypeError, ValueError): return (1, str(item.get("timestamp") or ""), str(item.get("filename") or "")) items.sort(key=order_key) current_idx = next( (i for i, item in enumerate(items) if item.get("filename") == snapshot_name), None, ) if current_idx is None or current_idx < 2: return { "available": False, "windows": [], "models": {}, "evidence": 0.0, "method_note": "Not enough already-observed history for county-maturation validation.", } current = fetch_json(base_url + snapshot_name) names = list(candidate_map(current).keys()) current_votes = sum(int(c.get("votes") or 0) for c in candidate_map(current).values()) state = current.get("province") resolved = discover_media_market_source(state, market_map_source) mapping = load_media_market_map(resolved, state) if resolved else None span = max(2, min(10, int(round(current_idx / max(2, max_windows + 1))))) boundaries = [current_idx] while len(boundaries) <= max_windows and boundaries[-1] - span >= 0: boundaries.append(boundaries[-1] - span) boundaries = sorted(set(boundaries)) pair_count = max(0, len(boundaries) - 1) if pair_count <= 0: return { "available": False, "windows": [], "models": {}, "evidence": 0.0, "method_note": "Not enough already-observed history for county-maturation validation.", } recencies = [i + 1 for i in range(pair_count)] rsum = sum(recencies) recencies = [x / rsum for x in recencies] windows = [] model_obs: Dict[str, List[Dict[str, Any]]] = {} all_counties = set() total_matured_votes = 0 weighted_advance_num = 0.0 weighted_advance_den = 0.0 for pair_idx in range(pair_count): before_idx = boundaries[pair_idx] after_idx = boundaries[pair_idx + 1] before_meta, after_meta = items[before_idx], items[after_idx] before = fetch_json(base_url + str(before_meta["filename"])) after = fetch_json(base_url + str(after_meta["filename"])) matured = _material_county_maturation(before, after, names) county_rows = matured["counties"] if not county_rows: continue bundle = component_snapshot_bundle( base_url, str(before_meta["filename"]), reference_url, resolved, ) eligible_models = { key: model for key, model in bundle["models"].items() if key != "temporal" } prediction_rows = {} for key, model in eligible_models.items(): err_num = 0.0 err_den = 0.0 informative_votes = 0.0 county_errors = [] skipped = 0 for county in county_rows: pred = _component_local_prediction( key, model, before, names, county, mapping ) if not pred.get("informative"): skipped += 1 continue expected = pred["shares"] actual = county["actual_shares"] mae = sum( abs(expected.get(name, 0.0) - actual.get(name, 0.0)) for name in names ) / max(1, len(names)) county_weight = ( math.sqrt(max(1.0, county["new_votes"])) * (0.65 + 0.35 * float(county["materiality"])) ) err_num += county_weight * mae err_den += county_weight informative_votes += county["new_votes"] county_errors.append({ "cid": county["cid"], "county": county["county"], "new_votes": county["new_votes"], "reporting_advance_points": 100.0 * county["reporting_advance"], "source": pred.get("source"), "geography_level": pred.get("geography_level"), "effective_sample_votes": pred.get("effective_sample_votes"), "expected_shares": {n: 100.0 * expected.get(n, 0.0) for n in names}, "actual_shares": {n: 100.0 * actual.get(n, 0.0) for n in names}, "mae_points": 100.0 * mae, }) if err_den <= 0: prediction_rows[key] = { "available": False, "counties_scored": 0, "counties_skipped_noninformative": skipped, "informative_vote_coverage": 0.0, } continue window_mae = err_num / err_den coverage = informative_votes / max(1.0, matured["new_votes"]) prediction_rows[key] = { "available": True, "mae_points": 100.0 * window_mae, "counties_scored": len(county_errors), "counties_skipped_noninformative": skipped, "informative_votes": informative_votes, "informative_vote_coverage": coverage, "county_examples": sorted( county_errors, key=lambda row: row["new_votes"], reverse=True, )[:12], } model_obs.setdefault(key, []).append({ "mae": window_mae, "new_votes": informative_votes, "window_total_votes": matured["new_votes"], "coverage": coverage, "recency": recencies[pair_idx], "counties": len(county_errors), }) total_matured_votes += matured["new_votes"] for county in county_rows: all_counties.add(county["cid"]) weighted_advance_num += county["new_votes"] * county["reporting_advance"] weighted_advance_den += county["new_votes"] windows.append({ "from_snapshot": before_meta.get("snapshot"), "from_filename": before_meta.get("filename"), "to_snapshot": after_meta.get("snapshot"), "to_filename": after_meta.get("filename"), "span_snapshots": after_idx - before_idx, "matured_votes": matured["new_votes"], "matured_counties": len(county_rows), "correction_votes": matured["correction_votes"], "predictions": prediction_rows, }) if not windows or not model_obs: return { "available": False, "windows": windows, "models": {}, "evidence": 0.0, "method_note": "No genuine model predictions were available over materially matured county windows.", } avg_reporting_advance = ( weighted_advance_num / weighted_advance_den if weighted_advance_den > 0 else 0.0 ) volume_evidence = min( 1.0, math.sqrt(total_matured_votes / max(1.0, 0.35 * current_votes)), ) if current_votes > 0 else 0.0 window_evidence = min(1.0, math.sqrt(len(windows) / 3.0)) geography_evidence = min(1.0, math.sqrt(len(all_counties) / 30.0)) maturation_evidence = min(1.0, avg_reporting_advance / 0.18) base_evidence = ( volume_evidence * window_evidence * (0.45 + 0.30 * geography_evidence + 0.25 * maturation_evidence) ) base_evidence = max(0.0, min(1.0, base_evidence)) models = {} coverage_values = [] for key, obs in model_obs.items(): if not obs: continue max_volume = max(row["new_votes"] for row in obs) num = 0.0 den = 0.0 covered = 0.0 possible = 0.0 for row in obs: volume_weight = math.sqrt(row["new_votes"] / max(1.0, max_volume)) geography_weight = min(1.0, math.sqrt(row["counties"] / 10.0)) weight = row["recency"] * volume_weight * (0.70 + 0.30 * geography_weight) num += weight * row["mae"] den += weight covered += row["new_votes"] possible += row["window_total_votes"] mae = num / den if den else 0.10 coverage = min(1.0, covered / max(1.0, possible)) coverage_values.append(coverage) model_evidence = ( base_evidence * math.sqrt(coverage) * min(1.0, math.sqrt(len(obs) / 3.0)) ) raw_score = math.exp(-mae / 0.045) usable_score = 0.55 + model_evidence * (raw_score - 0.55) models[key] = { "mae_points": 100.0 * mae, "raw_score": max(0.0, min(1.0, raw_score)), "usable_score": max(0.0, min(1.0, usable_score)), "windows_scored": len(obs), "informative_vote_coverage": coverage, "evidence": model_evidence, } avg_coverage = sum(coverage_values) / len(coverage_values) if coverage_values else 0.0 evidence = max(0.0, min(1.0, base_evidence * math.sqrt(avg_coverage))) if models: center = sum(row["usable_score"] for row in models.values()) / len(models) for key, row in models.items(): local_evidence = float(row.get("evidence") or 0.0) multiplier = math.exp( 2.75 * local_evidence * (row["usable_score"] - center) ) row["strength_multiplier"] = max(0.58, min(1.85, multiplier)) best = max(models, key=lambda key: models[key]["usable_score"]) if models else None result = { "available": bool(models), "span_snapshots": span, "windows": windows, "matured_votes": total_matured_votes, "unique_matured_counties": len(all_counties), "average_reporting_advance_points": 100.0 * avg_reporting_advance, "base_evidence_before_prediction_coverage": base_evidence, "average_informative_vote_coverage": avg_coverage, "evidence": evidence, "models": models, "best_completion_predictor": best, "method_note": ( "V6 scores only genuine ex-ante predictions. Missing county information " "does not silently become an equal-share prediction. Evidence is reduced " "by the fraction of matured vote for which each model was actually informative." ), } _COMPLETION_REFEREE_CACHE[cache_key] = result return result def _county_reporting_fraction(snapshot: Dict[str, Any], cid: str) -> float: for region in county_regions(snapshot): if _region_identity(region) == cid: try: return min(1.0, max(0.0, float(region.get("percent_reporting") or 0.0) / 100.0)) except (TypeError, ValueError): return 0.0 return 0.0 def raw_board_stability( base_url: str, snapshot_name: str, max_intervals: int = 4, ) -> Dict[str, Any]: """Backward-looking scoreboard stability diagnostic.""" history = snapshot_history_window(slug_base(base_url), snapshot_name, max_intervals=max_intervals) if len(history) < 2: return {"available": False, "score": 0.35} current = history[-1]["data"] names = list(candidate_map(current).keys()) current_shares = shares_from_votes(candidate_map(current)) ranked = sorted(names, key=lambda n: current_shares.get(n, 0.0), reverse=True) if len(ranked) < 2: return {"available": False, "score": 0.35} ref = ranked[0] margins = [] signs = [] for h in history: cands = candidate_map(h["data"]) shares = shares_from_votes(cands) opp = max((shares.get(n, 0.0) for n in names if n != ref), default=0.0) margin = shares.get(ref, 0.0) - opp margins.append(margin) signs.append(1 if margin >= 0 else -1) mean = sum(margins) / len(margins) variance = sum((x - mean) ** 2 for x in margins) / max(1, len(margins) - 1) dispersion = math.sqrt(max(0.0, variance)) sign_consistency = abs(sum(signs)) / len(signs) score = math.exp(-dispersion / 0.035) * (0.55 + 0.45 * sign_consistency) return { "available": True, "reference_candidate": ref, "score": max(0.0, min(1.0, score)), "dispersion_points": 100.0 * dispersion, "sign_consistency": sign_consistency, "margins_points": [100.0 * x for x in margins], } def v6_decision_maturity( snapshot: Dict[str, Any], v3_result: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ V6.2 data-maturity governor. VoteFlow's source percent_reporting is not assumed to equal final vote fraction. Prefer V3's expected-final-turnout estimate when available. """ current_votes = sum( int(c.get("votes") or 0) for c in candidate_map(snapshot).values() ) source_maturity = statewide_reporting_maturity(snapshot) # VoteFlow's top-level percent_reporting is NOT assumed to be an exact # statewide vote-counted percentage. It is, however, a valuable emergency # sanity signal: a race whose provider says only ~2% is reporting must not # become "100% counted" merely because the handful of municipalities that # have reported are individually marked 100% complete. try: source_reporting_fraction = min( 1.0, max( 0.0, float(snapshot.get("percent_reporting") or 0.0) / 100.0, ), ) except (TypeError, ValueError): source_reporting_fraction = 0.0 estimated_final = 0.0 if v3_result: v3d = v3_result.get("v3") or {} try: estimated_final = float( v3d.get("estimated_final_votes") or 0.0 ) except (TypeError, ValueError): estimated_final = 0.0 # A turnout projection must leave actual room for outstanding votes before # we trust it as the maturity source. Equality to current_votes is the # classic broken/no-reference condition and previously yielded 100%. if estimated_final > max(1.0, current_votes * 1.001): counted_fraction = max( 0.0, min(1.0, current_votes / estimated_final), ) source = "v3_expected_final_turnout" elif source_reporting_fraction > 0.0: # Conservative fallback. Regional maturity can be 100% when every # currently-reporting municipality is locally complete, so cap it by # the statewide provider reporting signal rather than treating those # few towns as the entire electorate. counted_fraction = min( max(0.0, source_maturity), source_reporting_fraction, ) source = "source_reporting_conservative_fallback" else: # If the provider has no statewide reporting value at all, do not allow # an early collection of completed municipalities to promote the model # straight to mature-data behavior. counted_fraction = min( max(0.0, source_maturity), 0.10, ) source = "regional_reporting_capped_fallback" if counted_fraction < 0.15: activation = 0.0 stage = "early" elif counted_fraction < 0.30: activation = ( 0.35 * (counted_fraction - 0.15) / 0.15 ) stage = "exploratory" elif counted_fraction < 0.50: activation = ( 0.35 + 0.65 * (counted_fraction - 0.30) / 0.20 ) stage = "serious" else: activation = 1.0 stage = "mature" return { "counted_fraction_estimate": counted_fraction, "counted_percent_estimate": 100.0 * counted_fraction, "activation": max(0.0, min(1.0, activation)), "stage": stage, "source": source, "serious_data": counted_fraction >= 0.30, "mature_data": counted_fraction >= 0.50, "policy_note": ( "V6.2.2 treats roughly 30% counted as the start of serious-data " "behavior and 50%+ as full-strength final-completion behavior." ), } def specialist_temporal_validation( base_url: str, snapshot_name: str, reference_url: Optional[str] = None, market_map_source: Optional[str] = None, max_intervals: int = 3, ) -> Dict[str, Any]: """ Second validation axis for specialist models. Spatial CV asks whether a model generalizes across counties. Temporal CV asks whether its projected final state survives additional reporting. Large projection drift and repeated leader flips reduce final trust. """ base_url = slug_base(base_url) cache_key = ( base_url, snapshot_name, normalize_reference_url(reference_url) or "", market_map_source or "", int(max_intervals), ) if cache_key in _SPECIALIST_TEMPORAL_CACHE: return _SPECIALIST_TEMPORAL_CACHE[cache_key] history = snapshot_history_window( base_url, snapshot_name, max_intervals=max_intervals, ) if len(history) < 2: result = { "available": False, "models": {}, "evidence": 0.0, } _SPECIALIST_TEMPORAL_CACHE[cache_key] = result return result specialist_keys = { "histgeo", "latent", "batchbias", "runoff_transfer", "precinct", } obs = {} current_votes = sum( int(c.get("votes") or 0) for c in candidate_map(history[-1]["data"]).values() ) for idx in range(len(history) - 1): before_meta = history[idx] after_meta = history[idx + 1] before_bundle = component_snapshot_bundle( base_url, str(before_meta["filename"]), reference_url, market_map_source, ) after_bundle = component_snapshot_bundle( base_url, str(after_meta["filename"]), reference_url, market_map_source, ) before_votes = sum( int(c.get("votes") or 0) for c in candidate_map(before_bundle["snapshot"]).values() ) after_votes = sum( int(c.get("votes") or 0) for c in candidate_map(after_bundle["snapshot"]).values() ) new_votes = max(0, after_votes - before_votes) if new_votes <= 0: continue names = list(candidate_map(after_bundle["snapshot"]).keys()) recency = (idx + 1) / max(1, len(history) - 1) for key in specialist_keys: before_model = before_bundle["models"].get(key) after_model = after_bundle["models"].get(key) if not before_model or not after_model: continue a = result_shares(before_model) b = result_shares(after_model) if not a or not b: continue drift = sum( abs(a.get(n, 0.0) - b.get(n, 0.0)) for n in names ) / max(1, len(names)) a_rank = sorted( names, key=lambda n: a.get(n, 0.0), reverse=True, ) b_rank = sorted( names, key=lambda n: b.get(n, 0.0), reverse=True, ) retained = bool( a_rank and b_rank and a_rank[0] == b_rank[0] ) weight = ( math.sqrt(max(1.0, new_votes)) * recency ) obs.setdefault(key, []).append({ "drift": drift, "leader_retained": retained, "weight": weight, "new_votes": new_votes, "from_snapshot": before_meta.get("snapshot"), "to_snapshot": after_meta.get("snapshot"), }) models = {} evidence_values = [] for key, rows in obs.items(): den = sum(float(r["weight"]) for r in rows) if den <= 0: continue drift = sum( float(r["weight"]) * float(r["drift"]) for r in rows ) / den retention = sum( float(r["weight"]) * (1.0 if r["leader_retained"] else 0.0) for r in rows ) / den votes = sum(int(r["new_votes"]) for r in rows) evidence = ( min(1.0, math.sqrt(len(rows) / 3.0)) * min( 1.0, math.sqrt( votes / max(1.0, 0.18 * current_votes) ), ) ) stability_score = math.exp(-drift / 0.030) score = ( stability_score * (0.55 + 0.45 * retention) ) usable = 0.55 + evidence * (score - 0.55) models[key] = { "available": True, "intervals": len(rows), "projection_drift_mae_points": 100.0 * drift, "leader_retention_rate": retention, "raw_score": max(0.0, min(1.0, score)), "usable_score": max(0.0, min(1.0, usable)), "evidence": evidence, "rows": rows, "exam": "final-projection survival across later snapshots", } evidence_values.append(evidence) result = { "available": bool(models), "models": models, "evidence": ( sum(evidence_values) / len(evidence_values) if evidence_values else 0.0 ), "method_note": ( "Temporal specialist validation measures whether a specialist's " "projected final shares and leader survive additional reporting. " "It is combined with spatial/held-out validation rather than " "replacing it." ), } _SPECIALIST_TEMPORAL_CACHE[cache_key] = result return result def build_horizon_validation( model_results: Dict[str, Dict[str, Any]], completion_referee: Dict[str, Any], path_referee: Dict[str, Any], specialist_temporal: Optional[Dict[str, Any]] = None, maturity_fraction: float = 0.0, ) -> Dict[str, Any]: """ Short path, county maturation, spatial specialist CV, and temporal specialist-survival validation. """ out = {} completion_models = completion_referee.get("models") or {} path_models = path_referee.get("models") or {} temporal_models = ( (specialist_temporal or {}).get("models") or {} ) maturity_fraction = max( 0.0, min(1.0, float(maturity_fraction or 0.0)) ) for key, model in model_results.items(): if key == "temporal": continue c = completion_models.get(key) or {} p = path_models.get(key) or {} internal = _model_internal_validation(key, model) temporal_exam = temporal_models.get(key) or { "available": False, "usable_score": 0.55, "raw_score": 0.55, "evidence": 0.0, "projection_drift_mae_points": None, "leader_retention_rate": None, "exam": None, } completion_quality = float( c.get("usable_score") or 0.55 ) completion_evidence = float( c.get("evidence") or 0.0 ) path_quality = float( p.get("usable_score") or 0.55 ) path_evidence = float( p.get("evidence") or 0.0 ) specialist_joint_score = None specialist_joint_evidence = 0.0 if internal.get("available"): ie = float(internal.get("evidence") or 0.0) iq = float(internal.get("score") or 0.0) spatial_usable = 0.55 + ie * (iq - 0.55) te = float(temporal_exam.get("evidence") or 0.0) tq = float( temporal_exam.get("usable_score") or 0.55 ) # Once serious data exists, a specialist that has never survived # a temporal exam should not receive full specialist credit. if ( key != "benchmark_completion" and maturity_fraction >= 0.30 and te < 0.10 ): tq = min(tq, 0.46) if ( key != "benchmark_completion" and maturity_fraction >= 0.50 and te < 0.10 ): tq = min(tq, 0.40) specialist_joint_score = math.sqrt( max(0.02, spatial_usable) * max(0.02, tq) ) specialist_joint_evidence = ( 1.0 - (1.0 - ie) * (1.0 - te) ) final_quality = ( (0.52 + 0.18 * completion_evidence) * completion_quality + (0.48 - 0.18 * completion_evidence) * specialist_joint_score ) final_evidence = ( 1.0 - (1.0 - completion_evidence) * (1.0 - specialist_joint_evidence) ) # Harder penalty after 30/50% if temporal survival is poor. temporal_raw = float( temporal_exam.get("usable_score") or 0.55 ) if ( key != "benchmark_completion" and maturity_fraction >= 0.30 and temporal_raw < 0.42 ): final_quality *= 0.86 if ( key != "benchmark_completion" and maturity_fraction >= 0.50 and temporal_raw < 0.38 ): final_quality *= 0.78 else: final_quality = completion_quality final_evidence = completion_evidence mae_values = [] if c.get("mae_points") is not None: mae_values.append(( float(c["mae_points"]), max(0.10, completion_evidence), )) if internal.get("mae_points") is not None: mae_values.append(( float(internal["mae_points"]), max( 0.10, float(internal.get("evidence") or 0.0), ), )) if temporal_exam.get( "projection_drift_mae_points" ) is not None: mae_values.append(( float( temporal_exam[ "projection_drift_mae_points" ] ), max( 0.10, float( temporal_exam.get("evidence") or 0.0 ), ), )) residual_mae = _weighted_mean( mae_values, 6.0 ) out[key] = { "short_path_quality": path_quality, "short_path_evidence": path_evidence, "maturation_quality": completion_quality, "maturation_evidence": completion_evidence, "specialist_exam": internal, "temporal_survival_exam": temporal_exam, "specialist_joint_score": specialist_joint_score, "specialist_joint_evidence": specialist_joint_evidence, "final_quality": max( 0.0, min(1.0, final_quality) ), "final_evidence": max( 0.0, min(1.0, final_evidence) ), "residual_mae_points": residual_mae, } return out def _apply_weight_caps( weights: Dict[str, float], caps: Dict[str, float], ) -> Dict[str, float]: weights = {k: max(0.0, float(v)) for k, v in weights.items()} for _ in range(8): excess = 0.0 free = [] for key, value in list(weights.items()): cap = caps.get(key, 1.0) if value > cap: excess += value - cap weights[key] = cap elif value < cap - 1e-12: free.append(key) if excess <= 1e-12 or not free: break free_total = sum(weights[k] for k in free) if free_total <= 0: add = excess / len(free) for key in free: weights[key] += add else: for key in free: weights[key] += excess * weights[key] / free_total total = sum(weights.values()) return {k: v / total for k, v in weights.items()} if total > 0 else {"v0": 1.0} def _completion_trust_memory( completion_referee: Dict[str, Any], model_key: str, ) -> Dict[str, Any]: """ Hysteresis/trust memory from multiple completed maturation windows. One unusual batch cannot instantly erase or create model trust. """ scores = [] for window in completion_referee.get("windows") or []: pred = (window.get("predictions") or {}).get(model_key) or {} if not pred.get("available") or pred.get("mae_points") is None: continue mae = float(pred["mae_points"]) / 100.0 coverage = float(pred.get("informative_vote_coverage") or 0.0) score = math.exp(-mae / 0.050) scores.append((score, coverage)) if not scores: return {"available": False, "score": 0.55, "windows": 0} # Oldest->newest EWMA, deliberately sticky. memory = 0.55 for score, coverage in scores: alpha = 0.22 + 0.28 * min(1.0, coverage) memory = (1.0 - alpha) * memory + alpha * score return { "available": True, "score": max(0.0, min(1.0, memory)), "windows": len(scores), } def v5_mixture_gate( model_results: Dict[str, Dict[str, Any]], structural_strengths: Dict[str, float], completion_referee: Dict[str, Any], path_referee: Dict[str, Any], specialist_temporal: Optional[Dict[str, Any]] = None, maturity_fraction: float = 0.0, ) -> Dict[str, Any]: """ Hierarchical mixture-of-experts gate. Generic completion performance and specialist-specific held-out exams both affect trust. Path accuracy is recorded separately and is not used as a substitute for final-result skill. """ horizon = build_horizon_validation( model_results, completion_referee, path_referee, specialist_temporal=specialist_temporal, maturity_fraction=maturity_fraction, ) logits = {} rows = {} specialist_keys = { "runoff_transfer", "histgeo", "latent", "batchbias", "precinct", "benchmark_completion" } for key, base_strength in structural_strengths.items(): if key == "temporal" or key not in model_results: continue base_strength = max(1e-5, float(base_strength)) hv = horizon.get(key) or {} quality = float(hv.get("final_quality") or 0.55) evidence = float(hv.get("final_evidence") or 0.0) trust_memory = _completion_trust_memory(completion_referee, key) if trust_memory.get("available"): memory_score = float(trust_memory.get("score") or 0.55) # 30% sticky memory limits whiplash from one newly completed window. quality = 0.70 * quality + 0.30 * memory_score completion_multiplier = float( ((completion_referee.get("models") or {}).get(key) or {}).get("strength_multiplier") or 1.0 ) # Convert FINAL/JOINED quality into a centered trust adjustment. # V6.2 deliberately does NOT separately reward a specialist's spatial # held-out exam. That was the loophole that let Oklahoma Runoff Transfer # look excellent cross-sectionally while remaining wrong about the final. validation_log = 3.0 * evidence * (quality - 0.55) if key in specialist_keys: joint_score = float( hv.get("specialist_joint_score") if hv.get("specialist_joint_score") is not None else quality ) joint_evidence = float( hv.get("specialist_joint_evidence") if hv.get("specialist_joint_evidence") is not None else evidence ) validation_log += ( 2.15 * joint_evidence * (joint_score - 0.50) ) # Generic completion still matters materially for specialists. completion_multiplier = completion_multiplier ** 0.72 uncertainty = float(hv.get("residual_mae_points") or 6.0) uncertainty_penalty = max(0.0, uncertainty - 4.0) / 20.0 internal_available = bool( (hv.get("specialist_exam") or {}).get("available") ) # V6 no longer lets an ungraded structural model quietly carry 20-25% # of the Needle. It can recover that weight once a real exam exists. if evidence < 0.12 and not internal_available: unvalidated_penalty = 0.52 if key == "market" else 0.28 else: unvalidated_penalty = 0.0 logit = ( math.log(base_strength) + math.log(max(0.40, min(2.20, completion_multiplier))) + validation_log - uncertainty_penalty - unvalidated_penalty ) logits[key] = logit rows[key] = { "structural_strength": base_strength, "completion_multiplier": completion_multiplier, "validation_log_adjustment": validation_log, "uncertainty_penalty": uncertainty_penalty, "unvalidated_penalty": unvalidated_penalty, "horizon_validation": hv, "trust_memory": trust_memory, } if not logits: return { "weights": {"v0": 1.0}, "experts": rows, "horizon_validation": horizon, } # Slightly cool the softmax so one expert can earn a meaningful edge # without instantly taking 90% of the model. temperature = 0.88 max_logit = max(logits.values()) raw = { key: math.exp((value - max_logit) / temperature) for key, value in logits.items() } total = sum(raw.values()) weights = {key: raw[key] / total for key in raw} caps = { "v0": 0.45, "v1": 0.42, "v2r": 0.34, "v3": 0.44, "market": 0.26, "histgeo": 0.27, "latent": 0.26, "batchbias": 0.20, "runoff_transfer": 0.58, "precinct": 0.38, "benchmark_completion": 0.32, } # Respect specialist-provided dynamic caps. rt = (model_results.get("runoff_transfer") or {}).get("runoff_transfer") or {} if rt: caps["runoff_transfer"] = min(caps["runoff_transfer"], float(rt.get("max_ensemble_influence") or caps["runoff_transfer"])) hg = (model_results.get("histgeo") or {}).get("historical_candidate_geo") or {} if hg: caps["histgeo"] = min(caps["histgeo"], float(hg.get("max_ensemble_influence") or caps["histgeo"])) latent = (model_results.get("latent") or {}).get("latent_coalition") or {} if latent: caps["latent"] = min(caps["latent"], float(latent.get("max_ensemble_influence") or caps["latent"])) bb = (model_results.get("batchbias") or {}).get("batch_bias") or {} if bb: caps["batchbias"] = min(caps["batchbias"], float(bb.get("max_ensemble_influence") or caps["batchbias"])) pr = (model_results.get("precinct") or {}).get("precinct_hierarchy") or {} if pr: caps["precinct"] = min(caps["precinct"], float(pr.get("max_ensemble_influence") or caps["precinct"])) bc = (model_results.get("benchmark_completion") or {}).get("benchmark_completion") or {} if bc: caps["benchmark_completion"] = min( caps["benchmark_completion"], float( bc.get("max_ensemble_influence") or caps["benchmark_completion"] ), ) # V6.2 weakest-link caps. Once serious data exists, a specialist cannot # carry large final weight unless its task/local skill also survives later # reporting. Missing/poor temporal survival becomes increasingly costly. maturity_fraction = max( 0.0, min(1.0, float(maturity_fraction or 0.0)) ) for key in specialist_keys: if key not in model_results: continue hv = horizon.get(key) or {} final_quality = float( hv.get("final_quality") or 0.0 ) final_evidence = float( hv.get("final_evidence") or 0.0 ) temporal_exam = hv.get( "temporal_survival_exam" ) or {} temporal_evidence = float( temporal_exam.get("evidence") or 0.0 ) temporal_score = float( temporal_exam.get("usable_score") or 0.55 ) base_cap = float(caps.get(key, 0.25)) if key == "benchmark_completion": # Its specialist exam already IS long-horizon maturation. if final_evidence < 0.20 or final_quality < 0.38: validation_cap = 0.07 elif final_quality < 0.50: validation_cap = 0.14 elif final_quality < 0.62: validation_cap = 0.24 else: validation_cap = base_cap else: if final_evidence < 0.20 or final_quality < 0.34: validation_cap = 0.08 elif final_quality < 0.42: validation_cap = 0.12 elif final_quality < 0.50: validation_cap = 0.18 elif final_quality < 0.58: validation_cap = 0.26 elif final_quality < 0.66: validation_cap = 0.36 else: validation_cap = base_cap if maturity_fraction >= 0.30 and temporal_evidence < 0.10: validation_cap = min(validation_cap, 0.12) if maturity_fraction >= 0.50 and temporal_evidence < 0.10: validation_cap = min(validation_cap, 0.08) if maturity_fraction >= 0.50 and temporal_score < 0.40: validation_cap = min(validation_cap, 0.12) # Before 30% these are exploratory specialists, not dominant answers. if maturity_fraction < 0.30: validation_cap = min(validation_cap, 0.18) caps[key] = min(base_cap, validation_cap) weights = _apply_weight_caps(weights, caps) for key in rows: rows[key]["weight"] = weights.get(key, 0.0) rows[key]["cap"] = caps.get(key, 1.0) return { "weights": weights, "experts": rows, "horizon_validation": horizon, "caps": caps, "temperature": temperature, "method_note": ( "V6.2.2 treats models as experts. Structural applicability, genuine " "completion validation, spatial specialist exams, temporal projection " "survival, uncertainty, and dynamic caps feed a softmax gate. Sticky " "multi-window trust memory adds hysteresis. Short-path accuracy remains " "separate from final-result authority." ), } def _v6_normalize_weights_for_models( weights: Dict[str, float], model_results: Dict[str, Dict[str, Any]], ) -> Dict[str, float]: filtered = { key: max(0.0, float(value)) for key, value in weights.items() if key in model_results and key != "temporal" and float(value) > 0 } total = sum(filtered.values()) return ( {key: value / total for key, value in filtered.items()} if total > 0 else {"v0": 1.0} ) def _v6_county_forecast_rows( snapshot: Dict[str, Any], names: List[str], model_results: Dict[str, Dict[str, Any]], weights: Dict[str, float], horizon_validation: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ Build one deterministic county completion table from the current expert mixture. This is the foundation for dynamic county benchmarks. """ horizon_validation = horizon_validation or {} current_candidates = candidate_map(snapshot) statewide_raw = shares_from_votes(current_candidates) counties_live = { _region_identity(region): region for region in county_regions(snapshot) } weights = _v6_normalize_weights_for_models(weights, model_results) local_maps = { key: _row_share_map(_model_county_rows(key, result), names) for key, result in model_results.items() if key != "temporal" } row_maps = {} for key, result in model_results.items(): rows = _model_county_rows(key, result) row_maps[key] = { str(row.get("cid") or row.get("fips") or ""): row for row in rows if row.get("cid") or row.get("fips") } all_ids = set(counties_live) for key in row_maps: all_ids.update(row_maps[key].keys()) rows = [] total_remaining = 0.0 projected_remaining_votes = {n: 0.0 for n in names} projected_current_county_votes = {n: 0.0 for n in names} for cid in all_ids: region = counties_live.get(cid) current_votes = ( _candidate_votes_in_region(region, names) if region else {n: 0 for n in names} ) current_total = sum(current_votes.values()) p = _safe_reporting_fraction(region) # Turnout/remaining-vote estimate is itself an ensemble. Models with # better final validation get more say in sizing the county. remaining_estimates = [] remaining_sources = [] for key, weight in weights.items(): row = row_maps.get(key, {}).get(cid) if not row or row.get("estimated_remaining_votes") is None: continue try: rem = max(0.0, float(row.get("estimated_remaining_votes") or 0.0)) except (TypeError, ValueError): continue quality = float( (horizon_validation.get(key) or {}).get( "final_quality", 0.55 ) ) source_weight = weight * (0.45 + 0.75 * quality) remaining_estimates.append((rem, source_weight)) remaining_sources.append({ "model": key, "remaining_votes": rem, "weight": source_weight, }) if remaining_estimates: remaining = _weighted_mean(remaining_estimates, 0.0) # A component can expose a zero-remaining placeholder even while # the source still reports an incomplete county. Never let that # suppress the basic live-turnout floor. live_turnout_floor = ( max( 0.0, current_total / max(0.05, p) - current_total, ) if current_total > 0 and 0.0 < p < 0.995 else 0.0 ) if live_turnout_floor > 0: remaining = max( remaining, 0.55 * live_turnout_floor, ) rem_mean = remaining rem_var = _weighted_mean( [ ((value - rem_mean) ** 2, weight) for value, weight in remaining_estimates ], 0.0, ) turnout_disagreement = math.sqrt(max(0.0, rem_var)) elif current_total > 0 and p > 0: remaining = max( 0.0, current_total / max(0.05, p) - current_total, ) turnout_disagreement = 0.50 * remaining else: remaining = 0.0 turnout_disagreement = 0.0 central = {n: 0.0 for n in names} den = 0.0 local_sources = [] for key, base_weight in weights.items(): result = model_results.get(key) if not result: continue local = None local_weight = base_weight source = None if key == "v0": local = statewide_raw local_weight *= 0.40 source = "statewide raw" elif key == "v1": if current_total >= 30: local = _normalized(current_votes, names) source = "county live" else: continue else: local = (local_maps.get(key) or {}).get(cid) if local is not None: source = "county remaining" else: # Statewide specialist fallback is weaker than a genuine # county-level prediction. local = result_shares(result) local_weight *= 0.42 source = "statewide fallback" if not local: continue quality = float( (horizon_validation.get(key) or {}).get( "final_quality", 0.55 ) ) local_weight *= 0.55 + 0.65 * quality for n in names: central[n] += local_weight * local.get(n, 0.0) den += local_weight local_sources.append({ "model": key, "weight": local_weight, "source": source, }) central = ( _normalized(central, names) if den > 0 else ( _normalized(current_votes, names) if current_total > 0 else statewide_raw ) ) for n in names: projected_current_county_votes[n] += current_votes.get(n, 0) projected_remaining_votes[n] += remaining * central[n] total_remaining += remaining rows.append({ "cid": cid, "county": (region or {}).get("name") or cid, "fips": (region or {}).get("fips"), "reporting": 100.0 * p, "current_votes": current_total, "current_candidate_votes": current_votes, "current_shares": ( {n: 100.0 * s for n, s in _normalized(current_votes, names).items()} if current_total > 0 else {n: None for n in names} ), "estimated_remaining_votes": remaining, "estimated_final_votes": current_total + remaining, "turnout_disagreement_votes": turnout_disagreement, "projected_remaining_shares": { n: 100.0 * central[n] for n in names }, "_remaining_shares_fraction": central, "remaining_sources": remaining_sources, "share_sources": local_sources, }) # Votes not represented in county rows are treated as already-counted # statewide residual and therefore cannot be "won" by a future county. statewide_current_votes = { n: float((current_candidates.get(n) or {}).get("votes") or 0.0) for n in names } residual = { n: max( 0.0, statewide_current_votes[n] - projected_current_county_votes[n], ) for n in names } projected_final_votes = { n: ( projected_current_county_votes[n] + residual[n] + projected_remaining_votes[n] ) for n in names } projected_final_shares = _normalized(projected_final_votes, names) expected_final_total = sum(projected_final_votes.values()) return { "counties": rows, "current_statewide_votes": statewide_current_votes, "statewide_residual_votes": residual, "projected_remaining_votes": projected_remaining_votes, "projected_final_votes": projected_final_votes, "projected_final_shares": projected_final_shares, "expected_final_total_votes": expected_final_total, "estimated_remaining_votes": total_remaining, } def _v6_recent_county_batches( base_url: str, snapshot_name: str, names: List[str], ) -> Dict[str, Any]: history = snapshot_history_window( slug_base(base_url), snapshot_name, max_intervals=1 ) if len(history) < 2: return {} actual = _actual_increment( history[-2]["data"], history[-1]["data"], names ) return { item["cid"]: { "new_votes": item["new_votes"], "shares": _normalized( item.get("actual_votes") or {}, names ), } for item in actual.get("changed") or [] if item.get("new_votes", 0) > 0 } def _v6_benchmark_status(excess_points: Optional[float]) -> str: if excess_points is None: return "NO BENCHMARK" if excess_points >= 3.0: return "WELL ABOVE" if excess_points >= 0.75: return "ABOVE" if excess_points > -0.75: return "ON BENCHMARK" if excess_points > -3.0: return "BELOW" return "WELL BELOW" def _v6_logit(p: float) -> float: p = max(1e-6, min(1.0 - 1e-6, float(p))) return math.log(p / (1.0 - p)) def _v6_logistic(x: float) -> float: if x >= 0: z = math.exp(-min(50.0, x)) return 1.0 / (1.0 + z) z = math.exp(max(-50.0, x)) return z / (1.0 + z) def _v6_distributed_win_path( forecast: Dict[str, Any], names: List[str], candidate: str, opponent: str, target_margin_points: float = 0.0, ) -> Dict[str, Any]: """ Solve a global path-to-victory benchmark. Apply one common log-odds swing to the candidate-vs-opponent pair in every remaining county, holding third-candidate mass fixed, until the aggregate statewide result reaches the requested margin. Each county's shifted share is then its dynamic benchmark. This produces realistic "Candidate needs 57% in County X" targets instead of asking one county to erase the entire statewide deficit by itself. """ rows = forecast.get("counties") or [] current_state = forecast.get("current_statewide_votes") or {} expected_final_total = max( 1.0, float(forecast.get("expected_final_total_votes") or 0.0) ) desired_margin_votes = ( float(target_margin_points) / 100.0 * expected_final_total ) current_margin_votes = ( float(current_state.get(candidate, 0.0)) - float(current_state.get(opponent, 0.0)) ) def county_shifted(row: Dict[str, Any], delta: float) -> Dict[str, float]: central = row.get("_remaining_shares_fraction") or {} c = max(0.0, float(central.get(candidate, 0.0))) o = max(0.0, float(central.get(opponent, 0.0))) pair_mass = c + o if pair_mass <= 1e-9: return { "candidate": c, "opponent": o, "pair_mass": pair_mass, } pair_c = c / pair_mass shifted_pair_c = _v6_logistic(_v6_logit(pair_c) + delta) return { "candidate": pair_mass * shifted_pair_c, "opponent": pair_mass * (1.0 - shifted_pair_c), "pair_mass": pair_mass, } def statewide_margin(delta: float) -> float: margin = current_margin_votes for row in rows: remaining = float(row.get("estimated_remaining_votes") or 0.0) if remaining <= 0: continue shifted = county_shifted(row, delta) margin += remaining * ( shifted["candidate"] - shifted["opponent"] ) return margin low, high = -12.0, 12.0 low_margin = statewide_margin(low) high_margin = statewide_margin(high) feasible = low_margin <= desired_margin_votes <= high_margin if not feasible: # Keep a bounded diagnostic even when virtually all remaining pair vote # would still be insufficient. delta = low if desired_margin_votes < low_margin else high else: for _ in range(60): mid = 0.5 * (low + high) margin = statewide_margin(mid) if margin < desired_margin_votes: low = mid else: high = mid delta = 0.5 * (low + high) county_required = {} for row in rows: shifted = county_shifted(row, delta) county_required[row["cid"]] = shifted["candidate"] achieved_margin_votes = statewide_margin(delta) return { "candidate": candidate, "opponent": opponent, "target_statewide_margin_points": target_margin_points, "log_odds_swing": delta, "feasible": feasible, "achieved_statewide_margin_points": ( 100.0 * achieved_margin_votes / expected_final_total ), "county_required_shares": county_required, "method": "distributed_common_log_odds_swing", } def build_v6_county_benchmarks( base_url: str, snapshot_name: str, model_results: Dict[str, Dict[str, Any]], weights: Dict[str, float], horizon_validation: Optional[Dict[str, Any]] = None, include_recent: bool = True, ) -> Dict[str, Any]: """ Dynamic path-to-victory benchmarks. For each candidate/county, solve for the share of that county's REMAINING vote needed to hit a statewide tie if every other remaining county follows the current V6 expectation. We also expose final-county-share benchmarks, +1/+2 statewide targets, leverage, recent batch performance, and pivotality. """ base_url = slug_base(base_url) snapshot = fetch_json(base_url + snapshot_name) names = list(candidate_map(snapshot).keys()) forecast = _v6_county_forecast_rows( snapshot, names, model_results, weights, horizon_validation=horizon_validation, ) rows = forecast["counties"] current_state = forecast["current_statewide_votes"] projected_remaining = forecast["projected_remaining_votes"] projected_final = forecast["projected_final_votes"] final_shares = forecast["projected_final_shares"] expected_final_total = max( 1.0, float(forecast["expected_final_total_votes"]) ) ranking = sorted( names, key=lambda n: projected_final.get(n, 0.0), reverse=True, ) projected_leader = ranking[0] if ranking else None projected_runner = ranking[1] if len(ranking) > 1 else None projected_margin = None if projected_leader and projected_runner: projected_margin = 100.0 * ( final_shares.get(projected_leader, 0.0) - final_shares.get(projected_runner, 0.0) ) recent = ( _v6_recent_county_batches(base_url, snapshot_name, names) if include_recent else {} ) # Build the statewide tie/+1/+2 path once per candidate. The strongest # projected opponent is used as the comparison candidate. distributed_paths = {} for candidate in names: opponent = max( (n for n in names if n != candidate), key=lambda n: projected_final.get(n, 0.0), default=None, ) if opponent is None: continue distributed_paths[candidate] = { "opponent": opponent, "targets": { target: _v6_distributed_win_path( forecast, names, candidate, opponent, target_margin_points=target, ) for target in (0.0, 1.0, 2.0) }, } county_outputs = [] leader_excess_weighted = [] leader_recent_weighted = [] for row in rows: cid = row["cid"] remaining = float(row.get("estimated_remaining_votes") or 0.0) county_current_total = float(row.get("current_votes") or 0.0) expected_final_county = county_current_total + remaining central = row["_remaining_shares_fraction"] recent_row = recent.get(cid) or {} recent_shares = recent_row.get("shares") or {} recent_votes = int(recent_row.get("new_votes") or 0) candidate_benchmarks = {} for candidate in names: opponent = max( ( n for n in names if n != candidate ), key=lambda n: projected_final.get(n, 0.0), default=None, ) if opponent is None: continue county_candidate_current = float( (row.get("current_candidate_votes") or {}).get( candidate, 0.0 ) ) candidate_remaining_expected = ( remaining * central.get(candidate, 0.0) ) opponent_remaining_expected = ( remaining * central.get(opponent, 0.0) ) other_remaining_margin = ( projected_remaining.get(candidate, 0.0) - candidate_remaining_expected - ( projected_remaining.get(opponent, 0.0) - opponent_remaining_expected ) ) other_margin_votes = ( current_state.get(candidate, 0.0) - current_state.get(opponent, 0.0) + other_remaining_margin ) pair_mass = max( 0.0, min( 1.0, central.get(candidate, 0.0) + central.get(opponent, 0.0), ), ) target_rows = {} # Secondary diagnostic: what would THIS county alone need to do if # every other remaining county stayed at its current expectation? marginal_required = None if remaining > 1: marginal_required = 0.5 * ( pair_mass - other_margin_votes / remaining ) candidate_paths = ( distributed_paths.get(candidate) or {} ).get("targets") or {} for target_margin_points in (0.0, 1.0, 2.0): path_target = candidate_paths.get( target_margin_points ) or {} required = ( (path_target.get("county_required_shares") or {}) .get(cid) ) feasible = bool(path_target.get("feasible")) required_clamped = ( max(0.0, min(1.0, float(required))) if required is not None else None ) final_benchmark = ( ( county_candidate_current + remaining * required_clamped ) / max(1.0, expected_final_county) if required_clamped is not None else None ) target_rows[ f"statewide_plus_{int(target_margin_points)}" ] = { "target_statewide_margin_points": target_margin_points, "required_remaining_share_points": ( 100.0 * required if required is not None else None ), "required_remaining_share_clamped_points": ( 100.0 * required_clamped if required_clamped is not None else None ), "required_final_county_share_points": ( 100.0 * final_benchmark if final_benchmark is not None else None ), "feasible": feasible, "distributed_path_log_odds_swing": path_target.get( "log_odds_swing" ), "distributed_path_achieved_statewide_margin_points": ( path_target.get( "achieved_statewide_margin_points" ) ), "method": path_target.get( "method" ) or "distributed_common_log_odds_swing", } # Keep the one-county break-even threshold separately. It is useful # for leverage diagnostics, but it is not the headline benchmark. marginal_break_even = { "required_remaining_share_points": ( 100.0 * marginal_required if marginal_required is not None else None ), "feasible": ( marginal_required is not None and 0.0 <= marginal_required <= pair_mass ), "already_ahead_without_this_county": ( marginal_required is not None and marginal_required < 0.0 ), "county_alone_cannot_erase_deficit": ( marginal_required is not None and marginal_required > pair_mass ), } tie = target_rows["statewide_plus_0"] required_tie = tie[ "required_remaining_share_points" ] expected_remaining = ( 100.0 * central.get(candidate, 0.0) ) excess = ( expected_remaining - required_tie if required_tie is not None else None ) recent_share = ( 100.0 * recent_shares.get(candidate, 0.0) if recent_votes > 0 else None ) recent_excess = ( recent_share - required_tie if recent_share is not None and required_tie is not None else None ) leverage = ( 2.0 * remaining / expected_final_total ) # This is statewide margin points moved by a one percentage-point # candidate-vs-opponent swing in this county. criticality = ( leverage / max( 0.35, abs(excess) if excess is not None else 25.0, ) ) candidate_benchmarks[candidate] = { "opponent": opponent, "expected_remaining_share_points": expected_remaining, "expected_final_county_share_points": ( 100.0 * ( county_candidate_current + candidate_remaining_expected ) / max(1.0, expected_final_county) ), "current_county_share_points": ( (row.get("current_shares") or {}).get(candidate) ), "recent_batch_votes": recent_votes, "recent_batch_share_points": recent_share, "tie_benchmark": tie, "marginal_single_county_break_even": marginal_break_even, "target_benchmarks": target_rows, "expected_excess_vs_win_benchmark_points": excess, "recent_excess_vs_win_benchmark_points": recent_excess, "status": _v6_benchmark_status(excess), "recent_status": _v6_benchmark_status( recent_excess ), "statewide_margin_points_per_county_share_point": leverage, "criticality_score": criticality, } leader_bench = ( candidate_benchmarks.get(projected_leader) if projected_leader else None ) if leader_bench: excess = leader_bench.get( "expected_excess_vs_win_benchmark_points" ) if excess is not None and remaining > 0: leader_excess_weighted.append( (excess, remaining) ) recent_excess = leader_bench.get( "recent_excess_vs_win_benchmark_points" ) if recent_excess is not None and recent_votes > 0: leader_recent_weighted.append( (recent_excess, recent_votes) ) county_output = { key: value for key, value in row.items() if not key.startswith("_") } county_output["candidate_benchmarks"] = candidate_benchmarks county_outputs.append(county_output) # Rank the watch list for the projected leader. leader_rows = [] if projected_leader: for row in county_outputs: bench = ( row.get("candidate_benchmarks") or {} ).get(projected_leader) if not bench: continue leader_rows.append({ "cid": row["cid"], "county": row["county"], "reporting": row.get("reporting"), "estimated_remaining_votes": row.get( "estimated_remaining_votes" ), "expected_remaining_share_points": bench.get( "expected_remaining_share_points" ), "required_remaining_share_points": ( (bench.get("tie_benchmark") or {}).get( "required_remaining_share_points" ) ), "required_final_county_share_points": ( (bench.get("tie_benchmark") or {}).get( "required_final_county_share_points" ) ), "expected_excess_points": bench.get( "expected_excess_vs_win_benchmark_points" ), "recent_batch_share_points": bench.get( "recent_batch_share_points" ), "recent_excess_points": bench.get( "recent_excess_vs_win_benchmark_points" ), "status": bench.get("status"), "recent_status": bench.get("recent_status"), "leverage": bench.get( "statewide_margin_points_per_county_share_point" ), "criticality_score": bench.get( "criticality_score" ), }) leader_expected_status_counts = { "well_above": sum(1 for r in leader_rows if r.get("status") == "WELL ABOVE"), "above": sum(1 for r in leader_rows if r.get("status") == "ABOVE"), "on_benchmark": sum(1 for r in leader_rows if r.get("status") == "ON BENCHMARK"), "below": sum(1 for r in leader_rows if r.get("status") == "BELOW"), "well_below": sum(1 for r in leader_rows if r.get("status") == "WELL BELOW"), } leader_recent_status_counts = { "well_above": sum(1 for r in leader_rows if r.get("recent_status") == "WELL ABOVE"), "above": sum(1 for r in leader_rows if r.get("recent_status") == "ABOVE"), "on_benchmark": sum(1 for r in leader_rows if r.get("recent_status") == "ON BENCHMARK"), "below": sum(1 for r in leader_rows if r.get("recent_status") == "BELOW"), "well_below": sum(1 for r in leader_rows if r.get("recent_status") == "WELL BELOW"), } top_leverage = sorted( leader_rows, key=lambda r: float(r.get("leverage") or 0.0), reverse=True, )[:12] most_pivotal = sorted( leader_rows, key=lambda r: float(r.get("criticality_score") or 0.0), reverse=True, )[:8] tipping_county = None if projected_leader and projected_margin is not None: contribution_rows = [] for row in county_outputs: bench = ( row.get("candidate_benchmarks") or {} ).get(projected_leader) if not bench: continue opponent = bench.get("opponent") if not opponent: continue remaining = float( row.get("estimated_remaining_votes") or 0.0 ) rem_shares = ( row.get("projected_remaining_shares") or {} ) contribution_votes = ( remaining * ( float( rem_shares.get( projected_leader, 0.0 ) or 0.0 ) - float( rem_shares.get(opponent, 0.0) or 0.0 ) ) / 100.0 ) contribution_points = ( 100.0 * contribution_votes / expected_final_total ) contribution_rows.append({ "cid": row["cid"], "county": row["county"], "expected_margin_contribution_points": ( contribution_points ), "distance_from_statewide_margin_points": abs( abs(contribution_points) - abs(projected_margin) ), "estimated_remaining_votes": remaining, }) if contribution_rows: candidate_tip = min( contribution_rows, key=lambda r: r[ "distance_from_statewide_margin_points" ], ) tipping_threshold = max( 0.20, 0.60 * abs(projected_margin), ) if ( candidate_tip[ "distance_from_statewide_margin_points" ] <= tipping_threshold ): tipping_county = candidate_tip county_outputs.sort( key=lambda row: float( row.get("estimated_remaining_votes") or 0.0 ), reverse=True, ) return { "available": bool(county_outputs), "projected_leader": projected_leader, "projected_runner_up": projected_runner, "projected_statewide_margin_points": projected_margin, "projected_statewide_shares": { n: 100.0 * final_shares.get(n, 0.0) for n in names }, "expected_final_votes": expected_final_total, "estimated_remaining_votes": forecast[ "estimated_remaining_votes" ], "leader_weighted_expected_excess_points": _weighted_mean( leader_excess_weighted, 0.0 ), "leader_recent_weighted_excess_points": _weighted_mean( leader_recent_weighted, 0.0 ), "leader_expected_status_counts": leader_expected_status_counts, "leader_recent_status_counts": leader_recent_status_counts, "top_leverage_counties": top_leverage, "most_pivotal_counties": most_pivotal, "tipping_county": tipping_county, "counties": county_outputs, "method_note": ( "A county's headline win benchmark is a distributed statewide " "path: V6 solves the common candidate-vs-opponent log-odds swing " "across all remaining counties required for a statewide tie, then " "translates that path into a target share for each county. A separate " "single-county break-even diagnostic is retained for leverage analysis. " "All benchmarks recalculate after every dump." ), } def _v6_structural_strengths_for_snapshot( snapshot: Dict[str, Any], model_results: Dict[str, Dict[str, Any]], reference_url: Optional[str], ) -> Dict[str, float]: """Reconstruct V6 structural priors for one historical snapshot.""" names = list(candidate_map(snapshot).keys()) maturity = statewide_reporting_maturity(snapshot) strengths = { "v0": 0.90 + 1.55 * maturity, "v1": 0.42 + 1.20 * maturity, "v3": 0.42, } v1 = model_results.get("v1") or {} geo = v1.get("geographic") or {} coverage = min( 1.0, max( 0.0, float(geo.get("current_vote_coverage_pct") or 0.0) / 100.0, ), ) capped = int(geo.get("counties_expansion_capped") or 0) used = int(geo.get("counties_used") or 0) cap_penalty = ( 1.0 if used <= 0 else max(0.30, 1.0 - capped / max(1, used)) ) strengths["v1"] *= ( 0.45 + 0.55 * coverage ) * cap_penalty ref_url = normalize_reference_url(reference_url) reference = None if ref_url: try: reference = load_reference_for_snapshot( ref_url, snapshot ) except Exception: reference = None fits = reference_fit_scores( snapshot, reference, names ) v3 = model_results.get("v3") or {} v3d = v3.get("v3") or {} if v3d.get("reference_available"): candidate_factor = ( fits["candidate_fit"] if fits["candidate_fit_available"] else 0.42 ) strengths["v3"] = ( 0.35 + 1.05 * fits["turnout_fit"] + 1.10 * candidate_factor + 0.70 * maturity ) else: strengths["v3"] = 0.34 + 0.55 * maturity if model_results.get("v2r"): strengths["v2r"] = ( 0.48 + 0.55 * fits["turnout_fit"] + 0.35 * ( fits["candidate_fit"] if fits["candidate_fit_available"] else 0.45 ) + 0.25 * maturity ) market = model_results.get("market") if market: md = market.get("media_markets") or {} confidence = ( float(md.get("mapped_current_vote_coverage") or 0.0) * float(md.get("market_coherence") or 0.0) * min( 1.0, 0.30 + 0.45 * math.sqrt( max( 0.0, float( md.get("weighted_market_progress") or 0.0 ), ) ) + 0.70 * min( 1.0, float( md.get("total_effective_influence") or 0.0 ), ), ) ) strengths["market"] = 0.20 + 1.25 * confidence histgeo = model_results.get("histgeo") if histgeo: confidence = float( (histgeo.get("historical_candidate_geo") or {}) .get("confidence") or 0.0 ) strengths["histgeo"] = 0.08 + 2.20 * confidence latent = model_results.get("latent") if latent: confidence = float( (latent.get("latent_coalition") or {}) .get("confidence") or 0.0 ) strengths["latent"] = 0.06 + 2.00 * confidence batchbias = model_results.get("batchbias") if batchbias: confidence = float( (batchbias.get("batch_bias") or {}) .get("confidence") or 0.0 ) strengths["batchbias"] = 0.05 + 1.25 * confidence runoff_transfer = model_results.get("runoff_transfer") if runoff_transfer: rd = runoff_transfer.get("runoff_transfer") or {} confidence = float(rd.get("confidence") or 0.0) cv_score = float( rd.get("cross_validation_score") or 0.12 ) strengths["runoff_transfer"] = ( 0.55 + 2.60 * confidence + 1.10 * confidence * cv_score ) precinct = model_results.get("precinct") if precinct: confidence = float( (precinct.get("precinct_hierarchy") or {}) .get("confidence") or 0.0 ) strengths["precinct"] = 0.10 + 2.20 * confidence return { key: value for key, value in strengths.items() if key in model_results and key != "temporal" } def _v6_ex_ante_gate_for_snapshot( base_url: str, snapshot_name: str, reference_url: Optional[str], market_map_source: Optional[str], bundle: Dict[str, Any], ) -> Dict[str, Any]: """ Fast ex-ante gate for benchmark replay. Benchmark breadth is itself a validation layer, so recursively rebuilding the full Path + Completion referees for every historical A→B interval was both computationally explosive and unnecessarily self-referential. V6.2 instead reconstructs the historical benchmark from: - the component projections available at A, - structural applicability at A, - specialist internal/held-out validation available at A. No future snapshot is used, and the full Path/Completion referees are still used normally for the CURRENT V6 final projection. """ cache_key = ( slug_base(base_url), snapshot_name, normalize_reference_url(reference_url) or "", market_map_source or "", "fast_benchmark_replay_v2", ) if cache_key in _V6_EX_ANTE_GATE_CACHE: return _V6_EX_ANTE_GATE_CACHE[cache_key] snapshot = bundle["snapshot"] models = { key: result for key, result in bundle["models"].items() if key != "temporal" } strengths = _v6_structural_strengths_for_snapshot( snapshot, models, reference_url ) # Empty generic referees deliberately prevent recursive historical # validation. v5_mixture_gate still reads each specialist's own held-out # validation through build_horizon_validation(). empty_completion = { "available": False, "evidence": 0.0, "models": {}, "windows": [], } empty_path = { "available": False, "evidence": 0.0, "models": {}, "intervals": [], } gate = v5_mixture_gate( models, strengths, empty_completion, empty_path, ) gate["benchmark_replay_mode"] = "fast_ex_ante_structural_plus_specialist" gate["benchmark_replay_note"] = ( "Historical county benchmarks use the model state, structural priors, " "and specialist held-out validation available at that snapshot. Full " "nested Path/Completion referee recursion is intentionally skipped." ) _V6_EX_ANTE_GATE_CACHE[cache_key] = gate return gate def rolling_v6_benchmark_signal( base_url: str, snapshot_name: str, weights: Dict[str, float], reference_url: Optional[str] = None, market_map_source: Optional[str] = None, max_intervals: int = 4, ) -> Dict[str, Any]: """ Ex-ante benchmark breadth signal. For each completed prior interval A→B, build A's county win benchmarks using only information available at A, then score the vote that actually arrived at B. Repeated benchmark beats across independent counties/markets become statewide evidence without confusing one friendly dump for a final. """ base_url = slug_base(base_url) cache_key = ( base_url, snapshot_name, normalize_reference_url(reference_url) or "", market_map_source or "", int(max_intervals), ) if cache_key in _V6_BENCHMARK_SIGNAL_CACHE: return _V6_BENCHMARK_SIGNAL_CACHE[cache_key] history = snapshot_history_window( base_url, snapshot_name, max_intervals=max_intervals ) if len(history) < 2: return { "available": False, "evidence": 0.0, "candidates": {}, } current = history[-1]["data"] names = list(candidate_map(current).keys()) current_votes = sum( int(c.get("votes") or 0) for c in candidate_map(current).values() ) state = current.get("province") resolved = discover_media_market_source( state, market_map_source ) mapping = ( load_media_market_map(resolved, state) if resolved else None ) candidate_obs = { n: { "weighted_excess_num": 0.0, "weight_den": 0.0, "above": 0, "below": 0, "on": 0, "counties": set(), "markets": set(), "votes": 0.0, } for n in names } interval_details = [] for idx in range(len(history) - 1): before_meta = history[idx] after_meta = history[idx + 1] before = before_meta["data"] after = after_meta["data"] actual = _actual_increment(before, after, names) if actual.get("new_votes", 0) <= 0: continue bundle = component_snapshot_bundle( base_url, str(before_meta["filename"]), reference_url, resolved, ) before_models = { key: result for key, result in bundle["models"].items() if key != "temporal" } # Replay the gate exactly as it existed at A. Current-snapshot # weights are never back-propagated into an older benchmark. ex_ante_gate = _v6_ex_ante_gate_for_snapshot( base_url, str(before_meta["filename"]), reference_url, resolved, bundle, ) before_weights = ( ex_ante_gate.get("weights") or _v6_normalize_weights_for_models( weights, before_models ) ) before_benchmarks = build_v6_county_benchmarks( base_url, str(before_meta["filename"]), before_models, before_weights, horizon_validation=ex_ante_gate.get( "horizon_validation" ), include_recent=False, ) bench_map = { row["cid"]: row for row in before_benchmarks.get("counties") or [] } after_regions = { _region_identity(r): r for r in county_regions(after) } recency = (idx + 1) / max(1, len(history) - 1) scored = 0 for item in actual.get("changed") or []: cid = item["cid"] county_bench = bench_map.get(cid) if not county_bench: continue new_votes = float(item.get("new_votes") or 0.0) if new_votes < 40: continue actual_shares = _normalized( item.get("actual_votes") or {}, names ) market = None if mapping and mapping.get("available"): market = market_for_region( after_regions.get(cid) or { "fips": cid, "name": item.get("county"), }, mapping, ) for candidate in names: bench = ( county_bench.get("candidate_benchmarks") or {} ).get(candidate) if not bench: continue tie = bench.get("tie_benchmark") or {} required = tie.get( "required_remaining_share_points" ) if required is None or not tie.get("feasible"): continue actual_share = ( 100.0 * actual_shares.get(candidate, 0.0) ) excess = actual_share - float(required) leverage = float( bench.get( "statewide_margin_points_per_county_share_point" ) or 0.0 ) leverage_factor = 0.45 + 0.55 * min( 1.0, leverage / 0.08 ) weight = ( math.sqrt(new_votes) * recency * leverage_factor ) obs = candidate_obs[candidate] obs["weighted_excess_num"] += weight * excess obs["weight_den"] += weight obs["votes"] += new_votes obs["counties"].add(cid) if market: obs["markets"].add(str(market)) if excess >= 0.75: obs["above"] += 1 elif excess <= -0.75: obs["below"] += 1 else: obs["on"] += 1 scored += 1 interval_details.append({ "from_snapshot": before_meta.get("snapshot"), "to_snapshot": after_meta.get("snapshot"), "new_votes": actual.get("new_votes"), "changed_counties": len( actual.get("changed") or [] ), "benchmark_observations_scored": scored, }) candidates = {} total_scored_votes = 0.0 all_counties = set() for candidate, obs in candidate_obs.items(): if obs["weight_den"] <= 0: continue excess = ( obs["weighted_excess_num"] / obs["weight_den"] ) comparisons = obs["above"] + obs["below"] + obs["on"] breadth_balance = ( (obs["above"] - obs["below"]) / max(1, comparisons) ) counties_n = len(obs["counties"]) markets_n = len(obs["markets"]) local_evidence = ( min(1.0, math.sqrt(obs["votes"] / max(1.0, 0.15 * current_votes))) * min(1.0, math.sqrt(counties_n / 18.0)) * min(1.0, math.sqrt(max(1, len(interval_details)) / 3.0)) ) signal = ( 0.62 * math.tanh(excess / 2.5) + 0.38 * breadth_balance ) * local_evidence candidates[candidate] = { "weighted_excess_points": excess, "above_benchmark_counties": obs["above"], "below_benchmark_counties": obs["below"], "on_benchmark_counties": obs["on"], "unique_counties": counties_n, "unique_markets": markets_n, "votes_scored": obs["votes"], "breadth_balance": breadth_balance, "evidence": local_evidence, "signal_score": signal, } total_scored_votes = max( total_scored_votes, obs["votes"] ) all_counties.update(obs["counties"]) if not candidates: return { "available": False, "evidence": 0.0, "candidates": {}, "intervals": interval_details, } favored = max( candidates, key=lambda n: candidates[n]["signal_score"], ) runner = min( candidates, key=lambda n: candidates[n]["signal_score"], ) favored_row = candidates[favored] overall_evidence = max( 0.0, min( 1.0, float(favored_row.get("evidence") or 0.0) * min( 1.0, math.sqrt(len(all_counties) / 20.0), ), ), ) result = { "available": True, "role": "path_only", "final_ensemble_weight": 0.0, "favored_candidate": favored, "least_favored_candidate": runner, "evidence": overall_evidence, "favored_signal_score": favored_row["signal_score"], "favored_weighted_excess_points": favored_row[ "weighted_excess_points" ], "favored_breadth_balance": favored_row[ "breadth_balance" ], "candidates": candidates, "intervals": interval_details, "method_note": ( "V6 rebuilds county tie benchmarks and the preliminary expert " "gate exactly as they existed at each prior snapshot, then grades " "only later-arriving vote. Broad, repeated benchmark beats across " "independent counties are stronger evidence than one friendly dump." ), } _V6_BENCHMARK_SIGNAL_CACHE[cache_key] = result return result def completion_v6_benchmark_signal( base_url: str, snapshot_name: str, weights: Dict[str, float], reference_url: Optional[str] = None, market_map_source: Optional[str] = None, max_windows: int = 3, ) -> Dict[str, Any]: """ Final-oriented county benchmark validation. Unlike the Path Benchmark signal, this uses longer disjoint windows and only counties that materially matured. The question is not "who won the next dump?" but "did the candidate continue beating the dynamic win path as meaningful additional county vote arrived?" """ base_url = slug_base(base_url) cache_key = ( base_url, snapshot_name, normalize_reference_url(reference_url) or "", market_map_source or "", int(max_windows), ) if cache_key in _V6_COMPLETION_BENCHMARK_CACHE: return _V6_COMPLETION_BENCHMARK_CACHE[cache_key] metadata = fetch_json( base_url + "snapshot_metadata.json" ) items = [ item for item in (metadata.get("snapshots") or []) if item.get("filename") ] def order_key(item): try: return (0, int(item.get("snapshot"))) except (TypeError, ValueError): return ( 1, str(item.get("timestamp") or ""), str(item.get("filename") or ""), ) items.sort(key=order_key) current_idx = next( ( i for i, item in enumerate(items) if item.get("filename") == snapshot_name ), None, ) if current_idx is None or current_idx < 2: result = { "available": False, "role": "final_completion", "evidence": 0.0, "candidates": {}, "windows": [], } _V6_COMPLETION_BENCHMARK_CACHE[ cache_key ] = result return result current = fetch_json(base_url + snapshot_name) names = list(candidate_map(current).keys()) current_votes = sum( int(c.get("votes") or 0) for c in candidate_map(current).values() ) state = current.get("province") resolved = discover_media_market_source( state, market_map_source ) mapping = ( load_media_market_map(resolved, state) if resolved else None ) span = max( 2, min( 12, int( round( current_idx / max(2, max_windows + 1) ) ), ), ) boundaries = [current_idx] while ( len(boundaries) <= max_windows and boundaries[-1] - span >= 0 ): boundaries.append( boundaries[-1] - span ) boundaries = sorted(set(boundaries)) pair_count = max(0, len(boundaries) - 1) if pair_count <= 0: result = { "available": False, "role": "final_completion", "evidence": 0.0, "candidates": {}, "windows": [], } _V6_COMPLETION_BENCHMARK_CACHE[ cache_key ] = result return result candidate_obs = { n: { "weighted_excess_num": 0.0, "weight_den": 0.0, "weighted_margin_lift_num": 0.0, "margin_lift_den": 0.0, "above": 0, "below": 0, "on": 0, "counties": set(), "markets": set(), "votes": 0.0, "window_lifts": [], "window_signs": [], } for n in names } windows = [] all_counties = set() total_matured_votes = 0.0 weighted_advance_num = 0.0 weighted_advance_den = 0.0 for pair_idx in range(pair_count): before_idx = boundaries[pair_idx] after_idx = boundaries[pair_idx + 1] before_meta = items[before_idx] after_meta = items[after_idx] before = fetch_json( base_url + str(before_meta["filename"]) ) after = fetch_json( base_url + str(after_meta["filename"]) ) matured = _material_county_maturation( before, after, names ) county_rows = matured.get("counties") or [] if not county_rows: continue bundle = component_snapshot_bundle( base_url, str(before_meta["filename"]), reference_url, resolved, ) before_models = { key: result for key, result in bundle["models"].items() if key != "temporal" } ex_ante_gate = _v6_ex_ante_gate_for_snapshot( base_url, str(before_meta["filename"]), reference_url, resolved, bundle, ) before_weights = ( ex_ante_gate.get("weights") or _v6_normalize_weights_for_models( weights, before_models ) ) before_benchmarks = ( build_v6_county_benchmarks( base_url, str(before_meta["filename"]), before_models, before_weights, horizon_validation=ex_ante_gate.get( "horizon_validation" ), include_recent=False, ) ) bench_map = { row["cid"]: row for row in ( before_benchmarks.get("counties") or [] ) } after_regions = { _region_identity(r): r for r in county_regions(after) } recency = ( pair_idx + 1 ) / max(1, pair_count) window_lifts = { n: 0.0 for n in names } window_votes = 0.0 scored_counties = set() for item in county_rows: cid = item["cid"] county_bench = bench_map.get(cid) if not county_bench: continue new_votes = float( item.get("new_votes") or 0.0 ) if new_votes < 100: continue actual_shares = ( item.get("actual_shares") or {} ) market = None if mapping and mapping.get("available"): market = market_for_region( after_regions.get(cid) or { "fips": cid, "name": item.get("county"), }, mapping, ) remaining_before = float( county_bench.get( "estimated_remaining_votes" ) or 0.0 ) realized_fraction = min( 1.0, new_votes / max(new_votes, remaining_before), ) materiality = float( item.get("materiality") or 0.0 ) base_weight = ( math.sqrt(new_votes) * recency * (0.65 + 0.35 * materiality) ) county_scored = False for candidate in names: bench = ( county_bench.get( "candidate_benchmarks" ) or {} ).get(candidate) if not bench: continue tie = bench.get( "tie_benchmark" ) or {} required = tie.get( "required_remaining_share_points" ) if ( required is None or not tie.get("feasible") ): continue actual_share = ( 100.0 * float( actual_shares.get( candidate, 0.0 ) ) ) excess = ( actual_share - float(required) ) leverage = float( bench.get( "statewide_margin_points_per_county_share_point" ) or 0.0 ) margin_lift = ( excess * leverage * realized_fraction ) obs = candidate_obs[candidate] obs["weighted_excess_num"] += ( base_weight * excess ) obs["weight_den"] += base_weight obs[ "weighted_margin_lift_num" ] += base_weight * margin_lift obs["margin_lift_den"] += base_weight obs["votes"] += new_votes obs["counties"].add(cid) if market: obs["markets"].add( str(market) ) if excess >= 0.75: obs["above"] += 1 elif excess <= -0.75: obs["below"] += 1 else: obs["on"] += 1 window_lifts[candidate] += ( margin_lift ) county_scored = True if county_scored: scored_counties.add(cid) window_votes += new_votes if not scored_counties: continue for candidate in names: lift = float( window_lifts.get(candidate) or 0.0 ) candidate_obs[candidate][ "window_lifts" ].append({ "lift_points": lift, "weight": max( 1.0, math.sqrt(window_votes) ) * recency, "matured_votes": window_votes, "matured_counties": len( scored_counties ), "recency": recency, }) candidate_obs[candidate][ "window_signs" ].append( 1 if lift > 0.03 else (-1 if lift < -0.03 else 0) ) total_matured_votes += window_votes all_counties.update(scored_counties) for item in county_rows: if item["cid"] not in scored_counties: continue weighted_advance_num += ( float(item.get("new_votes") or 0.0) * float( item.get( "reporting_advance" ) or 0.0 ) ) weighted_advance_den += float( item.get("new_votes") or 0.0 ) windows.append({ "from_snapshot": before_meta.get( "snapshot" ), "to_snapshot": after_meta.get( "snapshot" ), "from_filename": before_meta.get( "filename" ), "to_filename": after_meta.get( "filename" ), "span_snapshots": ( after_idx - before_idx ), "matured_votes_scored": window_votes, "matured_counties_scored": len( scored_counties ), "window_margin_lift_points": ( window_lifts ), }) if not windows: result = { "available": False, "role": "final_completion", "evidence": 0.0, "candidates": {}, "windows": [], } _V6_COMPLETION_BENCHMARK_CACHE[ cache_key ] = result return result avg_reporting_advance = ( weighted_advance_num / weighted_advance_den if weighted_advance_den > 0 else 0.0 ) volume_evidence = min( 1.0, math.sqrt( total_matured_votes / max(1.0, 0.30 * current_votes) ), ) window_evidence = min( 1.0, math.sqrt(len(windows) / 3.0), ) geography_evidence = min( 1.0, math.sqrt(len(all_counties) / 24.0), ) maturation_evidence = min( 1.0, avg_reporting_advance / 0.18, ) base_evidence = max( 0.0, min( 1.0, volume_evidence * window_evidence * ( 0.35 + 0.35 * geography_evidence + 0.30 * maturation_evidence ), ), ) candidates = {} for candidate, obs in candidate_obs.items(): if obs["weight_den"] <= 0: continue excess = ( obs["weighted_excess_num"] / obs["weight_den"] ) margin_lift = ( obs["weighted_margin_lift_num"] / max( 1e-9, obs["margin_lift_den"] ) ) comparisons = ( obs["above"] + obs["below"] + obs["on"] ) breadth_balance = ( (obs["above"] - obs["below"]) / max(1, comparisons) ) # V6.2 persistence is magnitude/recency weighted rather than a # democratic vote among historical windows. This matters at genuine # turning points: a large new regime should be able to outweigh two # small stale windows, while a flashy but minority-sized late batch # should not automatically rewrite the election. informative_windows = [ row for row in obs["window_lifts"] if abs(float(row["lift_points"])) > 0.03 ] positive_weight = sum( float(row["weight"]) for row in informative_windows if float(row["lift_points"]) > 0.0 ) negative_weight = sum( float(row["weight"]) for row in informative_windows if float(row["lift_points"]) < 0.0 ) signed_weight = ( positive_weight + negative_weight ) positive_weight_share = ( positive_weight / signed_weight if signed_weight > 0 else 0.50 ) # Candidate-specific persistence means "how much of the weighted # temporal evidence supports THIS candidate's positive lift?" persistence = positive_weight_share latest_window = ( obs["window_lifts"][-1] if obs["window_lifts"] else {} ) latest_lift = float( latest_window.get("lift_points") or 0.0 ) latest_votes = float( latest_window.get("matured_votes") or 0.0 ) total_window_votes = sum( float(row.get("matured_votes") or 0.0) for row in obs["window_lifts"] ) latest_vote_share = ( latest_votes / total_window_votes if total_window_votes > 0 else 0.0 ) latest_weight = float( latest_window.get("weight") or 0.0 ) total_window_weight = sum( float(row.get("weight") or 0.0) for row in obs["window_lifts"] ) latest_weight_share = ( latest_weight / total_window_weight if total_window_weight > 0 else 0.0 ) prior_rows = obs["window_lifts"][:-1] prior_vote_den = sum( float(row.get("matured_votes") or 0.0) for row in prior_rows ) prior_vote_weighted_lift = ( sum( float(row.get("matured_votes") or 0.0) * float(row.get("lift_points") or 0.0) for row in prior_rows ) / prior_vote_den if prior_vote_den > 0 else 0.0 ) regime_reversal = bool( latest_lift > 0.08 and prior_rows and prior_vote_weighted_lift < -0.02 ) window_lift_den = sum( float(row["weight"]) for row in obs["window_lifts"] ) persistent_lift = ( sum( float(row["weight"]) * float(row["lift_points"]) for row in obs["window_lifts"] ) / window_lift_den if window_lift_den > 0 else margin_lift ) local_evidence = ( base_evidence * min( 1.0, math.sqrt( len(obs["counties"]) / 18.0 ), ) ) signal = ( 0.38 * math.tanh(excess / 2.5) + 0.34 * math.tanh( persistent_lift / 0.55 ) + 0.18 * breadth_balance + 0.10 * ( 2.0 * persistence - 1.0 ) ) * local_evidence candidates[candidate] = { "weighted_excess_points": excess, "implied_statewide_margin_lift_points": ( persistent_lift ), "above_benchmark_counties": obs[ "above" ], "below_benchmark_counties": obs[ "below" ], "on_benchmark_counties": obs["on"], "unique_counties": len( obs["counties"] ), "unique_markets": len( obs["markets"] ), "votes_scored": obs["votes"], "breadth_balance": breadth_balance, "temporal_persistence": persistence, "positive_window_weight_share": ( positive_weight_share ), "latest_window_lift_points": latest_lift, "latest_window_vote_share": ( latest_vote_share ), "latest_window_weight_share": ( latest_weight_share ), "latest_window_matured_votes": ( latest_votes ), "latest_window_matured_counties": int( latest_window.get( "matured_counties" ) or 0 ), "prior_vote_weighted_lift_points": ( prior_vote_weighted_lift ), "regime_reversal": regime_reversal, "windows_scored": len( obs["window_lifts"] ), "evidence": local_evidence, "signal_score": signal, } if not candidates: result = { "available": False, "role": "final_completion", "evidence": 0.0, "candidates": {}, "windows": windows, } _V6_COMPLETION_BENCHMARK_CACHE[ cache_key ] = result return result favored = max( candidates, key=lambda n: candidates[n][ "signal_score" ], ) favored_row = candidates[favored] evidence = max( 0.0, min( 1.0, float( favored_row.get("evidence") or 0.0 ), ), ) persistent_actionable = bool( evidence >= 0.45 and abs( float( favored_row.get( "signal_score" ) or 0.0 ) ) >= 0.18 and float( favored_row.get( "breadth_balance" ) or 0.0 ) >= 0.05 and int( favored_row.get( "unique_counties" ) or 0 ) >= 8 and float( favored_row.get( "temporal_persistence" ) or 0.0 ) >= 0.55 ) validation_quality = max( 0.0, min( 1.0, evidence * ( 0.35 + 0.35 * min( 1.0, abs( float( favored_row.get( "signal_score" ) or 0.0 ) ) / 0.45, ) + 0.30 * float( favored_row.get( "temporal_persistence" ) or 0.0 ) ), ), ) result = { "available": True, "role": "final_completion", "favored_candidate": favored, "evidence": evidence, "base_evidence": base_evidence, "favored_signal_score": favored_row[ "signal_score" ], "favored_weighted_excess_points": ( favored_row[ "weighted_excess_points" ] ), "favored_implied_statewide_margin_lift_points": ( favored_row[ "implied_statewide_margin_lift_points" ] ), "favored_breadth_balance": favored_row[ "breadth_balance" ], "favored_temporal_persistence": ( favored_row[ "temporal_persistence" ] ), "favored_latest_window_lift_points": ( favored_row[ "latest_window_lift_points" ] ), "favored_latest_window_vote_share": ( favored_row[ "latest_window_vote_share" ] ), "favored_latest_window_weight_share": ( favored_row[ "latest_window_weight_share" ] ), "favored_prior_vote_weighted_lift_points": ( favored_row[ "prior_vote_weighted_lift_points" ] ), "favored_regime_reversal": bool( favored_row.get("regime_reversal") ), "directional": ( float( favored_row.get( "signal_score" ) or 0.0 ) > 0.0 ), "persistent_actionable_pre_maturity": persistent_actionable, "actionable_pre_maturity": persistent_actionable, "validation_quality": ( validation_quality ), "validation_mae_points": max( 0.35, 4.0 * (1.0 - validation_quality), ), "candidates": candidates, "windows": windows, "matured_votes_scored": ( total_matured_votes ), "unique_matured_counties": len( all_counties ), "average_reporting_advance_points": ( 100.0 * avg_reporting_advance ), "method_note": ( "V6.2 Completion Benchmark uses longer material county-maturation " "windows. Only this benchmark layer is eligible to influence the " "final Needle; the short-horizon Path Benchmark remains path-only." ), } _V6_COMPLETION_BENCHMARK_CACHE[ cache_key ] = result return result def v6_turning_point_signal( names: List[str], raw_shares: Dict[str, float], completion_signal: Dict[str, Any], path_signal: Dict[str, Any], model_results: Dict[str, Dict[str, Any]], horizon_validation: Dict[str, Any], maturity_info: Dict[str, Any], ) -> Dict[str, Any]: """ Detect a credible reporting-regime change without giving the Path Benchmark any direct final-result weight. The Path layer is used only as corroboration. A turning point also requires: * serious-data maturity, * a Completion Benchmark reversal, * the newest material window to dominate the NEW matured vote mass, * strong broad path evidence in the same direction, * at least one independently validated final-oriented expert agreeing, * and a sufficiently close raw scoreboard when the candidate differs from the raw leader. This is deliberately designed to distinguish Oklahoma-like late regime changes from Michigan-like flashy late batches that are large in recency weight but still a minority of the material completion evidence. """ maturity = float( maturity_info.get("counted_fraction_estimate") or 0.0 ) favored = completion_signal.get("favored_candidate") raw_rank = sorted( names, key=lambda n: raw_shares.get(n, 0.0), reverse=True, ) raw_leader = raw_rank[0] if raw_rank else None raw_runner = raw_rank[1] if len(raw_rank) > 1 else None raw_margin = ( 100.0 * ( raw_shares.get(raw_leader, 0.0) - raw_shares.get(raw_runner, 0.0) ) if raw_leader and raw_runner else None ) favored_gap_to_raw_leader = ( 100.0 * ( raw_shares.get(raw_leader, 0.0) - raw_shares.get(favored, 0.0) ) if raw_leader and favored else None ) if not favored or favored not in names: return { "available": False, "actionable": False, "candidate": None, "reason": "No valid Completion Benchmark favorite.", } latest_lift = float( completion_signal.get( "favored_latest_window_lift_points" ) or 0.0 ) latest_vote_share = float( completion_signal.get( "favored_latest_window_vote_share" ) or 0.0 ) latest_weight_share = float( completion_signal.get( "favored_latest_window_weight_share" ) or 0.0 ) prior_lift = float( completion_signal.get( "favored_prior_vote_weighted_lift_points" ) or 0.0 ) completion_score = float( completion_signal.get( "favored_signal_score" ) or 0.0 ) completion_evidence = float( completion_signal.get("evidence") or 0.0 ) path_same = bool( path_signal.get("available") and path_signal.get("favored_candidate") == favored ) path_score = float( path_signal.get("favored_signal_score") or 0.0 ) path_breadth = float( path_signal.get("favored_breadth_balance") or 0.0 ) path_evidence = float( path_signal.get("evidence") or 0.0 ) family_map = { "v1": "county", "v3": "remaining", "market": "market", "histgeo": "historical", "latent": "historical", "batchbias": "reporting_order", "v2r": "runoff", "runoff_transfer": "runoff", "precinct": "precinct", } independent_supporters = [] best_by_family = {} for key, family in family_map.items(): model = model_results.get(key) if not model: continue shares = result_shares(model) ranked = sorted( names, key=lambda n: shares.get(n, 0.0), reverse=True, ) if not ranked or ranked[0] != favored: continue hv = horizon_validation.get(key) or {} quality = float(hv.get("final_quality") or 0.0) evidence = float(hv.get("final_evidence") or 0.0) if quality < 0.45 or evidence < 0.45: continue row = { "model": key, "family": family, "quality": quality, "evidence": evidence, "weight": 0.0, "projected_leader": favored, } current = best_by_family.get(family) rank_score = quality * evidence if ( current is None or rank_score > float(current.get("_rank_score") or 0.0) ): row["_rank_score"] = rank_score best_by_family[family] = row for row in best_by_family.values(): row.pop("_rank_score", None) independent_supporters.append(row) latest_dominates = bool( latest_vote_share >= 0.50 and latest_weight_share >= 0.56 ) regime_reversal = bool( completion_signal.get("favored_regime_reversal") and latest_lift >= 0.10 and prior_lift <= -0.02 ) path_qualified = bool( path_same and path_evidence >= 0.70 and path_score >= 0.20 and path_breadth >= 0.10 ) independent_qualified = bool( len(independent_supporters) >= 1 ) close_raw = bool( raw_leader == favored or ( favored_gap_to_raw_leader is not None and favored_gap_to_raw_leader <= ( 1.00 if maturity < 0.50 else 0.60 ) ) ) serious = maturity >= 0.30 completion_qualified = bool( completion_evidence >= 0.70 and completion_score >= 0.08 ) actionable = bool( favored != raw_leader and serious and completion_qualified and regime_reversal and latest_dominates and path_qualified and independent_qualified and close_raw ) score_parts = { "maturity": max( 0.0, min(1.0, (maturity - 0.30) / 0.20) ), "latest_vote_dominance": max( 0.0, min(1.0, (latest_vote_share - 0.45) / 0.15), ), "latest_weight_dominance": max( 0.0, min(1.0, (latest_weight_share - 0.50) / 0.16), ), "latest_lift": max( 0.0, min(1.0, latest_lift / 0.30) ), "path": max( 0.0, min(1.0, path_score / 0.45) ), "path_breadth": max( 0.0, min(1.0, path_breadth / 0.30) ), "independent": max( 0.0, min(1.0, len(independent_supporters) / 2.0) ), } score = ( 0.12 * score_parts["maturity"] + 0.19 * score_parts["latest_vote_dominance"] + 0.16 * score_parts["latest_weight_dominance"] + 0.14 * score_parts["latest_lift"] + 0.16 * score_parts["path"] + 0.12 * score_parts["path_breadth"] + 0.11 * score_parts["independent"] ) failures = [] checks = { "serious_data": serious, "completion_direction": completion_qualified, "regime_reversal": regime_reversal, "latest_window_dominates": latest_dominates, "path_confirmation": path_qualified, "independent_final_expert": independent_qualified, "raw_dead_heat_or_same_leader": close_raw, } for key, ok in checks.items(): if not ok: failures.append(key) return { "available": True, "candidate": favored, "raw_leader": raw_leader, "raw_top_two_margin_points": raw_margin, "raw_gap_to_candidate_points": ( favored_gap_to_raw_leader ), "actionable": actionable, "score": max(0.0, min(1.0, score)), "checks": checks, "failed_checks": failures, "latest_window_lift_points": latest_lift, "latest_window_vote_share": latest_vote_share, "latest_window_weight_share": latest_weight_share, "prior_vote_weighted_lift_points": prior_lift, "completion_signal_score": completion_score, "completion_evidence": completion_evidence, "path_signal_score": path_score, "path_breadth_balance": path_breadth, "path_evidence": path_evidence, "independent_supporters": independent_supporters, "independent_families": len(independent_supporters), "score_parts": score_parts, "method_note": ( "Turning-Point Consensus requires the newest material Completion " "window to dominate the matured vote mass, reverse the older regime, " "and receive independent confirmation from the Path Benchmark plus " "at least one validated final-oriented expert. Path itself still has " "0% direct final weight." ), } def project_v6_completion_benchmark_expert( base_url: str, snapshot_name: str, raw_shares: Dict[str, float], completion_signal: Dict[str, Any], path_signal: Dict[str, Any], turning_point: Dict[str, Any], maturity_info: Dict[str, Any], ) -> Optional[Dict[str, Any]]: """ Independent V6.2 Completion Benchmark expert. V6.1 applied a small correction to the pre-existing ensemble; V6.2 replaces that behavior. That meant the "Completion Benchmark expert" could still predict the opposite candidate from its own benchmark signal. V6.2 instead starts from the OBSERVED raw board and estimates the remaining statewide margin shift from material county completion. It therefore has an independent signed margin. Path evidence can affect magnitude only after the Turning-Point detector has independently validated a regime change; Path itself still receives zero direct final ensemble weight. """ if not completion_signal.get("available"): return None base_url = slug_base(base_url) snapshot = fetch_json(base_url + snapshot_name) final = fetch_json(base_url + "latest.json") names = list(candidate_map(snapshot).keys()) favored = completion_signal.get("favored_candidate") if favored not in names: return None opponent = max( (n for n in names if n != favored), key=lambda n: raw_shares.get(n, 0.0), default=None, ) if opponent is None: return None activation = float( maturity_info.get("activation") or 0.0 ) maturity = float( maturity_info.get( "counted_fraction_estimate" ) or 0.0 ) evidence = float( completion_signal.get("evidence") or 0.0 ) persistence = float( completion_signal.get( "favored_temporal_persistence" ) or 0.0 ) validation_quality = float( completion_signal.get( "validation_quality" ) or 0.0 ) signal_score = float( completion_signal.get( "favored_signal_score" ) or 0.0 ) persistent_lift = float( completion_signal.get( "favored_implied_statewide_margin_lift_points" ) or 0.0 ) latest_lift = float( completion_signal.get( "favored_latest_window_lift_points" ) or 0.0 ) weighted_excess = float( completion_signal.get( "favored_weighted_excess_points" ) or 0.0 ) persistent_actionable = bool( completion_signal.get( "persistent_actionable_pre_maturity" ) and maturity >= 0.30 ) turning_actionable = bool( turning_point.get("actionable") and turning_point.get("candidate") == favored ) actionable = bool( persistent_actionable or turning_actionable ) raw_signed_margin = 100.0 * ( raw_shares.get(favored, 0.0) - raw_shares.get(opponent, 0.0) ) # Convert the short-horizon path excess into a bounded statewide-margin # scale using the empirically observed completion leverage. This term is # admitted ONLY after a full Turning-Point validation. leverage_scale = ( abs(persistent_lift) / max(0.25, abs(weighted_excess)) ) leverage_scale = max( 0.05, min(0.45, leverage_scale) ) path_excess = float( path_signal.get( "favored_weighted_excess_points" ) or 0.0 ) path_implied_lift = max( -1.25, min( 1.25, path_excess * leverage_scale, ), ) if turning_actionable: estimated_remaining_margin_shift = ( 0.35 * persistent_lift + 0.65 * latest_lift + 0.50 * path_implied_lift ) regime = "turning_point" elif persistent_actionable: estimated_remaining_margin_shift = ( 0.78 * persistent_lift + 0.22 * latest_lift ) regime = "persistent_completion" else: # Directional/non-actionable completion evidence is allowed to produce # a diagnostic signed estimate, but only a very conservative one. estimated_remaining_margin_shift = ( 0.35 * persistent_lift ) regime = "directional_only" estimated_remaining_margin_shift = max( -2.50, min(2.50, estimated_remaining_margin_shift), ) # V6.2.2 bug fix: # The ±2.5 bound applies to the *new completion correction*, not to the # entire projected race margin. Previous builds also clamped the resulting # candidate-vs-opponent margin to ±4 points, which made the standalone # Completion Benchmark nonsensical in blowouts (for example, turning a # raw +60 race into a +4 projection). independent_margin = ( raw_signed_margin + estimated_remaining_margin_shift ) # Preserve all other candidates. Within the favored/opponent pair, solve # directly for the desired independent signed margin. projected = dict( _normalized(raw_shares, names) ) pair_total = ( projected.get(favored, 0.0) + projected.get(opponent, 0.0) ) target_diff = independent_margin / 100.0 target_diff = max( -pair_total + 1e-6, min(pair_total - 1e-6, target_diff), ) projected[favored] = ( pair_total + target_diff ) / 2.0 projected[opponent] = ( pair_total - target_diff ) / 2.0 projected = _normalized(projected, names) turn_score = float( turning_point.get("score") or 0.0 ) if turning_actionable: confidence = ( activation * evidence * ( 0.36 + 0.34 * validation_quality + 0.30 * turn_score ) ) elif persistent_actionable: confidence = ( activation * evidence * ( 0.42 + 0.30 * persistence + 0.28 * validation_quality ) ) else: confidence = ( 0.10 * activation * evidence * ( 0.50 + 0.50 * validation_quality ) ) confidence = max( 0.0, min(1.0, confidence) ) result = _score_projection( base_url, snapshot_name, snapshot, final, projected, "V6_2_COMPLETION_BENCHMARK_EXPERT", ( "V6.2 Completion Benchmark is an independent signed-margin expert " "anchored to the observed raw board plus estimated remaining " "completion shift. Path evidence is admitted only after a validated " "Turning-Point regime change and never receives direct final weight." ), { "uses_final_results_for_projection": False, "specialist": "benchmark_completion", "benchmark_role": "independent_final_completion", "path_direct_final_weight": 0.0, }, ) result["benchmark_completion"] = { "available": True, "favored_candidate": favored, "opponent": opponent, "confidence": confidence, "maturity_activation": activation, "maturity_stage": maturity_info.get("stage"), "signal_score": signal_score, "evidence": evidence, "temporal_persistence": persistence, "validation_quality": validation_quality, "validation_mae_points": ( completion_signal.get( "validation_mae_points" ) ), "persistent_actionable": persistent_actionable, "turning_point_actionable": turning_actionable, "actionable": actionable, "regime": regime, "raw_signed_margin_points": raw_signed_margin, "completion_correction_cap_points": 2.50, "final_margin_is_not_artificially_capped": True, "persistent_completion_lift_points": persistent_lift, "latest_completion_lift_points": latest_lift, "path_implied_lift_points": ( path_implied_lift if turning_actionable else 0.0 ), "estimated_remaining_margin_shift_points": ( estimated_remaining_margin_shift ), "independent_signed_margin_points": ( independent_margin ), "turning_point": turning_point, "max_ensemble_influence": min( ( 0.30 if turning_actionable else ( 0.32 if persistent_actionable else 0.05 ) ), 0.025 + 0.38 * confidence, ), "source_signal": completion_signal, } return result def v6_crossover_consensus( names: List[str], raw_shares: Dict[str, float], model_results: Dict[str, Dict[str, Any]], weights: Dict[str, float], horizon_validation: Dict[str, Any], maturity_info: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ Detect independent expert families crossing against the raw scoreboard. V6.2.2 uses stricter rules before serious-data maturity and slightly more permissive rules at 50%+, where stable agreement should matter more. """ maturity_info = maturity_info or {} maturity = float( maturity_info.get( "counted_fraction_estimate" ) or 0.0 ) raw_rank = sorted( names, key=lambda n: raw_shares.get(n, 0.0), reverse=True, ) raw_leader = ( raw_rank[0] if raw_rank else None ) family_map = { "v0": "raw", "v1": "county", "v3": "remaining", "market": "market", "histgeo": "historical", "latent": "historical", "batchbias": "reporting_order", "v2r": "runoff", "runoff_transfer": "runoff", "precinct": "precinct", "benchmark_completion": ( "benchmark_completion" ), } candidate_families = {} supporters = {} for key, result in model_results.items(): if key in ("temporal", "v0"): continue shares = result_shares(result) ranked = sorted( names, key=lambda n: shares.get(n, 0.0), reverse=True, ) if not ranked: continue leader = ranked[0] if leader == raw_leader: continue if key == "benchmark_completion": bc = result.get("benchmark_completion") or {} if not ( bc.get("persistent_actionable") or bc.get("turning_point_actionable") ): continue hv = horizon_validation.get(key) or {} # Consensus is a FINAL-result vote. Ordinary experts use the combined # horizon score. The Completion Benchmark is special because its own # material-maturation exam IS the final-oriented validation layer. if key == "benchmark_completion": bc = result.get("benchmark_completion") or {} quality = float( bc.get("validation_quality") or hv.get("final_quality") or 0.55 ) evidence = max( float(bc.get("confidence") or 0.0), float( ( bc.get("source_signal") or {} ).get("evidence") or 0.0 ), ) else: quality = float( hv.get("final_quality") or 0.55 ) evidence = float( hv.get("final_evidence") or 0.0 ) quality_floor = ( 0.46 if maturity >= 0.50 else (0.44 if maturity >= 0.30 else 0.50) ) evidence_floor = ( 0.30 if maturity >= 0.50 else (0.34 if maturity >= 0.30 else 0.45) ) if ( quality < quality_floor or evidence < evidence_floor ): continue family = family_map.get(key, key) contribution = ( float(weights.get(key, 0.0)) * (0.45 + 0.75 * quality) * (0.40 + 0.60 * evidence) ) fam = candidate_families.setdefault( leader, {} ) if contribution > fam.get( family, 0.0 ): fam[family] = contribution supporters.setdefault( leader, [] ).append({ "model": key, "family": family, "weight": weights.get(key, 0.0), "quality": quality, "evidence": evidence, "contribution": contribution, }) if maturity < 0.30: required_families = 3 required_score = 0.115 elif maturity < 0.50: required_families = 2 required_score = 0.075 else: required_families = 2 required_score = 0.055 candidates = {} for candidate, families in ( candidate_families.items() ): score = sum(families.values()) candidates[candidate] = { "independent_families": len( families ), "families": families, "supporters": sorted( supporters.get(candidate, []), key=lambda r: r[ "contribution" ], reverse=True, ), "score": score, "eligible": ( len(families) >= required_families and score >= required_score ), } best = ( max( candidates, key=lambda n: candidates[n]["score"], ) if candidates else None ) return { "available": bool(candidates), "raw_leader": raw_leader, "candidate": best, "eligible": bool( best and candidates[best].get( "eligible" ) ), "score": ( candidates[best]["score"] if best else 0.0 ), "independent_families": ( candidates[best][ "independent_families" ] if best else 0 ), "required_families": ( required_families ), "required_score": required_score, "maturity_percent_estimate": ( 100.0 * maturity ), "candidates": candidates, } def apply_v6_benchmark_gate( preliminary_gate: Dict[str, Any], model_results: Dict[str, Dict[str, Any]], names: List[str], raw_shares: Dict[str, float], completion_benchmark_signal: Dict[str, Any], path_benchmark_signal: Optional[Dict[str, Any]] = None, turning_point: Optional[Dict[str, Any]] = None, maturity_info: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ V6.2.2 final benchmark gate. Material-maturation Completion Benchmark evidence is the benchmark layer that can reweight final experts. Path Benchmark still has zero direct final weight; it can only act as one prerequisite inside a separately validated Turning-Point Consensus. """ maturity_info = maturity_info or {} path_benchmark_signal = path_benchmark_signal or {} turning_point = turning_point or {} activation = float( maturity_info.get("activation") or 0.0 ) maturity = float( maturity_info.get( "counted_fraction_estimate" ) or 0.0 ) gate = dict(preliminary_gate) pre_weights = dict( preliminary_gate.get("weights") or {} ) horizon = ( preliminary_gate.get( "horizon_validation" ) or {} ) consensus = v6_crossover_consensus( names, raw_shares, model_results, pre_weights, horizon, maturity_info=maturity_info, ) favored = completion_benchmark_signal.get( "favored_candidate" ) bench_evidence = float( completion_benchmark_signal.get( "evidence" ) or 0.0 ) bench_score = float( completion_benchmark_signal.get( "favored_signal_score" ) or 0.0 ) persistence = float( completion_benchmark_signal.get( "favored_temporal_persistence" ) or 0.0 ) persistent_actionable = bool( completion_benchmark_signal.get( "persistent_actionable_pre_maturity" ) and maturity >= 0.30 ) turning_actionable = bool( turning_point.get("actionable") and turning_point.get("candidate") == favored ) actionable = bool( persistent_actionable or turning_actionable ) turning_strength = ( float(turning_point.get("score") or 0.0) if turning_actionable else 0.0 ) raw_rank = sorted( names, key=lambda n: raw_shares.get(n, 0.0), reverse=True, ) raw_leader = ( raw_rank[0] if raw_rank else None ) adjustments = {} turning_contradiction_penalties = {} turning_pair_gaps = {} logits = {} for key, weight in pre_weights.items(): if key not in model_results: continue shares = result_shares( model_results[key] ) ranked = sorted( names, key=lambda n: shares.get(n, 0.0), reverse=True, ) leader = ( ranked[0] if ranked else None ) hv = horizon.get(key) or {} quality = float( hv.get("final_quality") or 0.55 ) evidence = float( hv.get("final_evidence") or 0.0 ) adj = 0.0 # Starting around 30%, live completion validation becomes a meaningful # gate input. At 50%+, this is full-strength "decision mode". if maturity >= 0.30: decision_strength = ( 0.45 * activation if maturity < 0.50 else 1.0 ) adj += ( 1.35 * decision_strength * evidence * (quality - 0.50) ) # Only an ACTIONABLE long-horizon Completion Benchmark may directly # reweight final experts. Directional/non-actionable signals remain # diagnostics and can still be shown in the UI. if ( actionable and favored and activation > 0 and bench_evidence >= 0.30 and abs(bench_score) >= 0.08 ): magnitude = ( 1.30 * activation * bench_evidence * ( 0.45 + 0.55 * min( 1.0, abs(bench_score) / 0.45, ) ) * (0.50 + 0.50 * persistence) * (0.45 + 0.55 * quality) * (0.45 + 0.55 * evidence) ) if leader == favored: adj += magnitude elif ( favored != raw_leader and leader == raw_leader ): adj -= 0.55 * magnitude # A validated turning point is intentionally stronger than ordinary # directional completion evidence because it has already passed: # material-window dominance, regime reversal, broad Path confirmation, # an independent final expert, and a dead-heat check. if ( turning_actionable and favored and maturity >= 0.30 ): turn_magnitude = ( (0.85 + 0.95 * turning_strength) * max(0.45, activation) * (0.50 + 0.50 * quality) * (0.45 + 0.55 * evidence) ) if leader == favored: adj += turn_magnitude elif ( favored != raw_leader and leader == raw_leader ): adj -= 0.72 * turn_magnitude # V6.2.2 Turning-Point Contradiction Penalty. # # Once a turning point has passed every safeguard, experts that still # project the OLD raw leader ahead of the turning-point candidate are # treated as partially stale descriptions of the prior reporting # regime. The penalty scales with: # * turning-point strength, # * maturity activation, # * how far the expert still has the old leader ahead, # * and the expert's own final quality. # # This is intentionally impossible to trigger in the MI-31 style case # because turning_actionable is false there. contradiction_penalty = 0.0 pair_gap_points = 0.0 if ( turning_actionable and favored and raw_leader and favored != raw_leader and maturity >= 0.30 ): pair_gap_points = max( 0.0, 100.0 * ( shares.get(raw_leader, 0.0) - shares.get(favored, 0.0) ), ) if pair_gap_points > 0.0: contradiction_severity = min( 1.0, pair_gap_points / 0.75, ) # Better-validated experts retain more residual authority even # when contradicting the new regime; weak experts are penalized # more aggressively. quality_resistance = ( 0.86 - 0.30 * max(0.0, min(1.0, quality)) ) contradiction_penalty = ( (0.58 + 0.92 * turning_strength) * max(0.50, activation) * (0.58 + 0.42 * contradiction_severity) * quality_resistance ) adj -= contradiction_penalty turning_contradiction_penalties[key] = ( contradiction_penalty ) turning_pair_gaps[key] = pair_gap_points if ( consensus.get("eligible") and consensus.get( "candidate" ) == leader and maturity >= 0.30 ): consensus_strength = min( 1.0, float( consensus.get("score") or 0.0 ) / max( 0.055, float( consensus.get( "required_score" ) or 0.075 ), ), ) adj += ( 0.62 * activation * consensus_strength * (0.50 + 0.50 * quality) ) adjustments[key] = adj logits[key] = ( math.log(max(1e-8, weight)) + adj ) if logits: max_log = max(logits.values()) raw = { key: math.exp(value - max_log) for key, value in logits.items() } total = sum(raw.values()) new_weights = { key: value / total for key, value in raw.items() } new_weights = _apply_weight_caps( new_weights, preliminary_gate.get("caps") or {}, ) else: new_weights = pre_weights # -------------------------------------------------------------- # V6.2.2 earned-trust reallocation. # # Softmax penalties alone can be neutralized by ordinary expert caps: the # strongest agreeing Completion expert hits its cap and the freed weight is # redistributed right back into old-regime experts. When a Turning Point # has passed EVERY safeguard, allow a bounded amount of contradictory mass # to move only into independently validated experts that agree with the new # regime. No actionable turning point => exactly zero reallocation. # -------------------------------------------------------------- turning_trust_reallocation = { "available": bool(turning_actionable), "applied": False, "candidate": favored, "raw_leader": raw_leader, "requested_transfer": 0.0, "applied_transfer": 0.0, "supporters": [], "contradictors": [], "override_caps": {}, } if ( turning_actionable and favored and raw_leader and favored != raw_leader and new_weights ): supporters = [] contradictors = [] base_caps = ( preliminary_gate.get("caps") or {} ) override_caps = dict(base_caps) for key, weight in new_weights.items(): model = model_results.get(key) if not model or weight <= 0: continue shares = result_shares(model) ranked = sorted( names, key=lambda n: shares.get(n, 0.0), reverse=True, ) leader = ranked[0] if ranked else None hv = horizon.get(key) or {} quality = float( hv.get("final_quality") or 0.0 ) evidence = float( hv.get("final_evidence") or 0.0 ) if leader == favored: # Completion Benchmark has already passed its own independent # maturation exam, so it is always an eligible recipient here. validated = bool( key == "benchmark_completion" or ( quality >= 0.45 and evidence >= 0.45 ) ) if not validated: continue if key == "benchmark_completion": override_cap = min( 0.36, max( float( base_caps.get(key, 1.0) ), float( base_caps.get(key, 0.0) ) + 0.08 + 0.05 * turning_strength, ), ) else: override_cap = min( 0.27, max( float( base_caps.get(key, 1.0) ), float( base_caps.get(key, 0.0) ) + 0.04 + 0.03 * turning_strength, ), ) override_caps[key] = override_cap capacity = max( 0.0, override_cap - weight, ) supporters.append({ "model": key, "weight": weight, "quality": quality, "evidence": evidence, "capacity": capacity, "score": ( max(0.01, weight) * ( 0.65 + 0.35 * max( quality, float( ( model.get( "benchmark_completion" ) or {} ).get( "validation_quality" ) or 0.0 ), ) ) ), }) elif leader == raw_leader: pair_gap = max( 0.0, 100.0 * ( shares.get( raw_leader, 0.0 ) - shares.get( favored, 0.0 ) ), ) if pair_gap <= 0: continue contradictors.append({ "model": key, "weight": weight, "pair_gap_points": pair_gap, "severity": min( 1.0, pair_gap / 0.75 ), }) contradiction_mass = sum( row["weight"] for row in contradictors ) recipient_capacity = sum( row["capacity"] for row in supporters ) max_transfer = ( 0.22 * max(0.50, activation) * ( 0.65 + 0.35 * turning_strength ) ) desired_from_contradiction = ( contradiction_mass * ( 0.28 + 0.22 * turning_strength ) ) requested_transfer = min( max_transfer, desired_from_contradiction, ) applied_transfer = min( requested_transfer, recipient_capacity, ) if ( applied_transfer > 1e-9 and contradiction_mass > 1e-9 and supporters ): # Remove more from experts that contradict the new regime more # strongly, but retain a floor of authority for every model. removal_scores = { row["model"]: ( row["weight"] * ( 0.65 + 0.35 * row["severity"] ) ) for row in contradictors } removal_den = sum( removal_scores.values() ) removed = 0.0 for row in contradictors: key = row["model"] target_remove = ( applied_transfer * removal_scores[key] / max(1e-12, removal_den) ) # Never erase an expert entirely. actual_remove = min( target_remove, 0.75 * new_weights[key], ) new_weights[key] -= actual_remove row["removed_weight"] = ( actual_remove ) removed += actual_remove # Redistribute only to agreeing validated experts, respecting the # temporary Turning-Point override caps. remaining = removed for _ in range(6): eligible = [ row for row in supporters if ( override_caps[ row["model"] ] - new_weights[ row["model"] ] ) > 1e-10 ] if remaining <= 1e-10 or not eligible: break score_den = sum( row["score"] for row in eligible ) allocated = 0.0 for row in eligible: key = row["model"] capacity = max( 0.0, override_caps[key] - new_weights[key], ) target_add = ( remaining * row["score"] / max(1e-12, score_den) ) add = min( capacity, target_add ) new_weights[key] += add row["added_weight"] = ( row.get( "added_weight", 0.0 ) + add ) allocated += add if allocated <= 1e-12: break remaining -= allocated # Any tiny unallocated remainder is returned proportionally so the # gate remains normalized without silently deleting model mass. total_after = sum(new_weights.values()) if total_after > 0: new_weights = { key: value / total_after for key, value in new_weights.items() } turning_trust_reallocation[ "applied" ] = True turning_trust_reallocation[ "applied_transfer" ] = removed turning_trust_reallocation[ "requested_transfer" ] = requested_transfer turning_trust_reallocation[ "supporters" ] = supporters turning_trust_reallocation[ "contradictors" ] = contradictors turning_trust_reallocation[ "override_caps" ] = { key: override_caps[key] for key in override_caps if override_caps.get(key) != base_caps.get(key) } gate["weights_pre_benchmark"] = ( pre_weights ) gate["weights"] = new_weights gate[ "completion_benchmark_gate_adjustments" ] = adjustments gate[ "benchmark_gate_adjustments" ] = adjustments gate[ "turning_point_contradiction_penalties" ] = turning_contradiction_penalties gate[ "turning_point_pair_gaps_points" ] = turning_pair_gaps gate[ "turning_point_trust_reallocation" ] = turning_trust_reallocation gate[ "completion_benchmark_signal" ] = completion_benchmark_signal gate[ "benchmark_signal" ] = completion_benchmark_signal gate["crossover_consensus"] = consensus gate["maturity_info"] = maturity_info gate["path_benchmark_signal"] = path_benchmark_signal gate["turning_point"] = turning_point gate["persistent_completion_actionable"] = ( persistent_actionable ) gate["turning_point_actionable"] = ( turning_actionable ) gate["completion_benchmark_actionable"] = ( actionable ) gate["method_note"] = ( str( preliminary_gate.get( "method_note" ) or "" ) + " V6.2.2 final reweighting uses material-maturation Completion " "Benchmark evidence plus a separately validated Turning-Point layer. " "Path Benchmark remains 0% direct final weight and is only corroborating " "evidence inside the Turning-Point test. Once every Turning-Point " "safeguard passes, experts still favoring the old raw leader receive a " "bounded contradiction penalty. Influence ramps at ~30% " "estimated counted and reaches full strength at ~50%." ) for key, row in ( gate.get("experts") or {} ).items(): row[ "weight_pre_benchmark" ] = pre_weights.get(key, 0.0) row[ "benchmark_log_adjustment" ] = adjustments.get(key, 0.0) row[ "completion_benchmark_log_adjustment" ] = adjustments.get(key, 0.0) row[ "turning_point_contradiction_penalty" ] = turning_contradiction_penalties.get( key, 0.0 ) row[ "turning_point_pair_gap_points" ] = turning_pair_gaps.get(key, 0.0) row["weight"] = new_weights.get( key, 0.0 ) return gate def enrich_v6_benchmark_ranges( benchmarks: Dict[str, Any], monte_carlo: Dict[str, Any], ) -> Dict[str, Any]: """ Add benchmark uncertainty and approximate 50/80/90% win-probability targets using the signed statewide Monte Carlo margin distribution. """ if not benchmarks.get("available"): return benchmarks signed = ( monte_carlo.get( "signed_reference_margin_distribution_points" ) or {} ) reference = signed.get("reference_candidate") p10 = signed.get("p10") median = signed.get("median") p90 = signed.get("p90") final_total = float( benchmarks.get("expected_final_votes") or 0.0 ) if ( not reference or p10 is None or median is None or p90 is None or final_total <= 0 ): return benchmarks sigma_margin = max( 0.05, (float(p90) - float(p10)) / (2.0 * 1.2815515655446004), ) z_targets = { "50_percent_win": 0.0, "80_percent_win": 0.8416212335729143, "90_percent_win": 1.2815515655446004, } for row in benchmarks.get("counties") or []: remaining = float( row.get("estimated_remaining_votes") or 0.0 ) if remaining <= 1: continue for candidate, bench in ( row.get("candidate_benchmarks") or {} ).items(): tie = bench.get("tie_benchmark") or {} central = tie.get( "required_remaining_share_points" ) if central is None: continue sign = 1.0 if candidate == reference else -1.0 low_delta = sign * ( float(p10) - float(median) ) high_delta = sign * ( float(p90) - float(median) ) scale = final_total / (2.0 * remaining) req_a = central - low_delta * scale req_b = central - high_delta * scale lo = max( 0.0, min(100.0, min(req_a, req_b)) ) hi = max( 0.0, min(100.0, max(req_a, req_b)) ) tie["tie_benchmark_likely_range_points"] = { "p10": lo, "central": central, "p90": hi, } expected_remaining = float( bench.get( "expected_remaining_share_points" ) or 0.0 ) candidate_median_margin = ( float(median) if candidate == reference else -float(median) ) leverage = float( bench.get( "statewide_margin_points_per_county_share_point" ) or 0.0 ) probability_targets = {} if leverage > 1e-9: for label, z in z_targets.items(): target_margin = z * sigma_margin share_shift = ( target_margin - candidate_median_margin ) / leverage required_share = ( expected_remaining + share_shift ) probability_targets[label] = { "required_remaining_share_points": required_share, "required_remaining_share_clamped_points": max( 0.0, min(100.0, required_share) ), "feasible": ( 0.0 <= required_share <= 100.0 ), "already_above_target_without_county": ( required_share < 0.0 ), "county_alone_cannot_reach_target": ( required_share > 100.0 ), } bench[ "approx_probability_benchmarks" ] = probability_targets county_map = { row["cid"]: row for row in benchmarks.get("counties") or [] } for collection in ( "top_leverage_counties", "most_pivotal_counties", ): for short in benchmarks.get(collection) or []: full = county_map.get(short.get("cid")) if not full: continue bench = ( full.get("candidate_benchmarks") or {} ).get( benchmarks.get("projected_leader") ) or {} short["benchmark_likely_range_points"] = ( (bench.get("tie_benchmark") or {}).get( "tie_benchmark_likely_range_points" ) ) short["approx_probability_benchmarks"] = ( bench.get( "approx_probability_benchmarks" ) ) benchmarks["monte_carlo_benchmark_context"] = { "reference_candidate": reference, "signed_margin_p10": p10, "signed_margin_median": median, "signed_margin_p90": p90, "approx_margin_sigma_points": sigma_margin, "note": ( "50/80/90% county probability benchmarks use a normal approximation " "to the signed statewide Monte Carlo margin distribution. They are " "scenario guides, while the tie benchmark is exact conditional on " "the deterministic expectations elsewhere." ), } return benchmarks def apply_v5_raw_guard( base_url: str, snapshot_name: str, names: List[str], raw_shares: Dict[str, float], mixture_shares: Dict[str, float], weights: Dict[str, float], gate: Dict[str, Any], completion_referee: Dict[str, Any], model_results: Dict[str, Dict[str, Any]], benchmark_signal: Optional[Dict[str, Any]] = None, crossover_consensus: Optional[Dict[str, Any]] = None, turning_point: Optional[Dict[str, Any]] = None, maturity_info: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ Do-no-harm guard. Adjustments away from the scoreboard must earn release through demonstrated final-completion skill or a validated specialist. This is not a raw-vote veto: strong runoff/precinct/other specialist evidence can fully release it. """ raw_rank = sorted(names, key=lambda n: raw_shares.get(n, 0.0), reverse=True) mix_rank = sorted(names, key=lambda n: mixture_shares.get(n, 0.0), reverse=True) raw_leader = raw_rank[0] if raw_rank else None mix_leader = mix_rank[0] if mix_rank else None stability = raw_board_stability(base_url, snapshot_name) hv = gate.get("horizon_validation") or {} weighted_quality = sum( weights.get(key, 0.0) * float((hv.get(key) or {}).get("final_quality") or 0.55) for key in weights ) raw_quality = float((hv.get("v0") or {}).get("final_quality") or 0.55) skill_advantage = weighted_quality - raw_quality # Require a meaningful validation advantage before generic corrections get # full freedom. skill_release = max(0.0, min(1.0, (skill_advantage + 0.015) / 0.14)) evidence = float(completion_referee.get("evidence") or 0.0) release = 0.20 + 0.55 * evidence * skill_release specialist_override = 0.0 specialist_details = [] for key in ( "runoff_transfer", "precinct", "latent", "histgeo", "batchbias", "benchmark_completion", ): if key not in model_results: continue model_shares = result_shares(model_results[key]) ranked = sorted( names, key=lambda n: model_shares.get(n, 0.0), reverse=True, ) if not ranked or ranked[0] == raw_leader: continue hv_row = hv.get(key) or {} score = float( hv_row.get("final_quality") or 0.0 ) joint_evidence = float( hv_row.get("final_evidence") or 0.0 ) if score < 0.43 or joint_evidence < 0.30: override = 0.0 else: override = ( score * joint_evidence * ( 0.45 + 0.55 * weights.get(key, 0.0) / max( 0.05, weights.get(key, 0.0) + 0.10, ) ) ) specialist_override = max( specialist_override, override ) specialist_details.append({ "model": key, "leader": ranked[0], "joint_final_validation_score": score, "joint_final_validation_evidence": joint_evidence, "override": override, }) release = max(release, min(0.98, 0.15 + 0.95 * specialist_override)) benchmark_signal = benchmark_signal or {} crossover_consensus = crossover_consensus or {} turning_point = turning_point or {} maturity_info = maturity_info or {} benchmark_release = 0.0 consensus_release = 0.0 maturity_fraction = float( maturity_info.get( "counted_fraction_estimate" ) or 0.0 ) maturity_activation = float( maturity_info.get("activation") or 0.0 ) benchmark_candidate = benchmark_signal.get( "favored_candidate" ) benchmark_evidence = float( benchmark_signal.get("evidence") or 0.0 ) benchmark_score = abs( float( benchmark_signal.get( "favored_signal_score" ) or 0.0 ) ) persistent_benchmark_actionable = bool( benchmark_signal.get( "persistent_actionable_pre_maturity" ) and maturity_fraction >= 0.30 ) turning_actionable = bool( turning_point.get("actionable") and turning_point.get("candidate") == benchmark_candidate ) benchmark_actionable = bool( persistent_benchmark_actionable or turning_actionable ) if ( benchmark_candidate and benchmark_candidate != raw_leader and benchmark_actionable ): benchmark_release = min( 0.92, ( 0.28 + 0.42 * benchmark_evidence + 0.22 * min( 1.0, benchmark_score / 0.40, ) ) * max(0.35, maturity_activation), ) release = max( release, benchmark_release ) if ( crossover_consensus.get("eligible") and crossover_consensus.get( "candidate" ) != raw_leader and maturity_fraction >= 0.30 ): consensus_strength = min( 1.0, float( crossover_consensus.get( "score" ) or 0.0 ) / max( 0.055, float( crossover_consensus.get( "required_score" ) or 0.075 ), ), ) consensus_release = min( 0.96, ( 0.36 + 0.42 * consensus_strength + 0.12 * benchmark_evidence ) * max( 0.45, maturity_activation ), ) release = max( release, consensus_release ) turning_release = 0.0 raw_top_two_margin = ( 100.0 * ( raw_shares.get(raw_rank[0], 0.0) - raw_shares.get(raw_rank[1], 0.0) ) if len(raw_rank) > 1 else None ) turning_candidate_gap = turning_point.get( "raw_gap_to_candidate_points" ) if ( turning_actionable and benchmark_candidate and benchmark_candidate != raw_leader and crossover_consensus.get("eligible") and crossover_consensus.get("candidate") == benchmark_candidate and turning_candidate_gap is not None and float(turning_candidate_gap) <= ( 0.75 if maturity_fraction < 0.50 else 0.50 ) ): turn_strength = float( turning_point.get("score") or 0.0 ) turning_release = min( 0.98, 0.80 + 0.14 * turn_strength + ( 0.03 if maturity_fraction >= 0.50 else 0.0 ), ) release = max(release, turning_release) if raw_leader == mix_leader: release = max(release, 0.55) if ( maturity_fraction >= 0.50 and benchmark_actionable and crossover_consensus.get("eligible") and benchmark_candidate == crossover_consensus.get("candidate") and benchmark_candidate != raw_leader ): release = max(release, 0.94) # Very unstable raw boards get a little less veto power, but only a little. release += 0.10 * (1.0 - float(stability.get("score") or 0.35)) release = max(0.10, min(1.0, release)) guarded = _normalized({ n: (1.0 - release) * raw_shares.get(n, 0.0) + release * mixture_shares.get(n, 0.0) for n in names }, names) return { "shares": guarded, "release_fraction": release, "raw_leader": raw_leader, "mixture_leader": mix_leader, "raw_stability": stability, "weighted_final_quality": weighted_quality, "raw_final_quality": raw_quality, "skill_advantage": skill_advantage, "completion_evidence": evidence, "specialist_override": specialist_override, "specialist_details": specialist_details, "benchmark_release": benchmark_release, "consensus_release": consensus_release, "turning_point_release": turning_release, "benchmark_candidate": benchmark_candidate, "benchmark_evidence": benchmark_evidence, "persistent_benchmark_actionable": ( persistent_benchmark_actionable ), "turning_point_actionable": turning_actionable, "benchmark_actionable": benchmark_actionable, "turning_point": turning_point, "maturity_info": maturity_info, "crossover_consensus": crossover_consensus, "method_note": ( "The raw scoreboard is a guarded baseline, not a veto. V6.2.2 releases " "it through demonstrated final-completion skill, joint specialist temporal " "survival, actionable Completion Benchmark evidence, validated Turning-Point " "Consensus, or independent " "completion-validated expert consensus." ), } def _calibrate_county_centers_to_target( counties: List[Dict[str, Any]], current_candidates: Dict[str, Dict[str, Any]], names: List[str], target_shares: Dict[str, float], ) -> Dict[str, Any]: """ Iterative proportional fitting so expected simulated completion is centered on the deterministic Needle projection. """ current_votes = {n: float((current_candidates.get(n) or {}).get("votes") or 0.0) for n in names} current_total = sum(current_votes.values()) remaining_total = sum(float(c.get("remaining") or 0.0) for c in counties) final_total = current_total + remaining_total desired = { n: max(0.0, target_shares.get(n, 0.0) * final_total - current_votes[n]) for n in names } desired_sum = sum(desired.values()) if desired_sum <= 0 or remaining_total <= 0: return { "available": False, "baseline_center": shares_from_votes(current_candidates), "target": target_shares, } desired = { n: remaining_total * desired[n] / desired_sum for n in names } factors = {n: 1.0 for n in names} for _ in range(18): expected = {n: 0.0 for n in names} for c in counties: adjusted = _normalized({ n: c["central"].get(n, 0.0) * factors[n] for n in names }, names) for n in names: expected[n] += float(c.get("remaining") or 0.0) * adjusted[n] for n in names: if expected[n] > 1e-9 and desired[n] > 0: ratio = max(0.25, min(4.0, desired[n] / expected[n])) factors[n] *= ratio ** 0.60 for c in counties: c["central"] = _normalized({ n: c["central"].get(n, 0.0) * factors[n] for n in names }, names) expected_final = dict(current_votes) for c in counties: for n in names: expected_final[n] += float(c.get("remaining") or 0.0) * c["central"].get(n, 0.0) baseline_center = _normalized(expected_final, names) return { "available": True, "target": target_shares, "baseline_center": baseline_center, "candidate_factors": factors, "max_center_drift_points": 100.0 * max( abs(baseline_center.get(n, 0.0) - target_shares.get(n, 0.0)) for n in names ), } def monte_carlo_finish( snapshot: Dict[str, Any], names: List[str], model_results: Dict[str, Dict[str, Any]], weights: Dict[str, float], market_map_source: Optional[str] = None, simulations: int = 10000, seed_text: str = "", completion_referee: Optional[Dict[str, Any]] = None, path_referee: Optional[Dict[str, Any]] = None, target_projection: Optional[Dict[str, float]] = None, horizon_validation: Optional[Dict[str, Any]] = None, reporting_profile: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ V5 county/market Monte Carlo. The deterministic Needle is the center. Referee residuals determine uncertainty, while turnout means are blended across available experts. """ simulations = max(1000, min(50000, int(simulations or 10000))) current_candidates = candidate_map(snapshot) current_total = sum(int(current_candidates[n].get("votes") or 0) for n in names) if current_total <= 0: return {"available": False, "simulations": 0} completion_referee = completion_referee or {} path_referee = path_referee or {} horizon_validation = horizon_validation or {} reporting_profile = reporting_profile or {} completion_evidence = max(0.0, min(1.0, float(completion_referee.get("evidence") or 0.0))) path_evidence = max(0.0, min(1.0, float(path_referee.get("evidence") or 0.0))) state = snapshot.get("province") mapping = load_media_market_map(market_map_source, state) if market_map_source else None snapshot_counties = {_region_identity(r): r for r in county_regions(snapshot)} component_maps = { key: _row_share_map(_model_county_rows(key, result), names) for key, result in model_results.items() if key != "temporal" } # Collect turnout estimates from every expert that exposes county remaining vote. remaining_sources: Dict[str, List[tuple]] = {} for key, result in model_results.items(): if key == "temporal" or weights.get(key, 0.0) <= 0: continue for row in _model_county_rows(key, result): cid = str(row.get("cid") or row.get("fips") or "").strip() if not cid: continue rem = row.get("estimated_remaining_votes") if rem is None: continue try: rem = max(0.0, float(rem)) except (TypeError, ValueError): continue validation = horizon_validation.get(key) or {} quality = float(validation.get("final_quality") or 0.55) source_weight = max(0.02, weights.get(key, 0.0)) * (0.50 + quality) remaining_sources.setdefault(cid, []).append((rem, source_weight, key)) statewide_raw = shares_from_votes(current_candidates) counties = [] for cid, region in snapshot_counties.items(): current_votes = _candidate_votes_in_region(region, names) live_total = sum(current_votes.values()) p = _county_reporting_fraction(snapshot, cid) source_rows = remaining_sources.get(cid) or [] if source_rows: mean_remaining = _weighted_mean([(r, w) for r, w, _ in source_rows], 0.0) source_mean = mean_remaining source_dispersion = math.sqrt( _weighted_mean([ ((r - source_mean) ** 2, w) for r, w, _ in source_rows ], 0.0) ) turnout_cv = source_dispersion / max(100.0, source_mean) else: mean_remaining = ( max(0.0, live_total / max(0.05, p) - live_total) if live_total > 0 and p > 0 else 0.0 ) turnout_cv = 0.65 if mean_remaining > 0 else 0.0 central = {n: 0.0 for n in names} used_weight = 0.0 for key, weight in weights.items(): if weight <= 0 or key not in model_results: continue shares = None if key == "v0": shares = statewide_raw elif key == "v1": if live_total > 0: shares = _normalized(current_votes, names) else: shares = (component_maps.get(key) or {}).get(cid) if shares is None: # A statewide expert can contribute, but less strongly than # a genuine county prediction. shares = result_shares(model_results[key]) weight *= 0.55 if shares is None: continue shares = _normalized(shares, names) for n in names: central[n] += weight * shares.get(n, 0.0) used_weight += weight central = _normalized(central, names) if used_weight > 0 else statewide_raw market_name = market_for_region(region, mapping) if mapping and mapping.get("available") else None counties.append({ "cid": cid, "region": region, "current_votes": current_votes, "current_total": live_total, "remaining": mean_remaining, "turnout_cv": max(0.05, min(1.20, turnout_cv)), "p": p, "central": central, "market": market_name, }) # Statewide fallback if county rows are unavailable. if not counties: estimates = [] for key, result in model_results.items(): if key == "temporal" or weights.get(key, 0.0) <= 0: continue d = result.get("v3") or result.get("runoff_transfer") or result.get("latent_coalition") or {} rem = d.get("estimated_remaining_votes") if rem is not None: estimates.append((float(rem), weights.get(key, 0.0))) fallback_remaining = max(0.0, _weighted_mean(estimates, 0.0)) if fallback_remaining > 0: counties.append({ "cid": "STATEWIDE", "region": {"name": "Statewide", "type": "State"}, "current_votes": {n: 0 for n in names}, "current_total": 0, "remaining": fallback_remaining, "turnout_cv": 0.45, "p": statewide_reporting_maturity(snapshot), "central": _normalized({ n: sum( weights.get(k, 0.0) * result_shares(model_results[k]).get(n, 0.0) for k in weights if k in model_results ) for n in names }, names), "market": None, }) county_current = {n: 0 for n in names} for c in counties: for n in names: county_current[n] += c["current_votes"].get(n, 0) statewide_residual = { n: max(0, int(current_candidates[n].get("votes") or 0) - county_current[n]) for n in names } remaining_total = sum(c["remaining"] for c in counties) # Emergency live-data sanity guard. Historical/reference models should # normally size the outstanding vote. If they fail and the provider is # still in the very early reporting stage, do not allow the simulation # engine to treat the race as effectively complete. try: source_reporting_fraction = min( 1.0, max( 0.0, float(snapshot.get("percent_reporting") or 0.0) / 100.0, ), ) except (TypeError, ValueError): source_reporting_fraction = 0.0 counted_fraction_est = current_total / max( 1.0, current_total + remaining_total, ) if 0.0 < source_reporting_fraction < 0.10: # This is deliberately a loose safeguard rather than an assertion that # precinct/reporting-unit percentage equals vote percentage. At 2% # provider reporting, for example, the model may estimate as much as # ~6% counted, but it may not call the race 100% counted due to a # missing/broken turnout reference. early_counted_cap = max( 0.05, min(0.25, 3.0 * source_reporting_fraction), ) if counted_fraction_est > early_counted_cap: required_final_total = current_total / early_counted_cap safeguard_remaining = max( 0.0, required_final_total - (current_total + remaining_total), ) if safeguard_remaining > 0: counties.append({ "cid": "STATEWIDE_EARLY_REPORTING_SAFEGUARD", "region": { "name": "Statewide early-reporting safeguard", "type": "State", }, "current_votes": {n: 0 for n in names}, "current_total": 0, "remaining": safeguard_remaining, "turnout_cv": 0.60, "p": source_reporting_fraction, "central": _normalized( target_projection or { n: ( int(current_candidates[n].get("votes") or 0) / max(1, current_total) ) for n in names }, names, ), "market": None, "safeguard": True, }) remaining_total += safeguard_remaining counted_fraction_est = current_total / max( 1.0, current_total + remaining_total, ) target_projection = _normalized(target_projection or { n: sum( weights.get(k, 0.0) * result_shares(model_results[k]).get(n, 0.0) for k in weights if k in model_results ) for n in names }, names) calibration = _calibrate_county_centers_to_target( counties, current_candidates, names, target_projection, ) baseline_center = calibration.get("baseline_center") or target_projection # Model residuals drive uncertainty. A model that is doing badly live adds # width rather than merely losing weight. weighted_residual_mae_points = sum( weights.get(key, 0.0) * float((horizon_validation.get(key) or {}).get("residual_mae_points") or 6.0) for key in weights ) residual_share = max(0.012, min(0.15, weighted_residual_mae_points / 100.0)) seed = int(hashlib.sha256((seed_text or str(current_total)).encode("utf-8")).hexdigest()[:16], 16) rng = random.Random(seed) winner_counts = {n: 0 for n in names} margins = [] candidate_share_samples = {n: [] for n in names} target_rank = sorted( names, key=lambda n: target_projection.get(n, 0.0), reverse=True, ) signed_reference_candidate = target_rank[0] if target_rank else None signed_reference_margins = [] for _ in range(simulations): sim = {n: float(statewide_residual[n]) for n in names} for c in counties: for n in names: sim[n] += c["current_votes"].get(n, 0) maturity = counted_fraction_est global_sigma = ( 0.010 + 0.90 * residual_share * ((1.0 - maturity) ** 0.70) ) * (1.08 - 0.20 * completion_evidence) global_shock = {n: rng.gauss(0.0, global_sigma) for n in names} market_shocks = {} for c in counties: rem = c["remaining"] if rem <= 0: continue p = c["p"] empirical_cv = c.get("turnout_cv") or 0.35 turnout_sigma = ( 0.06 + 0.55 * empirical_cv * ((1.0 - p) ** 0.70) ) * (1.05 - 0.15 * completion_evidence) turnout_sigma = max(0.04, min(0.75, turnout_sigma)) factor = math.exp(rng.gauss(-0.5 * turnout_sigma * turnout_sigma, turnout_sigma)) sim_remaining = rem * max(0.20, min(3.0, factor)) market = c["market"] or "__unmapped__" if market not in market_shocks: market_sigma = ( 0.006 + 0.45 * residual_share * ((1.0 - maturity) ** 0.80) ) * (1.06 - 0.18 * completion_evidence) market_shocks[market] = { n: rng.gauss(0.0, market_sigma) for n in names } logits = {} for n in names: base = max(1e-6, c["central"].get(n, 0.0)) logits[n] = ( math.log(base) + global_shock[n] + market_shocks[market][n] ) max_logit = max(logits.values()) probs = _normalized({ n: math.exp(logits[n] - max_logit) for n in names }, names) concentration = max( 16.0, 1.0 / max( 0.001, (0.55 * residual_share + 0.035 * (1.0 - p)) ** 2, ), ) concentration = min(850.0, concentration) gammas = { n: rng.gammavariate( max(0.05, concentration * probs[n]), 1.0, ) for n in names } draw = _normalized(gammas, names) for n in names: sim[n] += sim_remaining * draw[n] total = sum(sim.values()) raw_sim_shares = {n: sim[n] / total for n in names} if total > 0 else {n: 0.0 for n in names} # Explicit deterministic recentering. Keep stochastic deviations from the # calibrated base, but anchor the simulation cloud on the V5 point estimate. adjusted = _normalized({ n: max( 1e-7, raw_sim_shares.get(n, 0.0) + target_projection.get(n, 0.0) - baseline_center.get(n, 0.0), ) for n in names }, names) ranked = sorted(names, key=lambda n: adjusted[n], reverse=True) if ranked: winner_counts[ranked[0]] += 1 for n in names: candidate_share_samples[n].append(100.0 * adjusted[n]) if len(ranked) > 1: margins.append(100.0 * (adjusted[ranked[0]] - adjusted[ranked[1]])) if signed_reference_candidate: reference_opp = max( ( adjusted[n] for n in names if n != signed_reference_candidate ), default=0.0, ) signed_reference_margins.append( 100.0 * ( adjusted[signed_reference_candidate] - reference_opp ) ) def quantile(values, q): if not values: return None vals = sorted(values) pos = q * (len(vals) - 1) lo = int(math.floor(pos)) hi = int(math.ceil(pos)) if lo == hi: return vals[lo] return vals[lo] + (vals[hi] - vals[lo]) * (pos - lo) win_probs = {n: winner_counts[n] / simulations for n in names} top = max(win_probs, key=win_probs.get) if win_probs else None central_shares = {n: quantile(candidate_share_samples[n], 0.5) for n in names} # ------------------------------------------------------------------ # Reporting-path simulator. It uses Path Referee + empirical dump profile. # ------------------------------------------------------------------ temporal_info = (model_results.get("temporal") or {}).get("temporal") or {} recent_sizes = [ int(x.get("new_votes") or 0) for x in (temporal_info.get("intervals") or []) if int(x.get("new_votes") or 0) > 0 ] typical_dump = ( float(reporting_profile.get("median_dump_votes") or 0.0) or (quantile(recent_sizes, 0.5) if recent_sizes else 0.0) or max(500.0, min(0.05 * max(1.0, remaining_total), 25000.0)) ) typical_dump = max(250.0, typical_dump) dump_sigma = float(reporting_profile.get("dump_log_sigma") or 0.45) dump_sigma = max(0.18, min(0.80, dump_sigma)) ensemble_state = target_projection reference_candidate = max(names, key=lambda n: ensemble_state.get(n, 0.0)) if names else None path_best = path_referee.get("best_recent_predictor") path_best_state = ( result_shares(model_results[path_best]) if path_best in model_results else ensemble_state ) temporal_rows = { str(r.get("market")): r for r in (temporal_info.get("media_markets") or []) if r.get("market") } market_activity_profile = reporting_profile.get("market_activity_votes") or {} market_component = (model_results.get("market") or {}).get("media_markets") or {} market_rows = market_component.get("markets") or [] def path_center_for_market(market_name: str) -> Dict[str, float]: if path_best == "temporal": tr = temporal_rows.get(market_name) or {} raw = tr.get("projected_remaining_shares") or {} if raw: return _normalized( {n: float(raw.get(n) or 0.0) / 100.0 for n in names}, names, ) relevant = [ c for c in counties if str(c.get("market") or "") == market_name and float(c.get("remaining") or 0.0) > 0 ] if path_best == "v1" and relevant: accum = {n: 0.0 for n in names} den = 0.0 for c in relevant: if c["current_total"] <= 0: continue shares = _normalized(c["current_votes"], names) w = max(1.0, float(c["remaining"])) for n in names: accum[n] += w * shares[n] den += w if den > 0: return _normalized(accum, names) if path_best in component_maps and relevant: accum = {n: 0.0 for n in names} den = 0.0 for c in relevant: shares = (component_maps[path_best].get(c["cid"]) or path_best_state) w = max(1.0, float(c["remaining"])) for n in names: accum[n] += w * shares.get(n, 0.0) den += w if den > 0: return _normalized(accum, names) return _normalized(path_best_state, names) market_pool = {} for mr in market_rows: market = str(mr.get("market") or "") if not market: continue rem = sum( float(c.get("remaining") or 0.0) for c in counties if str(c.get("market") or "") == market ) if rem <= 0: continue center = path_center_for_market(market) center = _normalized({ n: path_evidence * center.get(n, 0.0) + (1.0 - path_evidence) * ensemble_state.get(n, 0.0) for n in names }, names) recent = float(market_activity_profile.get(market) or 0.0) if recent <= 0: recent = float( (temporal_rows.get(market) or {}).get("recent_votes") or 0.0 ) market_pool[market] = { "remaining": rem, "center": center, "recent": recent, } if not market_pool and remaining_total > 0: fallback_center = _normalized({ n: path_evidence * path_best_state.get(n, 0.0) + (1.0 - path_evidence) * ensemble_state.get(n, 0.0) for n in names }, names) market_pool["STATEWIDE"] = { "remaining": remaining_total, "center": fallback_center, "recent": typical_dump, } path_steps = [ {"leader_counts": {n: 0 for n in names}, "margins": [], "new_votes": []} for _ in range(3) ] path_rng = random.Random(seed ^ 0xA47F19D2) path_sims = min(3500, max(1500, simulations // 3)) for _ in range(path_sims): sim_counts = { n: float(current_candidates[n].get("votes") or 0) for n in names } pools = { m: { "remaining": v["remaining"], "center": v["center"], "recent": v["recent"], } for m, v in market_pool.items() } max_recent = max([v["recent"] for v in pools.values()] + [1.0]) for step in range(3): remaining_now = sum(v["remaining"] for v in pools.values()) if remaining_now <= 1: break dump = typical_dump * math.exp( path_rng.gauss(-0.5 * dump_sigma * dump_sigma, dump_sigma) ) dump = max(100.0, min(remaining_now, dump)) raw_alloc = {} for market, v in pools.items(): if v["remaining"] <= 0: continue activity = 0.25 + 0.75 * min(1.0, v["recent"] / max_recent) raw_alloc[market] = ( v["remaining"] * activity * math.exp(path_rng.gauss(0.0, 0.30)) ) alloc_sum = sum(raw_alloc.values()) if alloc_sum <= 0: break actual_dump = 0.0 for market, raw_w in raw_alloc.items(): v = pools[market] take = min(v["remaining"], dump * raw_w / alloc_sum) if take <= 0: continue concentration = 22.0 + 100.0 * path_evidence gammas = { n: path_rng.gammavariate( max(0.05, concentration * max(1e-5, v["center"].get(n, 0.0))), 1.0, ) for n in names } draw = _normalized(gammas, names) for n in names: sim_counts[n] += take * draw[n] v["remaining"] -= take actual_dump += take total_now = sum(sim_counts.values()) shares_now = {n: sim_counts[n] / total_now for n in names} ranked = sorted(names, key=lambda n: shares_now[n], reverse=True) if ranked: path_steps[step]["leader_counts"][ranked[0]] += 1 if reference_candidate: opp = max( (shares_now[n] for n in names if n != reference_candidate), default=0.0, ) path_steps[step]["margins"].append( 100.0 * (shares_now[reference_candidate] - opp) ) path_steps[step]["new_votes"].append(actual_dump) future_path = [] for idx, row in enumerate(path_steps, 1): sims_done = max(1, sum(row["leader_counts"].values())) future_path.append({ "step": idx, "median_new_votes": quantile(row["new_votes"], 0.5), "leader_probabilities": { n: row["leader_counts"][n] / sims_done for n in names }, "reference_candidate": reference_candidate, "reference_margin_p10": quantile(row["margins"], 0.10), "reference_margin_median": quantile(row["margins"], 0.50), "reference_margin_p90": quantile(row["margins"], 0.90), }) return { "available": True, "experimental": True, "simulations": simulations, "seed": seed, "estimated_counted_fraction": counted_fraction_est, "estimated_remaining_votes": remaining_total, "completion_referee_evidence": completion_evidence, "path_referee_evidence": path_evidence, "path_referee_best_model": path_referee.get("best_recent_predictor"), "deterministic_target_share_points": { n: 100.0 * target_projection.get(n, 0.0) for n in names }, "centering": calibration, "weighted_live_residual_mae_points": weighted_residual_mae_points, "win_probabilities": win_probs, "most_likely_winner": top, "most_likely_winner_probability": win_probs.get(top) if top else None, "candidate_median_share_points": central_shares, "winning_margin_distribution_points": { "p05": quantile(margins, 0.05), "p10": quantile(margins, 0.10), "p25": quantile(margins, 0.25), "median": quantile(margins, 0.50), "p75": quantile(margins, 0.75), "p90": quantile(margins, 0.90), "p95": quantile(margins, 0.95), }, "signed_reference_margin_distribution_points": { "reference_candidate": signed_reference_candidate, "p10": quantile(signed_reference_margins, 0.10), "p25": quantile(signed_reference_margins, 0.25), "median": quantile(signed_reference_margins, 0.50), "p75": quantile(signed_reference_margins, 0.75), "p90": quantile(signed_reference_margins, 0.90), }, "future_dump_path": { "available": bool(future_path), "simulations": path_sims, "typical_recent_dump_votes": typical_dump, "learned_dump_log_sigma": dump_sigma, "median_cadence_seconds": reporting_profile.get("median_cadence_seconds"), "steps": future_path, "note": ( "V6 reporting-path simulation uses the Path Referee plus a " "backward-looking empirical dump-size/cadence/geographic profile." ), }, "note": ( "V6 Monte Carlo is explicitly centered on the deterministic V6 Needle. " "County turnout disagreement and live validation residuals determine " "distribution width; Path Referee evidence is reserved for future dumps." ), } def project_v4( base_url: str, snapshot_name: str, reference_url: Optional[str] = None, market_map_source: Optional[str] = None, monte_carlo_simulations: int = 10000, benchmark_replay_intervals: int = 4, ) -> Dict[str, Any]: """ AFI Needle V6.2.2 — Split Path/Completion Benchmarks. V6.2.2 keeps dynamic county path-to-victory benchmarks but separates the short-horizon reporting-path exam from the material county-completion exam. Only Completion Benchmark evidence can enter the final Needle. Specialist trust now combines spatial validation with temporal projection survival, and final influence ramps sharply from ~30% to full strength at ~50%. """ base_url = slug_base(base_url) bundle = component_snapshot_bundle( base_url, snapshot_name, reference_url, market_map_source ) snapshot = bundle["snapshot"] final = bundle["final"] names = bundle["names"] resolved_market_map_source = bundle["resolved_market_map_source"] model_results = dict(bundle["models"]) v0 = model_results["v0"] v1 = model_results["v1"] v3 = model_results["v3"] v2r = model_results.get("v2r") market = model_results.get("market") temporal = model_results.get("temporal") histgeo = model_results.get("histgeo") latent = model_results.get("latent") batchbias = model_results.get("batchbias") runoff_transfer = model_results.get("runoff_transfer") precinct = model_results.get("precinct") maturity = statewide_reporting_maturity(snapshot) raw_shares = result_shares(v0) ref_url = normalize_reference_url(reference_url) or round1_baseline_url(base_url) reference = None if ref_url: try: reference = load_reference_for_snapshot(ref_url, snapshot) except Exception: reference = None fits = reference_fit_scores(snapshot, reference, names) # -------------------------------------------------------------- # Structural applicability priors. These are not final weights. # -------------------------------------------------------------- strengths = { "v0": 0.90 + 1.55 * maturity, "v1": 0.42 + 1.20 * maturity, "v3": 0.42, } geo = v1.get("geographic") or {} coverage = min( 1.0, max(0.0, float(geo.get("current_vote_coverage_pct") or 0.0) / 100.0), ) capped = int(geo.get("counties_expansion_capped") or 0) used = int(geo.get("counties_used") or 0) cap_penalty = 1.0 if used <= 0 else max(0.30, 1.0 - capped / max(1, used)) strengths["v1"] *= (0.45 + 0.55 * coverage) * cap_penalty v3d = v3.get("v3") or {} if v3d.get("reference_available"): candidate_factor = ( fits["candidate_fit"] if fits["candidate_fit_available"] else 0.42 ) strengths["v3"] = ( 0.35 + 1.05 * fits["turnout_fit"] + 1.10 * candidate_factor + 0.70 * maturity ) else: strengths["v3"] = 0.34 + 0.55 * maturity if v2r: # Round-1 specialist prior. It is intentionally meaningful but the # learned runoff-transfer expert can supersede it when validated. strengths["v2r"] = ( 0.48 + 0.55 * fits["turnout_fit"] + 0.35 * (fits["candidate_fit"] if fits["candidate_fit_available"] else 0.45) + 0.25 * maturity ) market_confidence = 0.0 if market: md = market.get("media_markets") or {} market_coverage = float(md.get("mapped_current_vote_coverage") or 0.0) market_progress = float(md.get("weighted_market_progress") or 0.0) market_coherence = float(md.get("market_coherence") or 0.0) total_market_influence = float(md.get("total_effective_influence") or 0.0) market_confidence = ( market_coverage * market_coherence * min( 1.0, 0.30 + 0.45 * math.sqrt(max(0.0, market_progress)) + 0.70 * min(1.0, total_market_influence), ) ) strengths["market"] = 0.20 + 1.25 * market_confidence histgeo_confidence = 0.0 if histgeo: hd = histgeo.get("historical_candidate_geo") or {} histgeo_confidence = float(hd.get("confidence") or 0.0) strengths["histgeo"] = 0.08 + 2.20 * histgeo_confidence latent_confidence = 0.0 if latent: ld = latent.get("latent_coalition") or {} latent_confidence = float(ld.get("confidence") or 0.0) strengths["latent"] = 0.06 + 2.00 * latent_confidence batch_confidence = 0.0 if batchbias: bd = batchbias.get("batch_bias") or {} batch_confidence = float(bd.get("confidence") or 0.0) strengths["batchbias"] = 0.05 + 1.25 * batch_confidence runoff_confidence = 0.0 if runoff_transfer: rd = runoff_transfer.get("runoff_transfer") or {} runoff_confidence = float(rd.get("confidence") or 0.0) cv_score = float(rd.get("cross_validation_score") or 0.20) # Race-type expert gets a substantial applicability prior, but only # held-out validation can turn it into a dominant expert. strengths["runoff_transfer"] = ( 0.55 + 2.60 * runoff_confidence + 1.10 * runoff_confidence * cv_score ) precinct_confidence = 0.0 if precinct: pd = precinct.get("precinct_hierarchy") or {} precinct_confidence = float(pd.get("confidence") or 0.0) strengths["precinct"] = 0.10 + 2.20 * precinct_confidence # -------------------------------------------------------------- # Separate short-path and final-completion validation. # -------------------------------------------------------------- path_referee = rolling_component_referee( base_url, snapshot_name, reference_url, resolved_market_map_source, max_intervals=4, ) completion_referee = completion_component_referee( base_url, snapshot_name, reference_url, resolved_market_map_source, max_windows=4, ) reporting_profile = learn_reporting_path_profile( base_url, snapshot_name, resolved_market_map_source, max_intervals=12, estimated_final_votes=float((v3d.get("estimated_final_votes") or 0.0)), ) final_model_results = { key: model for key, model in model_results.items() if key != "temporal" } maturity_info = v6_decision_maturity( snapshot, v3_result=v3 ) specialist_temporal = ( specialist_temporal_validation( base_url, snapshot_name, reference_url, resolved_market_map_source, max_intervals=3, ) ) # Base gate: no county-benchmark final influence yet. base_gate = v5_mixture_gate( final_model_results, strengths, completion_referee, path_referee, specialist_temporal=specialist_temporal, maturity_fraction=float( maturity_info.get( "counted_fraction_estimate" ) or 0.0 ), ) base_weights = base_gate["weights"] base_mix = {n: 0.0 for n in names} for key, weight in base_weights.items(): shares = result_shares( final_model_results[key] ) for n in names: base_mix[n] += ( weight * shares.get(n, 0.0) ) base_mix = _normalized( base_mix, names ) # -------------------------------------------------------------- # V6.2 splits county benchmarks exactly like Temporal: # PATH benchmark -> next-dump / UI only # COMPLETION benchmark -> final-result eligible # -------------------------------------------------------------- path_benchmark_signal = ( rolling_v6_benchmark_signal( base_url, snapshot_name, base_weights, reference_url, resolved_market_map_source, max_intervals=max( 2, min( 4, int( benchmark_replay_intervals or 4 ), ), ), ) ) completion_benchmark_signal = ( completion_v6_benchmark_signal( base_url, snapshot_name, base_weights, reference_url, resolved_market_map_source, max_windows=3, ) ) turning_point = v6_turning_point_signal( names, raw_shares, completion_benchmark_signal, path_benchmark_signal, final_model_results, base_gate.get("horizon_validation") or {}, maturity_info, ) benchmark_completion = ( project_v6_completion_benchmark_expert( base_url, snapshot_name, raw_shares, completion_benchmark_signal, path_benchmark_signal, turning_point, maturity_info, ) ) if benchmark_completion: final_model_results[ "benchmark_completion" ] = benchmark_completion bc = ( benchmark_completion.get( "benchmark_completion" ) or {} ) strengths[ "benchmark_completion" ] = ( 0.04 + 2.15 * float( bc.get("confidence") or 0.0 ) ) # Rebuild the main gate with temporal survival + the new Completion # Benchmark expert, then allow only Completion Benchmark evidence to do # the second-stage final reweighting. preliminary_gate = v5_mixture_gate( final_model_results, strengths, completion_referee, path_referee, specialist_temporal=specialist_temporal, maturity_fraction=float( maturity_info.get( "counted_fraction_estimate" ) or 0.0 ), ) gate = apply_v6_benchmark_gate( preliminary_gate, final_model_results, names, raw_shares, completion_benchmark_signal, path_benchmark_signal=path_benchmark_signal, turning_point=turning_point, maturity_info=maturity_info, ) weights = gate["weights"] share_maps = { key: result_shares(model) for key, model in final_model_results.items() } # The displayed county benchmark map is built from the underlying county # experts only. The statewide Completion Benchmark expert is deliberately # excluded to prevent a circular benchmark definition. county_model_results = { key: model for key, model in final_model_results.items() if key != "benchmark_completion" } county_weights = ( _v6_normalize_weights_for_models( weights, county_model_results, ) ) county_benchmarks = build_v6_county_benchmarks( base_url, snapshot_name, county_model_results, county_weights, horizon_validation=gate.get( "horizon_validation" ), include_recent=True, ) # -------------------------------------------------------------- # Cross-expert disagreement is now diagnostic/uncertainty, not a # simplistic popularity penalty. The gate already knows skill. # -------------------------------------------------------------- mixture = {n: 0.0 for n in names} for key, weight in weights.items(): shares = share_maps.get(key) or {} for n in names: mixture[n] += weight * shares.get(n, 0.0) mixture = _normalized(mixture, names) evidence_center = dict(mixture) disagreement = {} for key, shares in share_maps.items(): disagreement[key] = sum( abs(shares.get(n, 0.0) - evidence_center.get(n, 0.0)) for n in names ) / max(1, len(names)) raw_guard = apply_v5_raw_guard( base_url, snapshot_name, names, raw_shares, mixture, weights, gate, completion_referee, final_model_results, benchmark_signal=completion_benchmark_signal, crossover_consensus=gate.get("crossover_consensus"), turning_point=turning_point, maturity_info=maturity_info, ) projected = raw_guard["shares"] result = _score_projection( base_url, snapshot_name, snapshot, final, projected, "V6_2_2_TURNING_POINT_TRUST_ENSEMBLE", ( "V6.2.2 keeps Path Benchmark at 0% direct final weight, upgrades " "Completion Benchmark to an independent signed-margin expert, and " "adds a validated Turning-Point Consensus for genuine reporting-" "regime changes during close races." ), { "uses_final_results_for_projection": False, "v6_mixture_of_experts": True, "v6_dynamic_county_benchmarks": True, "v6_path_benchmark_final_weight": 0.0, "v6_completion_benchmark_final_eligible": True, "v6_completion_benchmark_independent_signed_margin": True, "v6_turning_point_consensus": True, "v6_turning_point_contradiction_penalty": True, "v6_turning_point_latest_window_dominance_required": True, "v6_benchmark_breadth_gate": True, "v6_crossover_consensus": True, "v6_specialist_spatial_plus_temporal_validation": True, "v6_serious_data_threshold_estimate": 0.30, "v6_full_strength_threshold_estimate": 0.50, "v6_path_referee_separate": True, "v6_completion_referee_informative_only": True, "v6_raw_guard": True, "v6_monte_carlo_centered_on_deterministic": True, "v6_media_market_source": resolved_market_map_source, "v6_reference_url": ref_url, }, ) labels = { "v0": "V0 Raw", "v1": "V1 County Geo", "v2r": "V2R Round 1", "v3": "V3 Remaining Vote", "market": "Media Markets", "histgeo": "Historical Coalition Geography", "latent": "Latent Coalition Factors", "batchbias": "Within-County Batch Bias", "runoff_transfer": "Learned Runoff Transfer", "precinct": "Precinct Hierarchy", "benchmark_completion": "Completion Benchmark", "temporal": "Recent Returns / Path", } components = [] horizon = gate.get("horizon_validation") or {} for key, model_result in final_model_results.items(): shares = share_maps[key] ranked = sorted(names, key=lambda n: shares.get(n, 0.0), reverse=True) leader = ranked[0] if ranked else None runner = ranked[1] if len(ranked) > 1 else None margin = ( 100.0 * (shares[leader] - shares[runner]) if leader and runner else None ) gate_row = (gate.get("experts") or {}).get(key) or {} components.append({ "model": key, "label": labels.get(key, key), "weight": weights.get(key, 0.0), "structural_strength": gate_row.get("structural_strength"), "gate_cap": gate_row.get("cap"), "validation_log_adjustment": gate_row.get("validation_log_adjustment"), "benchmark_log_adjustment": gate_row.get("benchmark_log_adjustment"), "turning_point_contradiction_penalty": gate_row.get( "turning_point_contradiction_penalty" ), "turning_point_pair_gap_points": gate_row.get( "turning_point_pair_gap_points" ), "weight_pre_benchmark": gate_row.get("weight_pre_benchmark"), "uncertainty_penalty": gate_row.get("uncertainty_penalty"), "unvalidated_penalty": gate_row.get("unvalidated_penalty"), "horizon_validation": horizon.get(key), "completion_referee": ( (completion_referee.get("models") or {}).get(key) ), "path_referee": ( (path_referee.get("models") or {}).get(key) ), "predicted_leader": leader, "projected_margin_points": margin, "weighted_margin_contribution_points": ( margin * weights.get(key, 0.0) if margin is not None else None ), "disagreement_points": 100.0 * disagreement.get(key, 0.0), }) components.sort(key=lambda row: row["weight"], reverse=True) completion_evidence = float(completion_referee.get("evidence") or 0.0) path_evidence = float(path_referee.get("evidence") or 0.0) weighted_residual = sum( weights.get(key, 0.0) * float((horizon.get(key) or {}).get("residual_mae_points") or 6.0) for key in weights ) weighted_quality = sum( weights.get(key, 0.0) * float((horizon.get(key) or {}).get("final_quality") or 0.55) for key in weights ) gate_entropy = -sum( w * math.log(max(1e-9, w)) for w in weights.values() ) / max(1e-9, math.log(max(2, len(weights)))) agreement = 1.0 - min( 1.0, sum( weights.get(key, 0.0) * disagreement.get(key, 0.0) for key in weights ) / 0.10, ) model_confidence = max( 0.0, min( 1.0, 0.18 * maturity + 0.18 * agreement + 0.22 * completion_evidence + 0.20 * weighted_quality + 0.10 * (1.0 - min(1.0, weighted_residual / 10.0)) + 0.05 * (1.0 - gate_entropy) + 0.07 * float( completion_benchmark_signal.get( "evidence" ) or 0.0 ) + 0.05 * max( runoff_confidence, histgeo_confidence, latent_confidence, batch_confidence, precinct_confidence, ), ), ) explanations = [] if components: top = components[0] explanations.append( f"{top['label']} currently has the largest V6 expert weight at " f"{100.0 * top['weight']:.1f}%." ) if completion_referee.get("available"): best_key = completion_referee.get("best_completion_predictor") best_row = (completion_referee.get("models") or {}).get(best_key) or {} explanations.append( f"Completion Referee has {100.0 * completion_evidence:.0f}% usable " f"evidence after informative-prediction coverage; " f"{labels.get(best_key, best_key)} has the best generic maturation score " f"at {float(best_row.get('mae_points') or 0.0):.2f} points." ) if runoff_transfer: rd = runoff_transfer.get("runoff_transfer") or {} explanations.append( f"Learned Runoff Transfer uses {int(rd.get('training_counties') or 0)} " f"live counties; held-out share MAE is " f"{float(rd.get('cross_validation_mae_points') or 0.0):.2f} points, " f"aggregate turnout-weighted margin MAE is " f"{float(rd.get('cross_validation_aggregate_margin_mae_points') or 0.0):.2f} points, " f"and specialist confidence is " f"{100.0 * float(rd.get('confidence') or 0.0):.0f}%." ) if latent: ld = latent.get("latent_coalition") or {} explanations.append( f"Latent coalition model compresses {int(ld.get('historical_patterns') or 0)} " f"historical candidate maps into {int(ld.get('factors') or 0)} factors." ) if batchbias: bd = batchbias.get("batch_bias") or {} explanations.append( f"Within-county reporting-order model has " f"{int(bd.get('informative_counties') or 0)} informative counties " f"and {100.0 * float(bd.get('confidence') or 0.0):.0f}% confidence." ) if path_benchmark_signal.get("available"): explanations.append( f"Path Benchmark currently favors " f"{path_benchmark_signal.get('favored_candidate')} with " f"{float(path_benchmark_signal.get('favored_weighted_excess_points') or 0.0):+.2f} " f"points short-horizon benchmark excess. It has 0% direct final weight." ) trust_reallocation = ( gate.get( "turning_point_trust_reallocation" ) or {} ) if trust_reallocation.get("applied"): explanations.append( f"Turning-Point Trust moved " f"{100.0 * float(trust_reallocation.get('applied_transfer') or 0.0):.1f}% " f"ensemble weight away from old-regime experts and into validated " f"{turning_point.get('candidate')} experts after every turning-point " f"safeguard passed." ) if turning_point.get("available"): explanations.append( f"Turning-Point Consensus for " f"{turning_point.get('candidate')} is " f"{'ACTIONABLE' if turning_point.get('actionable') else 'not actionable'} " f"(score {100.0 * float(turning_point.get('score') or 0.0):.0f}%). " f"Latest material window is " f"{100.0 * float(turning_point.get('latest_window_vote_share') or 0.0):.0f}% " f"of matured vote mass with " f"{float(turning_point.get('latest_window_lift_points') or 0.0):+.2f} " f"points statewide lift." ) if completion_benchmark_signal.get("available"): explanations.append( f"Completion Benchmark favors " f"{completion_benchmark_signal.get('favored_candidate')} with " f"{float(completion_benchmark_signal.get('favored_weighted_excess_points') or 0.0):+.2f} " f"points weighted maturation excess, " f"{float(completion_benchmark_signal.get('favored_implied_statewide_margin_lift_points') or 0.0):+.2f} " f"points implied statewide lift, and " f"{100.0 * float(completion_benchmark_signal.get('favored_temporal_persistence') or 0.0):.0f}% " f"temporal persistence." ) consensus = gate.get("crossover_consensus") or {} if consensus.get("available"): explanations.append( f"Crossover consensus currently points to " f"{consensus.get('candidate') or 'no candidate'} across " f"{int(consensus.get('independent_families') or 0)} independent " f"expert families" + (" and is actionable." if consensus.get("eligible") else ".") ) explanations.append( f"Raw guard released {100.0 * float(raw_guard.get('release_fraction') or 0.0):.0f}% " f"of the experts' requested correction away from the scoreboard." ) if path_referee.get("available"): best_key = path_referee.get("best_recent_predictor") best_row = (path_referee.get("models") or {}).get(best_key) or {} explanations.append( f"Path Referee separately favors {labels.get(best_key, best_key)} for " f"near-term reporting at {float(best_row.get('mae_points') or 0.0):.2f} " f"points MAE; this does not directly change final expert weights." ) v6_block = { "version": "V6.2.2", "architecture": "split_path_completion_benchmark_ensemble", "weights": weights, "structural_strengths": strengths, "components": components, "gate": gate, "preliminary_gate": preliminary_gate, "county_benchmarks": county_benchmarks, "path_benchmark_signal": path_benchmark_signal, "completion_benchmark_signal": completion_benchmark_signal, "benchmark_signal": completion_benchmark_signal, "turning_point": turning_point, "benchmark_completion_expert": ( (benchmark_completion or {}).get( "benchmark_completion" ) if benchmark_completion else None ), "crossover_consensus": gate.get("crossover_consensus"), "maturity_info": maturity_info, "specialist_temporal_validation": specialist_temporal, "horizon_validation": horizon, "completion_referee": completion_referee, "path_referee": path_referee, "raw_guard": raw_guard, "reporting_path_profile": reporting_profile, "model_disagreement_points": { key: 100.0 * value for key, value in disagreement.items() }, "model_confidence": model_confidence, "model_confidence_note": ( "Diagnostic model-quality score only. Candidate win probability is " "reported separately by Monte Carlo." ), "weighted_validation_residual_mae_points": weighted_residual, "weighted_final_quality": weighted_quality, "gate_entropy": gate_entropy, "reference_url": ref_url, "reference_library": ( reference.get("reference_library") if isinstance(reference, dict) else None ), "media_market_source": resolved_market_map_source, "media_market_available": bool(market), "temporal_available": bool(temporal), "temporal_role": "path_only", "specialists": { "runoff_transfer_available": bool(runoff_transfer), "latent_coalition_available": bool(latent), "batch_bias_available": bool(batchbias), "precinct_hierarchy_available": bool(precinct), "historical_candidate_geo_available": bool(histgeo), "completion_benchmark_available": bool(benchmark_completion), }, "why": explanations, } # Compatibility aliases keep older Test Lab consumers working. result["v6"] = v6_block result["v5"] = v6_block result["v4"] = v6_block result["county_benchmarks"] = county_benchmarks result["path_benchmark_signal"] = path_benchmark_signal result["completion_benchmark_signal"] = completion_benchmark_signal result["turning_point"] = turning_point # Compatibility alias now points to the final-oriented completion signal. result["benchmark_signal"] = completion_benchmark_signal result["monte_carlo"] = monte_carlo_finish( snapshot, names, final_model_results, weights, resolved_market_map_source, simulations=monte_carlo_simulations, seed_text=base_url + snapshot_name + "|V6.2.2-turning-point-trust", completion_referee=completion_referee, path_referee=path_referee, target_projection=projected, horizon_validation=horizon, reporting_profile=reporting_profile, ) county_benchmarks = enrich_v6_benchmark_ranges( county_benchmarks, result["monte_carlo"], ) result["county_benchmarks"] = county_benchmarks result["v6"]["county_benchmarks"] = county_benchmarks result["v5"]["county_benchmarks"] = county_benchmarks result["v4"]["county_benchmarks"] = county_benchmarks if market: result["media_markets"] = market.get("media_markets") if histgeo: result["historical_candidate_geo"] = histgeo.get("historical_candidate_geo") result["historical_candidate_geo_projection"] = { "predicted_leader": histgeo.get("predicted_leader"), "projected_margin_points": histgeo.get("projected_margin_points"), "projection": histgeo.get("projection"), } if latent: result["latent_coalition"] = latent.get("latent_coalition") result["latent_coalition_projection"] = { "predicted_leader": latent.get("predicted_leader"), "projected_margin_points": latent.get("projected_margin_points"), "projection": latent.get("projection"), } if batchbias: result["batch_bias"] = batchbias.get("batch_bias") result["batch_bias_projection"] = { "predicted_leader": batchbias.get("predicted_leader"), "projected_margin_points": batchbias.get("projected_margin_points"), "projection": batchbias.get("projection"), } if runoff_transfer: result["runoff_transfer"] = runoff_transfer.get("runoff_transfer") result["runoff_transfer_projection"] = { "predicted_leader": runoff_transfer.get("predicted_leader"), "projected_margin_points": runoff_transfer.get("projected_margin_points"), "projection": runoff_transfer.get("projection"), } if benchmark_completion: result["benchmark_completion_model"] = benchmark_completion result["benchmark_completion"] = ( benchmark_completion.get( "benchmark_completion" ) ) result[ "benchmark_completion_projection" ] = { "predicted_leader": ( benchmark_completion.get( "predicted_leader" ) ), "projected_margin_points": ( benchmark_completion.get( "projected_margin_points" ) ), "projection": ( benchmark_completion.get( "projection" ) ), } if precinct: result["precinct_hierarchy"] = precinct.get("precinct_hierarchy") if temporal: result["temporal"] = temporal.get("temporal") result["temporal_projection"] = { "predicted_leader": temporal.get("predicted_leader"), "projected_margin_points": temporal.get("projected_margin_points"), "projection": temporal.get("projection"), "role": "path_only", "final_ensemble_weight": 0.0, } return result def checkpoint_tolerance(target: float) -> float: if target <= 10: return 2.0 if target <= 30: return 5.0 return 7.5 def summarize_batch(rows, prefix): valid = [r for r in rows if r.get("eligible")] leaders = [r.get(prefix + "_leader_correct") for r in valid if r.get(prefix + "_leader_correct") is not None] margins = [abs(r[prefix + "_winner_margin_error_points"]) for r in valid if r.get(prefix + "_winner_margin_error_points") is not None] shares = [abs(r[prefix + "_winner_share_error_points"]) for r in valid if r.get(prefix + "_winner_share_error_points") is not None] maes = [r[prefix + "_candidate_share_mae_points"] for r in valid if r.get(prefix + "_candidate_share_mae_points") is not None] return { "eligible_tests": len(leaders) if leaders else (len(margins) if margins else 0), "checkpoint_rows_eligible": len(valid), "leader_accuracy": (sum(1 for x in leaders if x) / len(leaders) if leaders else None), "mean_absolute_winner_margin_error_points": (sum(margins) / len(margins) if margins else None), "mean_absolute_winner_share_error_points": (sum(shares) / len(shares) if shares else None), "mean_all_candidate_share_mae_points": (sum(maes) / len(maes) if maes else None), "worst_absolute_winner_margin_error_points": (max(margins) if margins else None), "within_1_point": (sum(1 for x in margins if x <= 1.0) / len(margins) if margins else None), "within_2_points": (sum(1 for x in margins if x <= 2.0) / len(margins) if margins else None), "within_5_points": (sum(1 for x in margins if x <= 5.0) / len(margins) if margins else None), } class WikipediaElectionTableParser(HTMLParser): """Small dependency-free parser for Wikipedia headings and HTML tables.""" def __init__(self) -> None: super().__init__(convert_charrefs=True) self.tables: List[Dict[str, Any]] = [] self.heading_stack: Dict[int, str] = {} self.heading_level: Optional[int] = None self.heading_parts: List[str] = [] self.title_parts: List[str] = [] self.in_title = False self.table_depth = 0 self.current_table: Optional[Dict[str, Any]] = None self.current_row: Optional[List[Dict[str, Any]]] = None self.current_cell: Optional[Dict[str, Any]] = None self.in_caption = False self.caption_parts: List[str] = [] def handle_starttag(self, tag: str, attrs) -> None: tag = tag.lower() attr = dict(attrs) if tag == "title" and self.table_depth == 0: self.in_title = True return if tag in ("h2", "h3", "h4", "h5") and self.table_depth == 0: self.heading_level = int(tag[1]) self.heading_parts = [] return if tag == "table": if self.table_depth == 0: context = [ self.heading_stack[level] for level in sorted(self.heading_stack) if self.heading_stack.get(level) ] self.current_table = { "context": context, "class": attr.get("class", ""), "caption": "", "rows": [], } self.table_depth += 1 return if self.table_depth != 1 or self.current_table is None: return if tag == "caption": self.in_caption = True self.caption_parts = [] elif tag == "tr": self.current_row = [] elif tag in ("td", "th") and self.current_row is not None: try: rowspan = max(1, int(attr.get("rowspan", "1") or "1")) except ValueError: rowspan = 1 try: colspan = max(1, int(attr.get("colspan", "1") or "1")) except ValueError: colspan = 1 self.current_cell = { "text_parts": [], "header": tag == "th", "rowspan": rowspan, "colspan": colspan, } elif tag == "br" and self.current_cell is not None: self.current_cell["text_parts"].append(" ") def handle_data(self, data: str) -> None: if self.in_title: self.title_parts.append(data) if self.heading_level is not None: self.heading_parts.append(data) if self.table_depth == 1: if self.in_caption: self.caption_parts.append(data) if self.current_cell is not None: self.current_cell["text_parts"].append(data) def handle_endtag(self, tag: str) -> None: tag = tag.lower() if tag == "title": self.in_title = False return if self.heading_level is not None and tag == f"h{self.heading_level}": text = clean_wiki_text(" ".join(self.heading_parts)) level = self.heading_level if text: self.heading_stack[level] = text for deeper in list(self.heading_stack): if deeper > level: self.heading_stack.pop(deeper, None) self.heading_level = None self.heading_parts = [] return if tag == "table" and self.table_depth > 0: self.table_depth -= 1 if self.table_depth == 0 and self.current_table is not None: self.tables.append(self.current_table) self.current_table = None self.current_row = None self.current_cell = None self.in_caption = False return if self.table_depth != 1 or self.current_table is None: return if tag in ("td", "th") and self.current_cell is not None: text = clean_wiki_text(" ".join(self.current_cell.pop("text_parts", []))) self.current_cell["text"] = text if self.current_row is not None: self.current_row.append(self.current_cell) self.current_cell = None elif tag == "tr": if self.current_row: self.current_table["rows"].append(self.current_row) self.current_row = None self.current_cell = None elif tag == "caption": self.current_table["caption"] = clean_wiki_text(" ".join(self.caption_parts)) self.in_caption = False self.caption_parts = [] @property def page_title(self) -> str: title = clean_wiki_text(" ".join(self.title_parts)) return re.sub(r"\s*-\s*Wikipedia\s*$", "", title, flags=re.I) def clean_wiki_text(value: Any) -> str: text = unescape(str(value or "")) text = re.sub(r"\[[a-z0-9]+\]", "", text, flags=re.I) text = re.sub(r"\[\d+\]", "", text) text = re.sub(r"\s+", " ", text).strip() return text def validate_wikipedia_url(url: str) -> str: url = str(url or "").strip() parsed = urlparse(url) host = (parsed.hostname or "").lower() if parsed.scheme not in ("http", "https") or not ( host == "wikipedia.org" or host.endswith(".wikipedia.org") ): raise ValueError("Only Wikipedia article URLs are accepted.") return url def fetch_wikipedia_html(url: str) -> str: url = validate_wikipedia_url(url) req = urllib.request.Request( url, headers={ "User-Agent": USER_AGENT, "Accept": "text/html,application/xhtml+xml", }, ) with urllib.request.urlopen(req, timeout=30) as r: final_url = r.geturl() validate_wikipedia_url(final_url) raw = r.read(12_000_001) if len(raw) > 12_000_000: raise ValueError("Wikipedia article is too large to scan safely.") return raw.decode("utf-8", errors="replace") def expand_wikipedia_rows(rows: List[List[Dict[str, Any]]]) -> List[List[Dict[str, Any]]]: """Expand rowspan/colspan into a rectangular-ish cell matrix.""" expanded: List[List[Dict[str, Any]]] = [] pending: Dict[int, Dict[str, Any]] = {} for source_row in rows: row: List[Dict[str, Any]] = [] col = 0 def fill_pending_until_free() -> None: nonlocal col while col in pending: item = pending[col] while len(row) <= col: row.append({"text": "", "header": False}) row[col] = dict(item["cell"]) item["remaining"] -= 1 if item["remaining"] <= 0: pending.pop(col, None) col += 1 for cell in source_row: fill_pending_until_free() colspan = max(1, int(cell.get("colspan") or 1)) rowspan = max(1, int(cell.get("rowspan") or 1)) simple = { "text": clean_wiki_text(cell.get("text")), "header": bool(cell.get("header")), } for _ in range(colspan): while len(row) <= col: row.append({"text": "", "header": False}) row[col] = dict(simple) if rowspan > 1: pending[col] = { "cell": dict(simple), "remaining": rowspan - 1, } col += 1 # Fill any pending cells that immediately follow this row's explicit cells. while col in pending: item = pending[col] while len(row) <= col: row.append({"text": "", "header": False}) row[col] = dict(item["cell"]) item["remaining"] -= 1 if item["remaining"] <= 0: pending.pop(col, None) col += 1 expanded.append(row) width = max((len(r) for r in expanded), default=0) for row in expanded: while len(row) < width: row.append({"text": "", "header": False}) return expanded def parse_vote_integer(value: Any) -> Optional[int]: text = clean_wiki_text(value) if not text or "%" in text: return None # Remove formatting, footnote symbols, winner markers, and parenthetical notes. text = re.sub(r"\([^)]*\)", "", text) text = text.replace(",", "").replace("†", "").replace("‡", "").replace("*", "") text = text.replace("−", "-").strip() if re.fullmatch(r"[+-]?\d+", text): try: return int(text) except ValueError: return None return None def wikipedia_candidate_name(header: str) -> str: parts = [clean_wiki_text(p) for p in header.split("|")] noise = { "votes", "vote", "#", "no.", "number", "popular vote", "democratic", "republican", "independent", "party", "candidate" } useful = [] for part in parts: p = part.strip(" -") low = p.casefold() if not p or low in noise or low in ("%", "percent", "percentage"): continue p = re.sub(r"\b(votes?|popular vote)\b", "", p, flags=re.I).strip(" -") p = re.sub(r"\b(democratic|republican|independent)\b", "", p, flags=re.I).strip(" -") if p and p.casefold() not in noise: useful.append(p) if not useful: return "" # Candidate is normally the upper/header-level label, not the subheader. return useful[0] def analyze_wikipedia_table(table: Dict[str, Any], table_id: int) -> Optional[Dict[str, Any]]: matrix = expand_wikipedia_rows(table.get("rows") or []) if len(matrix) < 4: return None # Find first data row. County rows usually have a textual first cell and # at least one or two integer-valued result cells. first_data = None for i, row in enumerate(matrix): texts = [clean_wiki_text(c.get("text")) for c in row] if not any(texts): continue numeric = sum(parse_vote_integer(x) is not None for x in texts[1:]) first = texts[0].casefold() if numeric >= 1 and first not in ("county", "counties", "parish", "borough"): first_data = i break if first_data is None or first_data == 0: return None header_rows = matrix[:first_data] data_rows = matrix[first_data:] width = max((len(r) for r in matrix), default=0) headers = [] for col in range(width): pieces = [] for hr in header_rows: if col >= len(hr): continue text = clean_wiki_text(hr[col].get("text")) if text and (not pieces or pieces[-1] != text): pieces.append(text) headers.append(" | ".join(pieces)) county_col = None for col, header in enumerate(headers): if re.search(r"\b(county|parish|borough|census area)\b", header, flags=re.I): county_col = col break if county_col is None: county_col = 0 # Identify usable data rows first. usable_rows = [] skip_names = { "total", "totals", "statewide", "state total", "all counties", "total votes", "grand total" } for row in data_rows: if county_col >= len(row): continue county = clean_wiki_text(row[county_col].get("text")) if not county or county.casefold() in skip_names: continue ints = [parse_vote_integer(c.get("text")) for c in row] if sum(v is not None for v in ints) < 1: continue usable_rows.append(row) if len(usable_rows) < 5: return None total_col = None candidate_cols = [] for col, header in enumerate(headers): if col == county_col: continue low = header.casefold() if re.search(r"\b(total votes?|votes cast|total ballots?)\b", low): total_col = col continue if any(token in low for token in ( "margin", "turnout", "registered", "eligible", "rejected", "blank ballot", "overvote", "undervote" )): continue if "%" in header or "percent" in low or "percentage" in low: continue vals = [ parse_vote_integer(row[col].get("text")) if col < len(row) else None for row in usable_rows ] good = [v for v in vals if v is not None and v >= 0] if len(good) < max(3, int(0.60 * len(usable_rows))): continue name = wikipedia_candidate_name(header) if name and name.casefold() not in ("total", "margin"): candidate_cols.append({"column": col, "name": name, "header": header}) # Deduplicate repeated candidate names, preferring the first integer column. dedup = {} for item in candidate_cols: dedup.setdefault(item["name"], item) candidate_cols = list(dedup.values()) # County-level turnout alone is still useful as a historical footprint. if not candidate_cols and total_col is None: return None context = [clean_wiki_text(x) for x in table.get("context") or [] if clean_wiki_text(x)] caption = clean_wiki_text(table.get("caption")) context_text = " > ".join(context) label_parts = context[:] if caption and (not label_parts or caption.casefold() != label_parts[-1].casefold()): label_parts.append(caption) label = " > ".join(label_parts) or f"Table {table_id}" all_text = f"{context_text} {caption}".casefold() if "democratic" in all_text: party_guess = "democratic" elif "republican" in all_text: party_guess = "republican" else: party_guess = None score = 0 if re.search(r"\bcounty\b", " ".join(headers), flags=re.I): score += 50 if "results" in all_text: score += 25 if "primary" in all_text: score += 20 if party_guess: score += 10 score += min(30, len(usable_rows) // 3) score += min(10, len(candidate_cols)) return { "id": table_id, "label": label, "context": context, "caption": caption, "party_guess": party_guess, "counties": len(usable_rows), "candidates": [x["name"] for x in candidate_cols], "candidate_columns": candidate_cols, "county_column": county_col, "total_column": total_col, "headers": headers, "score": score, } # --------------------------------------------------------------------------- # Official state-results file importer # --------------------------------------------------------------------------- _OFFICIAL_COLUMN_ALIASES = { "election_date": ["electiondate", "election_date", "elec_date", "date"], "office_description": ["officedescription", "officedesc", "office", "contestname", "racename", "race_description", "contest"], "office_code": ["officecode", "contestcode", "racecode", "race_number"], # Oklahoma CountyResults uses race_county_owner to distinguish local races. # Statewide races leave it blank, which is exactly what we want. "district_code": ["districtcode", "district", "districtnumber", "districtid", "race_county_owner", "juris1num", "juris1", "jurisdiction1"], "subdistrict_code": ["subdistrictcode", "seat", "seatnumber", "juris2num", "juris2", "jurisdiction2"], "party_description": ["partydescription", "partyname", "race_party", "party", "partycode"], "county_name": ["countyname", "county", "jurisdictionname", "jurisdiction"], "county_code": ["countycode", "countyid", "jurisdictioncode"], "candidate_name": ["candidatename", "candidate", "ballotname", "cand_name"], "candidate_first": ["candidatefirstname", "cannamefirst", "firstname", "first"], "candidate_middle": ["candidatemiddlename", "cannamemiddle", "middlename", "middle"], "candidate_last": ["candidatelastname", "cannamelast", "lastname", "last"], "candidate_id": ["candidateid", "candidatecode", "cand_number"], "candidate_votes": ["candidatevotes", "canvotes", "votecount", "votes", "vote", "cand_tot_votes"], # Optional vote-method metadata. These are preserved in imported references # but V6.2 does not require the live API to expose them. "candidate_absentee_votes": ["cand_absmail_votes", "absentee_mail_votes", "absentee_votes", "mail_votes"], "candidate_early_votes": ["cand_early_votes", "early_votes", "advance_votes"], "candidate_election_day_votes": ["cand_elecday_votes", "election_day_votes", "elecday_votes"], "total_precincts": ["tot_race_prec", "total_precincts", "precincts_total", "precincts"], "precincts_reporting": ["race_prec_reporting", "precincts_reporting", "reporting_precincts"], } def _official_norm_header(value: Any) -> str: return re.sub(r"[^a-z0-9]+", "", str(value or "").strip().casefold()) def _official_find_column(headers: List[str], aliases: List[str]) -> Optional[str]: normalized = {h: _official_norm_header(h) for h in headers} wanted = [_official_norm_header(a) for a in aliases] for alias in wanted: for original, norm in normalized.items(): if norm == alias: return original # Handles exports such as OfficeCode(text) -> officecodetext without # accidentally treating CandidateID as a generic Candidate column. for alias in wanted: for original, norm in normalized.items(): suffix = norm[len(alias):] if norm.startswith(alias) else "" if suffix in ("text", "string", "str"): return original return None def _official_parse_int(value: Any) -> int: text = str(value or "").strip() if not text: return 0 text = text.replace(",", "").replace(" ", "") match = re.search(r"-?\d+", text) if not match: return 0 try: return max(0, int(match.group(0))) except ValueError: return 0 def _official_title_name(value: Any) -> str: text = re.sub(r"\s+", " ", str(value or "").strip()) if not text: return "" # SOS exports commonly use ALL CAPS; title casing produces names much # closer to VoteFlow candidate naming while preserving apostrophes. if text.upper() == text: text = text.title() return text def _official_party_slug(value: Any) -> str: text = str(value or "").strip().casefold() if text in ("dem", "d") or "democrat" in text: return "democratic" if text in ("rep", "gop", "r") or "republic" in text: return "republican" return re.sub(r"[^a-z0-9]+", "_", text).strip("_") or "unknown" def _official_candidate_name(row: Dict[str, str], columns: Dict[str, Optional[str]]) -> str: direct = columns.get("candidate_name") if direct and str(row.get(direct) or "").strip(): return _official_title_name(row.get(direct)) parts = [] for key in ("candidate_first", "candidate_middle", "candidate_last"): col = columns.get(key) if col and str(row.get(col) or "").strip(): parts.append(str(row.get(col) or "").strip()) return _official_title_name(" ".join(parts)) def _official_vote_methods(row: Dict[str, str], columns: Dict[str, Optional[str]]) -> Dict[str, int]: out = {} for out_key, column_key in ( ("absentee_mail", "candidate_absentee_votes"), ("early", "candidate_early_votes"), ("election_day", "candidate_election_day_votes"), ): col = columns.get(column_key) if col: out[out_key] = _official_parse_int(row.get(col)) return out def _official_candidate_total_votes(row: Dict[str, str], columns: Dict[str, Optional[str]]) -> int: total_col = columns.get("candidate_votes") if total_col and str(row.get(total_col) or "").strip() != "": return _official_parse_int(row.get(total_col)) methods = _official_vote_methods(row, columns) return sum(methods.values()) def _official_has_vote_data(row: Dict[str, str], columns: Dict[str, Optional[str]]) -> bool: total_col = columns.get("candidate_votes") if total_col and str(row.get(total_col) or "").strip() != "": return True for key in ("candidate_absentee_votes", "candidate_early_votes", "candidate_election_day_votes"): col = columns.get(key) if col and str(row.get(col) or "").strip() != "": return True return False def _official_iowa_party_name(token: str) -> str: value = str(token or "").strip().casefold() if value in ("rep", "republican", "gop"): return "Republican" if value in ("dem", "democratic", "democrat"): return "Democratic" return _official_title_name(token) def _official_iowa_is_write_in(name: str) -> bool: text = re.sub( r"[^a-z]+", "", str(name or "").strip().casefold(), ) return text in { "writein", "writeins", "writeinvotes", } def _official_iowa_district_from_office( office_description: str, ) -> str: match = re.search( r"\bDistrict\s+([A-Za-z0-9-]+)\b", str(office_description or ""), flags=re.I, ) return match.group(1) if match else "" def _official_sc_party_name(token: str) -> str: value = str(token or "").strip().casefold() if value in ("rep", "republican", "gop"): return "Republican" if value in ("dem", "democratic", "democrat"): return "Democratic" return _official_title_name(token) def _official_sc_district_from_office( office_description: str, ) -> str: match = re.search( r"\bDistrict\s+([A-Za-z0-9-]+)\b", str(office_description or ""), flags=re.I, ) return match.group(1) if match else "" def _official_try_load_south_carolina_detail( path: Path, text: str, encoding: str, election_date_override: Optional[str] = None, ) -> Optional[Dict[str, Any]]: """ Parse South Carolina Election Commission fixed-width primary detail files. Candidate names are sliced using the repeated "Election Day" column starts rather than whitespace splitting. SC sometimes prints multiple spaces inside a candidate name, so position-based parsing is substantially safer. ElectionDate remains blank in synthetic rows so a manual date override never changes the stable race-selection ID. """ lines = text.splitlines() section_re = re.compile( r"^\s*(.*?)\s*-\s*" r"(DEM|REP)\s*" r"\(Vote\s+For\s+(\d+)\)\s*$", flags=re.I, ) section_indices = [ i for i, line in enumerate(lines) if section_re.match(line) ] if len(section_indices) < 2: return None synthetic_headers = [ "ElectionDate", "OfficeDesc", "DistrictCode", "PartyName", "CountyName", "CandidateName", "CandidateVotes", ] rows: List[Dict[str, str]] = [] parsed_sections = 0 malformed_sections = 0 counties_seen = set() for section_no, start in enumerate( section_indices ): end = ( section_indices[section_no + 1] if section_no + 1 < len(section_indices) else len(lines) ) match = section_re.match(lines[start]) if not match: continue office_desc = re.sub( r"\s+", " ", match.group(1).strip() ) party_name = _official_sc_party_name( match.group(2) ) vote_for = _official_parse_int( match.group(3) ) district_code = ( _official_sc_district_from_office( office_desc ) ) meaningful = [ idx for idx in range(start + 1, end) if lines[idx].strip() ] if len(meaningful) < 3: malformed_sections += 1 continue candidate_line_idx = meaningful[0] header_line_idx = meaningful[1] candidate_line = lines[ candidate_line_idx ] header_line = lines[ header_line_idx ] block_starts = [ m.start() for m in re.finditer( r"Election\s+Day", header_line, flags=re.I, ) ] if not block_starts: malformed_sections += 1 continue candidate_names = [] for candidate_idx, block_start in enumerate( block_starts ): block_end = ( block_starts[candidate_idx + 1] if candidate_idx + 1 < len(block_starts) else len(candidate_line) ) name = re.sub( r"\s+", " ", candidate_line[ block_start:block_end ].strip(), ) if name: candidate_names.append( name ) header_tokens = [ token.strip() for token in re.split( r"\s{2,}", header_line.strip(), ) if token.strip() ] if ( not candidate_names or len(header_tokens) < 5 or _official_norm_header( header_tokens[0] ) != "county" or _official_norm_header( header_tokens[1] ) != "registeredvoters" ): malformed_sections += 1 continue subheaders = header_tokens[2:-1] candidate_count = len( candidate_names ) if ( candidate_count <= 0 or len(subheaders) % candidate_count != 0 ): malformed_sections += 1 continue block_width = ( len(subheaders) // candidate_count ) if block_width <= 0: malformed_sections += 1 continue first_block = subheaders[ :block_width ] total_positions = [ idx for idx, value in enumerate( first_block ) if _official_norm_header( value ) in ("choicetotal", "total") ] if not total_positions: malformed_sections += 1 continue candidate_total_offset = ( total_positions[-1] ) section_rows = 0 for line_idx in range( header_line_idx + 1, end, ): line = lines[line_idx] if not line.strip(): continue tokens = [ token.strip() for token in re.split( r"\s{2,}", line.strip(), ) if token.strip() ] if not tokens: continue if tokens[0].casefold().startswith( "totals:" ): break minimum_fields = ( 3 + candidate_count * block_width ) if len(tokens) < minimum_fields: continue county_name = tokens[0] counties_seen.add( normalize_county_name( county_name ) ) for candidate_idx, candidate_name in enumerate( candidate_names ): vote_idx = ( 2 + candidate_idx * block_width + candidate_total_offset ) if vote_idx >= len(tokens): continue votes = _official_parse_int( tokens[vote_idx] ) rows.append({ "ElectionDate": "", "OfficeDesc": ( office_desc ), "DistrictCode": ( district_code ), "PartyName": ( party_name ), "CountyName": ( county_name ), "CandidateName": ( candidate_name ), "CandidateVotes": ( str(votes) ), }) section_rows += 1 if section_rows: parsed_sections += 1 else: malformed_sections += 1 if parsed_sections < 2 or not rows: return None columns = { key: _official_find_column( synthetic_headers, aliases, ) for key, aliases in _OFFICIAL_COLUMN_ALIASES.items() } return { "path": str(path), "filename": path.name, "encoding": encoding, "delimiter": None, "delimiter_name": ( "fixed-width" ), "header_line": 0, "headers": synthetic_headers, "columns": columns, "rows": rows, "source_format_hint": ( "south_carolina_scec_detail_fixed_width" ), "south_carolina_detail": { "sections_detected": len( section_indices ), "sections_parsed": ( parsed_sections ), "malformed_sections": ( malformed_sections ), "unique_counties_seen": len( [ value for value in counties_seen if value ] ), "election_date_override": ( str( election_date_override or "" ).strip() or None ), }, } def _official_try_load_iowa_detail( path: Path, text: str, encoding: str, election_date_override: Optional[str] = None, ) -> Optional[Dict[str, Any]]: """ Parse Iowa Secretary of State fixed-width "detail.txt" election reports. Typical section: Auditor of State - Rep. ( 1) Todd Halbur Mary Ann Hanusa Write-in Registered Voters Polling Total Polling Total Polling Total Total Adair 0 343 343 ... The "( 1)" suffix is "vote for one", NOT a district/seat identifier. Write-in columns are excluded from the modeled candidate field but their votes are recorded in parser metadata. """ lines = text.splitlines() section_re = re.compile( r"^\s*(.*?)\s*-\s*" r"(Rep|Dem)\.\s*" r"\(\s*\d+\s*\)\s*$", flags=re.I, ) section_indices = [ i for i, line in enumerate(lines) if section_re.match(line) ] if len(section_indices) < 2: return None synthetic_headers = [ "ElectionDate", "OfficeDesc", "DistrictCode", "PartyName", "CountyName", "CandidateName", "CandidateVotes", ] rows: List[Dict[str, str]] = [] excluded_write_in_votes = 0 excluded_write_in_rows = 0 parsed_sections = 0 malformed_sections = 0 for section_no, start in enumerate( section_indices ): end = ( section_indices[section_no + 1] if section_no + 1 < len(section_indices) else len(lines) ) match = section_re.match(lines[start]) if not match: continue office_desc = re.sub( r"\s+", " ", match.group(1).strip() ) party_name = _official_iowa_party_name( match.group(2) ) district_code = ( _official_iowa_district_from_office( office_desc ) ) # First two non-empty lines after the section heading are: # candidate names, then repeated vote-method subheaders. meaningful = [ idx for idx in range(start + 1, end) if lines[idx].strip() ] if len(meaningful) < 3: malformed_sections += 1 continue candidate_line_idx = meaningful[0] header_line_idx = meaningful[1] candidate_names = [ token.strip() for token in re.split( r"\s{2,}", lines[candidate_line_idx].strip(), ) if token.strip() ] header_tokens = [ token.strip() for token in re.split( r"\s{2,}", lines[header_line_idx].strip(), ) if token.strip() ] if ( not candidate_names or len(header_tokens) < 4 or _official_norm_header( header_tokens[0] ) != "registeredvoters" ): malformed_sections += 1 continue # Strip the final all-candidate Total column, leaving repeated blocks # such as [Polling, Total] for each candidate. subheaders = header_tokens[1:-1] candidate_count = len(candidate_names) if ( candidate_count <= 0 or len(subheaders) % candidate_count != 0 ): malformed_sections += 1 continue block_width = ( len(subheaders) // candidate_count ) if block_width <= 0: malformed_sections += 1 continue first_block = subheaders[:block_width] total_positions = [ idx for idx, value in enumerate(first_block) if _official_norm_header(value) == "total" ] candidate_total_offset = ( total_positions[-1] if total_positions else block_width - 1 ) section_rows = 0 for line_idx in range( header_line_idx + 1, end ): line = lines[line_idx] if not line.strip(): continue tokens = [ token.strip() for token in re.split( r"\s{2,}", line.strip() ) if token.strip() ] if not tokens: continue if tokens[0].casefold().startswith( "totals:" ): break minimum_fields = ( 3 + candidate_count * block_width ) if len(tokens) < minimum_fields: continue county_name = tokens[0] for candidate_idx, candidate_name in enumerate( candidate_names ): vote_idx = ( 2 + candidate_idx * block_width + candidate_total_offset ) if vote_idx >= len(tokens): continue votes = _official_parse_int( tokens[vote_idx] ) if _official_iowa_is_write_in( candidate_name ): excluded_write_in_votes += votes excluded_write_in_rows += 1 continue rows.append({ "ElectionDate": "", "OfficeDesc": office_desc, "DistrictCode": district_code, "PartyName": party_name, "CountyName": county_name, "CandidateName": candidate_name, "CandidateVotes": str(votes), }) section_rows += 1 if section_rows: parsed_sections += 1 else: malformed_sections += 1 if parsed_sections < 2 or not rows: return None columns = { key: _official_find_column( synthetic_headers, aliases ) for key, aliases in _OFFICIAL_COLUMN_ALIASES.items() } return { "path": str(path), "filename": path.name, "encoding": encoding, "delimiter": None, "delimiter_name": "fixed-width", "header_line": 0, "headers": synthetic_headers, "columns": columns, "rows": rows, "source_format_hint": ( "iowa_sos_detail_fixed_width" ), "iowa_detail": { "sections_detected": len( section_indices ), "sections_parsed": parsed_sections, "malformed_sections": ( malformed_sections ), "excluded_write_in_votes": ( excluded_write_in_votes ), "excluded_write_in_rows": ( excluded_write_in_rows ), "election_date_override": ( str( election_date_override or "" ).strip() or None ), }, } def _official_ma_filename_metadata(path: Path) -> Dict[str, Optional[str]]: """ Infer the single-race metadata carried in Massachusetts PD43+ filenames. Example: PD43+__2024_President_Democratic_Primary.csv """ stem = path.stem cleaned = re.sub(r"^PD43\+\s*_+", "", stem, flags=re.I) cleaned = re.sub(r"_+", " ", cleaned).strip() cleaned = re.sub(r"\s+", " ", cleaned) year_match = re.search(r"\b(19|20)\d{2}\b", cleaned) year = year_match.group(0) if year_match else None party = None party_match = re.search(r"\b(Democratic|Republican)\b", cleaned, flags=re.I) if party_match: party = _official_title_name(party_match.group(1)) office_text = cleaned if year_match: office_text = office_text[year_match.end():].strip() office_text = re.sub( r"\b(Democratic|Republican)\b.*$", "", office_text, flags=re.I, ).strip() office_text = re.sub( r"\b(?:State\s+)?Primary\b.*$", "", office_text, flags=re.I, ).strip(" -_") return { "year": year, "party": party, "office": office_text or "Historical Election", } def _official_try_load_massachusetts_pd43( path: Path, text: str, encoding: str, election_date_override: Optional[str] = None, ) -> Optional[Dict[str, Any]]: """ Parse Massachusetts Secretary of the Commonwealth PD43+ City/Town CSVs. These are wide single-race tables: City/Town,,,Candidate A,Candidate B,...,Blanks,Total Votes Cast Candidate columns are converted into the same synthetic long-row layout used by the other official-results importers. Non-candidate choices such as All Others, No Preference, blanks, and total-vote columns are preserved only in parser metadata and are not treated as modeled candidates. """ lines = text.splitlines() if not lines: return None header_index = None headers = None for idx, line in enumerate(lines[:12]): try: fields = next(csv.reader([line], delimiter=",")) except Exception: continue normalized = [_official_norm_header(x) for x in fields] if ( normalized and normalized[0] in {"citytown", "cityortown", "municipality"} and "totalvotescast" in normalized ): header_index = idx headers = [str(x or "").strip() for x in fields] break if header_index is None or not headers: return None geography_col = headers[0] noise_headers = { "", "allothers", "allother", "nopreference", "blank", "blanks", "totalvotescast", "totalvotes", "votescast", "writein", "writeins", } candidate_headers = [] excluded_headers = [] for header in headers[1:]: normalized = _official_norm_header(header) if normalized in noise_headers: if normalized: excluded_headers.append(header) continue candidate_headers.append(header) if len(candidate_headers) < 2: return None metadata = _official_ma_filename_metadata(path) party_name = metadata.get("party") or "" office_desc = metadata.get("office") or "Historical Election" synthetic_headers = [ "ElectionDate", "OfficeDesc", "DistrictCode", "PartyName", "CountyName", "CandidateName", "CandidateVotes", ] reader = csv.DictReader(lines[header_index:], delimiter=",") rows: List[Dict[str, str]] = [] municipalities = set() excluded_votes: Dict[str, int] = { str(header): 0 for header in excluded_headers if str(header).strip() } for source_row in reader: if not isinstance(source_row, dict): continue municipality = str(source_row.get(geography_col) or "").strip() if not municipality: continue if municipality.casefold() in {"totals", "total", "statewide"}: continue municipalities.add(municipality.casefold()) for candidate_name in candidate_headers: name = re.sub(r"\s+", " ", str(candidate_name or "").strip()) if not name: continue votes = _official_parse_int(source_row.get(candidate_name)) rows.append({ # Keep the grouping key independent from a manually entered # date, matching the IA/SC importer behavior. "ElectionDate": "", "OfficeDesc": office_desc, "DistrictCode": "", "PartyName": party_name, "CountyName": municipality, "CandidateName": name, "CandidateVotes": str(votes), }) for excluded_header in excluded_headers: normalized = _official_norm_header(excluded_header) if normalized in {"totalvotescast", "totalvotes", "votescast"}: continue excluded_votes[excluded_header] = ( excluded_votes.get(excluded_header, 0) + _official_parse_int(source_row.get(excluded_header)) ) if len(municipalities) < 100 or not rows: return None columns = { key: _official_find_column( synthetic_headers, aliases, ) for key, aliases in _OFFICIAL_COLUMN_ALIASES.items() } return { "path": str(path), "filename": path.name, "encoding": encoding, "delimiter": ",", "delimiter_name": "comma", "header_line": header_index + 1, "headers": synthetic_headers, "columns": columns, "rows": rows, "source_format_hint": "massachusetts_pd43_city_town_csv", "region_type_hint": "Town", "region_label_hint": "municipalities", "massachusetts_pd43": { "municipalities_seen": len(municipalities), "candidate_columns": candidate_headers, "excluded_columns": excluded_headers, "excluded_votes": excluded_votes, "filename_year": metadata.get("year"), "filename_party": metadata.get("party"), "filename_office": metadata.get("office"), "election_date_override": ( str(election_date_override or "").strip() or None ), }, } def _official_load_table( file_path: str, state: Optional[str] = None, election_date_override: Optional[str] = None, ) -> Dict[str, Any]: path = Path(file_path) if not path.is_file(): raise FileNotFoundError(f"Official results file does not exist: {file_path}") raw = path.read_bytes() text = None encoding = None for candidate_encoding in ("utf-8-sig", "utf-8", "cp1252", "latin-1"): try: text = raw.decode(candidate_encoding) encoding = candidate_encoding break except UnicodeDecodeError: continue if text is None: raise ValueError("Could not decode the official results file.") # Massachusetts PD43+ exports are wide single-race City/Town CSVs. if str(state or "").upper() == "MA": massachusetts = _official_try_load_massachusetts_pd43( path, text, encoding or "unknown", election_date_override=election_date_override, ) if massachusetts is not None: return massachusetts # South Carolina and Iowa both publish human-readable fixed-width detail # reports rather than normal delimited tables. Detect them first. if str(state or "").upper() == "SC": south_carolina = ( _official_try_load_south_carolina_detail( path, text, encoding or "unknown", election_date_override=( election_date_override ), ) ) if south_carolina is not None: return south_carolina if str(state or "").upper() == "IA": iowa = _official_try_load_iowa_detail( path, text, encoding or "unknown", election_date_override=( election_date_override ), ) if iowa is not None: return iowa lines = text.splitlines() delimiters = ["\t", ",", "|", ";"] best = None required_groups = ("county_name",) for line_no, line in enumerate(lines[:100]): if not line.strip(): continue for delimiter in delimiters: try: fields = next(csv.reader([line], delimiter=delimiter)) except Exception: continue if len(fields) < 3: continue headers = [str(x or "").strip() for x in fields] columns = { key: _official_find_column(headers, aliases) for key, aliases in _OFFICIAL_COLUMN_ALIASES.items() } score = sum(1 for value in columns.values() if value) if not all(columns.get(k) for k in required_groups): score -= 5 if not ( columns.get("candidate_votes") or columns.get("candidate_absentee_votes") or columns.get("candidate_early_votes") or columns.get("candidate_election_day_votes") ): score -= 5 if not ( columns.get("candidate_name") or columns.get("candidate_last") or columns.get("candidate_first") ): score -= 3 if best is None or score > best[0]: best = (score, line_no, delimiter, headers, columns) if best is None or best[0] < 3: raise ValueError( "Could not detect a county/candidate/votes header row. " "Expected a TXT/TSV/CSV-style official results export." ) _, header_line, delimiter, headers, columns = best reader = csv.DictReader(lines[header_line:], delimiter=delimiter) rows: List[Dict[str, str]] = [] for source_row in reader: if not isinstance(source_row, dict): continue row = { str(k or "").strip(): str(v or "").strip() for k, v in source_row.items() if k is not None } county_col = columns.get("county_name") if county_col and (row.get(county_col) or _official_has_vote_data(row, columns)): rows.append(row) if not rows: raise ValueError("The detected official-results table contained no usable rows.") return { "path": str(path), "filename": path.name, "encoding": encoding, "delimiter": delimiter, "delimiter_name": {"\t": "tab", ",": "comma", "|": "pipe", ";": "semicolon"}.get(delimiter, delimiter), "header_line": header_line + 1, "headers": headers, "columns": columns, "rows": rows, } def _official_row_selection_key(row: Dict[str, str], columns: Dict[str, Optional[str]]) -> tuple: def val(key: str) -> str: col = columns.get(key) return re.sub(r"\s+", " ", str(row.get(col) or "").strip()) if col else "" return ( val("election_date"), val("office_code"), val("office_description"), val("district_code"), val("subdistrict_code"), val("party_description"), ) def _official_selection_id(key: tuple) -> str: raw = "\x1f".join(str(x or "") for x in key) return hashlib.sha1(raw.encode("utf-8", errors="replace")).hexdigest()[:16] def _official_source_format(table: Dict[str, Any], state: Optional[str] = None) -> str: if table.get("source_format_hint"): return str(table["source_format_hint"]) normalized_headers = { _official_norm_header(h) for h in (table.get("headers") or []) } florida_signature = { "electiondate", "partycode", "partyname", "racecode", "officedesc", "countycode", "countyname", "juris1num", "juris2num", "precincts", "precinctsreporting", "cannamelast", "cannamefirst", "canvotes", } if ( str(state or "").upper() == "FL" and len( florida_signature & normalized_headers ) >= 10 ): return "florida_dos_election_txt" return "generic_official_results_table" def scan_official_results_file( file_path: str, state: Optional[str] = None, election_date_override: Optional[str] = None, ) -> Dict[str, Any]: table = _official_load_table( file_path, state=state, election_date_override=( election_date_override ), ) columns = table["columns"] source_format = _official_source_format( table, state ) groups: Dict[tuple, Dict[str, Any]] = {} for row in table["rows"]: county_col = columns.get("county_name") county = str(row.get(county_col) or "").strip() if county_col else "" candidate = _official_candidate_name(row, columns) if not county or not candidate or not _official_has_vote_data(row, columns): continue key = _official_row_selection_key(row, columns) group = groups.setdefault(key, { "key": key, "counties": set(), "candidates": set(), "votes": 0, "rows": 0, }) group["counties"].add(normalize_county_name(county)) group["candidates"].add(candidate) group["votes"] += _official_candidate_total_votes(row, columns) group["rows"] += 1 selections = [] for key, group in groups.items(): election_date, office_code, office_desc, district, subdistrict, party = key display_election_date = election_date if ( source_format in ( "iowa_sos_detail_fixed_width", "south_carolina_scec_detail_fixed_width", "massachusetts_pd43_city_town_csv", ) and str( election_date_override or "" ).strip() ): display_election_date = str( election_date_override ).strip() county_count = len(group["counties"]) if county_count < 2: continue scope_guess = "statewide" if str(district).strip().casefold() in ("", "0", "00", "statewide", "state") else "district" party_slug = _official_party_slug(party) label_parts = [ x for x in ( display_election_date, party or "No party", office_desc or office_code or "Unknown office", ) if x ] if scope_guess == "district" and district: label_parts.append(f"District {district}") if subdistrict: label_parts.append(f"Seat {subdistrict}") selections.append({ "id": _official_selection_id(key), "election_date": ( display_election_date or None ), "office_code": office_code or None, "office_description": office_desc or office_code or "Unknown office", "district_code": district or None, "subdistrict_code": subdistrict or None, "party": party or None, "party_slug": party_slug, "scope_guess": scope_guess, "regions": county_count, "region_type": str(table.get("region_type_hint") or "County"), "region_label": str(table.get("region_label_hint") or "counties"), # Backward-compatible field name used by older Test Lab builds. "counties": county_count, "candidates": sorted(group["candidates"]), "candidate_count": len(group["candidates"]), "statewide_votes_from_rows": int(group["votes"]), "row_count": int(group["rows"]), "label": " — ".join(label_parts), }) selections.sort( key=lambda x: ( x.get("scope_guess") == "statewide", x.get("counties", 0), x.get("statewide_votes_from_rows", 0), ), reverse=True, ) warnings = [] if not selections: warnings.append( "No multi-region races were detected in this official results file." ) if ( source_format == "iowa_sos_detail_fixed_width" and not str( election_date_override or "" ).strip() ): warnings.append( "Iowa SOS detail.txt does not contain an election date. Enter an " "Election Date Override (for example 2022-06-07) before importing " "so historical recency weighting is correct." ) if ( source_format == "south_carolina_scec_detail_fixed_width" and not str( election_date_override or "" ).strip() ): warnings.append( "South Carolina's fixed-width detail report does not contain a " "reliable election date field. Enter an Election Date Override " "(for this 2022 primary file: 2022-06-14) before importing so " "historical recency weighting is correct." ) if ( source_format == "massachusetts_pd43_city_town_csv" and not str( election_date_override or "" ).strip() ): warnings.append( "Massachusetts PD43+ City/Town CSVs do not contain the full election " "date. Enter an Election Date Override (YYYY-MM-DD) before importing " "so historical recency weighting is correct." ) return { "schema_version": 1, "mode": "OFFICIAL_RESULTS_REFERENCE_SCAN", "state": str(state or "").upper() or None, "source_file": table["filename"], "source_path": table["path"], "source_format": source_format, "encoding": table["encoding"], "delimiter": table["delimiter_name"], "header_line": table["header_line"], "detected_columns": table["columns"], "selections": selections, "warnings": warnings, "election_date_override": ( str( election_date_override or "" ).strip() or None ), "parser_metadata": ( table.get( "south_carolina_detail" ) or table.get("iowa_detail") or table.get("massachusetts_pd43") or {} ), } def import_official_results_reference( file_path: str, selection_id: str, state: str, race_url: Optional[str] = None, election_date_override: Optional[str] = None, ) -> Dict[str, Any]: table = _official_load_table( file_path, state=state, election_date_override=( election_date_override ), ) columns = table["columns"] selected_key = None selected_meta = None scan = scan_official_results_file( file_path, state, election_date_override=( election_date_override ), ) for selection in scan.get("selections") or []: if str(selection.get("id")) == str(selection_id): selected_meta = selection break if not selected_meta: raise ValueError( "The selected race/party was not found in this official results file." ) if ( scan.get("source_format") in ( "iowa_sos_detail_fixed_width", "south_carolina_scec_detail_fixed_width", ) and not str( selected_meta.get( "election_date" ) or "" ).strip() ): source_format = scan.get("source_format") source_label = { "iowa_sos_detail_fixed_width": "Iowa SOS detail.txt", "south_carolina_scec_detail_fixed_width": ( "South Carolina fixed-width detail report" ), "massachusetts_pd43_city_town_csv": ( "Massachusetts PD43+ City/Town CSV" ), }.get(source_format, "Official results file") raise ValueError( source_label + " does not include a reliable election date. " + "Enter an Election Date Override before importing." ) # Reconstruct the exact grouping key by ID so the import remains stable even # if the display ordering changes. for row in table["rows"]: key = _official_row_selection_key(row, columns) if _official_selection_id(key) == str(selection_id): selected_key = key break if selected_key is None: raise ValueError("Could not reconstruct the selected official-results grouping.") region_type = str(table.get("region_type_hint") or "County") region_label = str(table.get("region_label_hint") or "counties") is_county_layer = region_type.strip().casefold() == "county" fips_map = current_race_county_fips(race_url) if is_county_layer else {} county_votes: Dict[str, Dict[str, int]] = {} county_display: Dict[str, str] = {} statewide: Dict[str, int] = {} county_col = columns.get("county_name") if not county_col: raise ValueError("Official results file is missing a geographic-unit column.") if not ( columns.get("candidate_votes") or columns.get("candidate_absentee_votes") or columns.get("candidate_early_votes") or columns.get("candidate_election_day_votes") ): raise ValueError("Official results file is missing total votes and recognizable vote-method columns.") county_vote_methods: Dict[str, Dict[str, Dict[str, int]]] = {} statewide_vote_methods: Dict[str, Dict[str, int]] = {} county_precincts: Dict[str, Dict[str, int]] = {} vote_method_rows = 0 vote_method_total_mismatch_rows = 0 for row in table["rows"]: if _official_row_selection_key(row, columns) != selected_key: continue county_raw = str(row.get(county_col) or "").strip() candidate = _official_candidate_name(row, columns) votes = _official_candidate_total_votes(row, columns) if not county_raw or not candidate: continue norm_county = normalize_county_name(county_raw) if not norm_county: continue county_display.setdefault(norm_county, _official_title_name(county_raw)) county_votes.setdefault(norm_county, {}) county_votes[norm_county][candidate] = county_votes[norm_county].get(candidate, 0) + votes statewide[candidate] = statewide.get(candidate, 0) + votes methods = _official_vote_methods(row, columns) if methods: vote_method_rows += 1 method_sum = sum(methods.values()) if columns.get("candidate_votes") and method_sum != votes: vote_method_total_mismatch_rows += 1 county_vote_methods.setdefault(norm_county, {}).setdefault(candidate, {}) statewide_vote_methods.setdefault(candidate, {}) for method, method_votes in methods.items(): county_vote_methods[norm_county][candidate][method] = ( county_vote_methods[norm_county][candidate].get(method, 0) + method_votes ) statewide_vote_methods[candidate][method] = ( statewide_vote_methods[candidate].get(method, 0) + method_votes ) total_prec_col = columns.get("total_precincts") reporting_prec_col = columns.get("precincts_reporting") if total_prec_col or reporting_prec_col: cmeta = county_precincts.setdefault(norm_county, {}) if total_prec_col: cmeta["total"] = max(cmeta.get("total", 0), _official_parse_int(row.get(total_prec_col))) if reporting_prec_col: cmeta["reporting"] = max(cmeta.get("reporting", 0), _official_parse_int(row.get(reporting_prec_col))) if len(county_votes) < 5: raise ValueError( f"Fewer than five {region_label} were imported from the selected official race." ) party_raw = selected_meta.get("party") or None party_name = _official_title_name(party_raw) if party_raw else None region_results: Dict[str, Any] = {} matched_fips = 0 for norm_county, votes_by_candidate in sorted(county_votes.items()): county_total = sum(votes_by_candidate.values()) if county_total <= 0: continue fips = fips_map.get(norm_county) if is_county_layer else None if fips: matched_fips += 1 candidates = [ { "name": name, "party": party_name, "votes": int(votes), "percent": 100.0 * votes / county_total if county_total else 0.0, "major_candidate": True, **({"vote_methods": county_vote_methods.get(norm_county, {}).get(name, {})} if county_vote_methods.get(norm_county, {}).get(name) else {}), } for name, votes in sorted(votes_by_candidate.items(), key=lambda kv: kv[1], reverse=True) ] key = fips or norm_county precinct_meta = county_precincts.get(norm_county) or {} region_results[key] = { "name": county_display.get(norm_county) or norm_county, "type": region_type, "fips": fips, "percent_reporting": 100.0, "candidates": candidates, **({"precincts_total": precinct_meta.get("total")} if precinct_meta.get("total") is not None else {}), **({"precincts_reporting": precinct_meta.get("reporting")} if precinct_meta.get("reporting") is not None else {}), } statewide_total = sum(statewide.values()) statewide_candidates = [ { "name": name, "party": party_name, "votes": int(votes), "percent": 100.0 * votes / statewide_total if statewide_total else 0.0, "major_candidate": True, **({"vote_methods": statewide_vote_methods.get(name, {})} if statewide_vote_methods.get(name) else {}), } for name, votes in sorted(statewide.items(), key=lambda kv: kv[1], reverse=True) ] election_date = selected_meta.get("election_date") office_desc = selected_meta.get("office_description") or "Historical Election" election_name = " — ".join( x for x in (str(election_date or "").strip(), str(party_raw or "").strip(), str(office_desc).strip()) if x ) return { "schema_version": 1, "source_type": "official_results_file_historical_reference", "election_name": election_name, "province": str(state or "").upper(), "election_type": ( "Runoff" if "runoff" in Path(file_path).name.casefold() else "Primary" if (party_raw or "primary" in Path(file_path).name.casefold()) else "Historical" ), "election_scope": "State" if selected_meta.get("scope_guess") == "statewide" else "District", "last_updated": election_date, "percent_reporting": 100.0, "candidates": statewide_candidates, "region_results": region_results, "source": { "official_results_file": str(Path(file_path).name), "source_format": _official_source_format( table, state ), "selection_id": selection_id, "selection": selected_meta, "delimiter": table["delimiter_name"], "header_line": table["header_line"], "detected_columns": table["columns"], "parser_metadata": ( table.get( "south_carolina_detail" ) or table.get( "iowa_detail" ) or table.get( "massachusetts_pd43" ) or {} ), "current_race_used_for_fips": race_url, "vote_method_metadata_preserved": bool( columns.get("candidate_absentee_votes") or columns.get("candidate_early_votes") or columns.get("candidate_election_day_votes") ), "vote_method_note": ( "Vote-method fields are preserved for research, but V6.2 uses geographic-unit total votes; Oklahoma can therefore be tested even when the live API does not expose voting method." ), }, "import_summary": { "regions_imported": len(region_results), "region_type": region_type, "region_label": region_label, "counties_imported": len(region_results) if is_county_layer else 0, "municipalities_imported": ( len(region_results) if region_type.strip().casefold() in { "town", "city", "municipality", "city/town", "city or town" } else 0 ), "fips_matched": matched_fips, "fips_match_rate": ( matched_fips / len(region_results) if is_county_layer and region_results else None ), "candidates": [c["name"] for c in statewide_candidates], "statewide_votes_from_regions": statewide_total, # Retained for backward compatibility with old report/UI readers. "statewide_votes_from_counties": statewide_total, "party": party_name, "office": office_desc, "office_code": selected_meta.get( "office_code" ), "district_code": selected_meta.get( "district_code" ), "subdistrict_code": selected_meta.get( "subdistrict_code" ), "election_date": election_date, "source_file": Path(file_path).name, "source_format": ( scan.get("source_format") ), "source_file_excluded_write_in_votes": int( ( table.get("iowa_detail") or {} ).get( "excluded_write_in_votes" ) or 0 ), "vote_method_rows": vote_method_rows, "vote_method_total_mismatch_rows": vote_method_total_mismatch_rows, "vote_method_columns_detected": [ key for key in ( "candidate_absentee_votes", "candidate_early_votes", "candidate_election_day_votes", ) if columns.get(key) ], }, } def _official_safe_filename_part(value: Any) -> str: text = re.sub( r"[^a-z0-9]+", "_", str(value or "").strip().casefold(), ).strip("_") return text or "unknown" def _official_selection_is_ballot_measure( selection: Dict[str, Any], ) -> bool: office = str( selection.get("office_description") or selection.get("office_code") or "" ).casefold() measure_terms = ( "advisory question", "ballot question", "referendum", "constitutional amendment", "proposed amendment", ) return any( term in office for term in measure_terms ) def import_all_statewide_official_references( file_path: str, state: str, output_dir: str, race_url: Optional[str] = None, election_date_override: Optional[str] = None, ) -> Dict[str, Any]: """ Import every statewide race/party discovered in one official-results file. This is especially useful for Florida DOS, Iowa SOS, and South Carolina Election Commission exports, where one file can contain statewide party primaries plus many congressional, legislative, judicial, local, and ballot question contests. Only actual multi-candidate statewide elections are imported here. """ scan = scan_official_results_file( file_path, state, election_date_override=( election_date_override ), ) statewide = [ row for row in (scan.get("selections") or []) if row.get("scope_guess") == "statewide" and int(row.get("candidate_count") or 0) >= 2 and not _official_selection_is_ballot_measure( row ) ] if not statewide: raise ValueError( "No multi-candidate statewide races were detected in this official results file." ) out_dir = Path(output_dir) out_dir.mkdir(parents=True, exist_ok=True) imported = [] errors = [] for selection in statewide: try: ref = import_official_results_reference( file_path, str(selection["id"]), state, race_url, election_date_override=( election_date_override ), ) date_part = _official_safe_filename_part( selection.get("election_date") ) party_part = _official_party_slug( selection.get("party") ) office_part = _official_safe_filename_part( selection.get("office_code") or selection.get( "office_description" ) ) filename = ( f"{str(state).lower()}_" f"{date_part}_" f"{party_part}_" f"{office_part}_" f"{selection['id']}.json" ) out_path = out_dir / filename save_json(out_path, ref) imported.append({ "selection_id": selection["id"], "label": selection.get("label"), "party": selection.get("party"), "party_slug": selection.get( "party_slug" ), "office": selection.get( "office_description" ), "office_code": selection.get( "office_code" ), "election_date": selection.get( "election_date" ), "regions": ( ref.get("import_summary") or {} ).get("regions_imported"), "region_type": ( ref.get("import_summary") or {} ).get("region_type"), "counties": ( ref.get("import_summary") or {} ).get("counties_imported"), "candidates": ( ref.get("import_summary") or {} ).get("candidates"), "votes": ( ref.get("import_summary") or {} ).get( "statewide_votes_from_regions" ), "output": str(out_path), "filename": filename, }) except Exception as exc: errors.append({ "selection_id": selection.get("id"), "label": selection.get("label"), "error": str(exc), }) if not imported: raise ValueError( "Statewide races were detected, but none could be imported." ) return { "schema_version": 1, "mode": "OFFICIAL_RESULTS_IMPORT_ALL_STATEWIDE", "state": str(state or "").upper(), "source_file": Path(file_path).name, "source_format": scan.get( "source_format" ), "statewide_races_detected": len( statewide ), "statewide_races_imported": len( imported ), "imported": imported, "errors": errors, "method_note": ( "Imports only multi-candidate statewide candidate elections. " "District, legislative, judicial, local, and ballot-question " "contests remain available through the normal selected-race " "scanner but are not bulk-added to the statewide historical " "reference library." ), } def scan_wikipedia_article(url: str, state: Optional[str] = None) -> Dict[str, Any]: html = fetch_wikipedia_html(url) parser = WikipediaElectionTableParser() parser.feed(html) found = [] for i, table in enumerate(parser.tables): analyzed = analyze_wikipedia_table(table, i) if analyzed: found.append(analyzed) found.sort( key=lambda x: ( x.get("party_guess") == "democratic", x.get("score", 0), x.get("counties", 0), ), reverse=True, ) article_title = parser.page_title if not article_title: article_title = unquote(urlparse(url).path.rsplit("/", 1)[-1]).replace("_", " ") warnings = [] if not found: warnings.append( "No importable county vote table was detected. Some Wikipedia election " "articles show a county map but only publish statewide primary totals; " "a map alone is not enough to reconstruct county vote totals." ) return { "schema_version": 1, "mode": "WIKIPEDIA_REFERENCE_SCAN", "url": validate_wikipedia_url(url), "state": state, "article_title": article_title, "tables": found, "warnings": warnings, } def current_race_county_fips(race_url: Optional[str]) -> Dict[str, str]: if not race_url: return {} try: current = fetch_json(slug_base(race_url) + "latest.json") except Exception: return {} out = {} for region in county_regions(current): name = normalize_county_name(region.get("name") or region.get("_key")) fips = re.sub(r"\D", "", str(region.get("fips") or region.get("county_fips") or "")) if name and fips: out[name] = fips return out def import_wikipedia_reference( url: str, table_id: int, state: str, race_url: Optional[str] = None, ) -> Dict[str, Any]: html = fetch_wikipedia_html(url) parser = WikipediaElectionTableParser() parser.feed(html) if table_id < 0 or table_id >= len(parser.tables): raise ValueError(f"Wikipedia table id {table_id} does not exist on this page.") table = parser.tables[table_id] analysis = analyze_wikipedia_table(table, table_id) if not analysis: raise ValueError("The selected table is not an importable county-results table.") matrix = expand_wikipedia_rows(table.get("rows") or []) # Re-identify first data row exactly as scan did. first_data = None for i, row in enumerate(matrix): texts = [clean_wiki_text(c.get("text")) for c in row] if not any(texts): continue numeric = sum(parse_vote_integer(x) is not None for x in texts[1:]) first = texts[0].casefold() if numeric >= 1 and first not in ("county", "counties", "parish", "borough"): first_data = i break if first_data is None: raise ValueError("Could not locate county data rows in the selected table.") county_col = int(analysis["county_column"]) total_col = analysis.get("total_column") candidate_cols = analysis.get("candidate_columns") or [] fips_map = current_race_county_fips(race_url) region_results: Dict[str, Any] = {} statewide = {item["name"]: 0 for item in candidate_cols} imported_counties = 0 matched_fips = 0 skip_names = { "total", "totals", "statewide", "state total", "all counties", "total votes", "grand total" } for row in matrix[first_data:]: if county_col >= len(row): continue county = clean_wiki_text(row[county_col].get("text")) if not county or county.casefold() in skip_names: continue candidate_votes = {} for item in candidate_cols: col = int(item["column"]) value = parse_vote_integer(row[col].get("text")) if col < len(row) else None candidate_votes[item["name"]] = max(0, int(value or 0)) if candidate_cols: county_total = sum(candidate_votes.values()) else: county_total = 0 if total_col is not None and int(total_col) < len(row): displayed_total = parse_vote_integer(row[int(total_col)].get("text")) else: displayed_total = None # If the table has only a total-vote column, preserve historical turnout # using a synthetic candidate bucket. This remains turnout-only in V3/V4.5. if not candidate_cols: if displayed_total is None or displayed_total < 0: continue candidate_votes = {"Historical Total": int(displayed_total)} county_total = int(displayed_total) elif county_total <= 0 and displayed_total is not None and displayed_total > 0: # Candidate extraction failed for this individual row; don't invent a split. continue if county_total <= 0: continue norm_name = normalize_county_name(county) fips = fips_map.get(norm_name) if fips: matched_fips += 1 candidates = [] for name, votes in candidate_votes.items(): candidates.append({ "name": name, "party": "Democratic" if analysis.get("party_guess") == "democratic" else ( "Republican" if analysis.get("party_guess") == "republican" else None ), "votes": int(votes), "percent": (100.0 * votes / county_total) if county_total else 0.0, "major_candidate": True, }) statewide[name] = statewide.get(name, 0) + int(votes) key = fips or norm_name or f"county_{imported_counties + 1}" region_results[key] = { "name": county, "type": "County", "fips": fips, "percent_reporting": 100.0, "candidates": candidates, } imported_counties += 1 if imported_counties < 5: raise ValueError("Fewer than five county result rows could be imported.") statewide_total = sum(statewide.values()) party_name = ( "Democratic" if analysis.get("party_guess") == "democratic" else "Republican" if analysis.get("party_guess") == "republican" else None ) statewide_candidates = [ { "name": name, "party": party_name, "votes": int(votes), "percent": (100.0 * votes / statewide_total) if statewide_total else 0.0, "major_candidate": True, } for name, votes in sorted(statewide.items(), key=lambda kv: kv[1], reverse=True) ] article_title = parser.page_title or unquote( urlparse(url).path.rsplit("/", 1)[-1] ).replace("_", " ") return { "schema_version": 1, "source_type": "wikipedia_historical_reference", "election_name": f"{article_title} — {analysis['label']}", "province": str(state or "").upper(), "election_type": "Primary" if "primary" in analysis["label"].casefold() else "Historical", "election_scope": "State", "last_updated": None, "percent_reporting": 100.0, "candidates": statewide_candidates, "region_results": region_results, "source": { "wikipedia_url": validate_wikipedia_url(url), "article_title": article_title, "table_id": table_id, "table_label": analysis["label"], "party_guess": analysis.get("party_guess"), "headers": analysis.get("headers"), "current_race_used_for_fips": race_url, }, "import_summary": { "counties_imported": imported_counties, "fips_matched": matched_fips, "fips_match_rate": ( matched_fips / imported_counties if imported_counties else 0.0 ), "candidates": [c["name"] for c in statewide_candidates], "statewide_votes_from_counties": statewide_total, "turnout_only": bool( len(statewide_candidates) == 1 and statewide_candidates[0]["name"] == "Historical Total" ), }, } def batch_test_percentages( base_url: str, targets: List[float], reference_url: Optional[str] = None, market_map_source: Optional[str] = None, monte_carlo_simulations: int = 1200, benchmark_replay_intervals: int = 3, ) -> Dict[str, Any]: base_url = slug_base(base_url) metadata = fetch_json(base_url + "snapshot_metadata.json") final = fetch_json(base_url + "latest.json") final_candidates = candidate_map(final) final_shares = shares_from_votes(final_candidates) final_ranked_names = rank( final_candidates, final_shares ) final_rank_lookup = { name: idx + 1 for idx, name in enumerate( final_ranked_names ) } final_votes = sum( c["votes"] for c in final_candidates.values() ) available = [] for item in metadata.get("snapshots") or []: filename = item.get("filename") votes = int(item.get("votes") or 0) if filename and votes > 0 and final_votes > 0: available.append((filename, 100.0 * votes / final_votes, item)) rows = [] used = set() any_market = False resolved_market_source = None available_model_prefixes = set() for target in targets: pool = [x for x in available if x[0] not in used] or available if not pool: break filename, completion, meta = min( pool, key=lambda x: abs(x[1] - target) ) distance = abs(completion - target) tolerance = checkpoint_tolerance(target) eligible = distance <= tolerance row = { "target_pct": target, "actual_vote_counted_pct_backtest_only": completion, "checkpoint_distance_points": distance, "checkpoint_tolerance_points": tolerance, "eligible": eligible, "snapshot": filename if eligible else None, "snapshot_number": meta.get("snapshot") if eligible else None, "timestamp": meta.get("timestamp") if eligible else None, } if not eligible: rows.append(row) continue used.add(filename) checkpoint_snapshot = fetch_json(base_url + filename) checkpoint_candidates = candidate_map(checkpoint_snapshot) checkpoint_shares = shares_from_votes(checkpoint_candidates) checkpoint_rank = rank( checkpoint_candidates, checkpoint_shares ) current_rank_lookup = { name: idx + 1 for idx, name in enumerate( checkpoint_rank ) } raw_leader = checkpoint_rank[0] if checkpoint_rank else None raw_runner = ( checkpoint_rank[1] if len(checkpoint_rank) > 1 else None ) raw_margin = ( 100.0 * ( checkpoint_shares.get(raw_leader, 0.0) - checkpoint_shares.get(raw_runner, 0.0) ) if raw_leader and raw_runner else None ) v6 = project_v4( base_url, filename, reference_url, market_map_source, monte_carlo_simulations=max( 100, int(monte_carlo_simulations) ), benchmark_replay_intervals=max( 2, int(benchmark_replay_intervals) ), ) # Full candidate projection audit for multi-candidate batch tests. # This is scoring/reporting-only: final shares/ranks are never passed # back into the live projection. v6_projection_rows = ( v6.get("projection") or [] ) projected_ranked_names = [ r.get("name") for r in sorted( v6_projection_rows, key=lambda r: float( r.get("projected_share") or 0.0 ), reverse=True, ) if r.get("name") ] projected_rank_lookup = { name: idx + 1 for idx, name in enumerate( projected_ranked_names ) } projection_map = { r.get("name"): r for r in v6_projection_rows if r.get("name") } all_candidate_names = sorted( set(checkpoint_candidates) | set(final_candidates) | set(projection_map) ) candidate_projection_breakdown = [] exact_rank_calls = 0 absolute_rank_error = 0.0 for name in all_candidate_names: prow = projection_map.get(name) or {} current_rank_value = ( current_rank_lookup.get(name) ) projected_rank_value = ( projected_rank_lookup.get(name) ) final_rank_value = ( final_rank_lookup.get(name) ) current_share_points = ( 100.0 * checkpoint_shares.get( name, 0.0 ) ) projected_share_points = ( 100.0 * float( prow.get( "projected_share" ) or 0.0 ) ) final_share_points = ( 100.0 * final_shares.get( name, 0.0 ) ) share_error_points = ( projected_share_points - final_share_points ) rank_error = None rank_call_correct = None if ( projected_rank_value is not None and final_rank_value is not None ): rank_error = ( projected_rank_value - final_rank_value ) absolute_rank_error += abs( rank_error ) rank_call_correct = ( projected_rank_value == final_rank_value ) if rank_call_correct: exact_rank_calls += 1 current_to_projected = None final_movement = None if ( current_rank_value is not None and projected_rank_value is not None ): # Positive means the model expects the candidate to move UP. current_to_projected = ( current_rank_value - projected_rank_value ) if ( current_rank_value is not None and final_rank_value is not None ): # Positive means the candidate actually moved UP by the end. final_movement = ( current_rank_value - final_rank_value ) candidate_projection_breakdown.append({ "name": name, "party": ( (checkpoint_candidates.get(name) or {}).get( "party" ) or (final_candidates.get(name) or {}).get( "party" ) ), "current_votes": int( ( checkpoint_candidates.get(name) or {} ).get("votes") or 0 ), "current_rank": current_rank_value, "current_share_points": ( current_share_points ), "projected_rank": ( projected_rank_value ), "projected_share_points": ( projected_share_points ), "projected_change_points": ( projected_share_points - current_share_points ), "projected_rank_movement": ( current_to_projected ), "final_rank_backtest_only": ( final_rank_value ), "final_share_points_backtest_only": ( final_share_points ), "share_error_points_backtest_only": ( share_error_points ), "final_rank_movement_backtest_only": ( final_movement ), "rank_error_backtest_only": ( rank_error ), "rank_call_correct_backtest_only": ( rank_call_correct ), "hidden_movement": bool( current_to_projected ), }) candidate_projection_breakdown.sort( key=lambda r: ( r.get("projected_rank") if r.get("projected_rank") is not None else 999 ) ) scored_rank_candidates = sum( 1 for r in candidate_projection_breakdown if r.get("projected_rank") is not None and r.get( "final_rank_backtest_only" ) is not None ) row["v6_candidate_projection_breakdown"] = ( candidate_projection_breakdown ) row["v6_candidate_rank_summary"] = { "candidates_scored": ( scored_rank_candidates ), "exact_rank_calls": exact_rank_calls, "exact_rank_accuracy": ( exact_rank_calls / scored_rank_candidates if scored_rank_candidates else None ), "mean_absolute_rank_error": ( absolute_rank_error / scored_rank_candidates if scored_rank_candidates else None ), "projected_order": ( projected_ranked_names ), "current_order": checkpoint_rank, "final_order_backtest_only": ( final_ranked_names ), "hidden_movement_candidates": [ r["name"] for r in candidate_projection_breakdown if r.get("hidden_movement") ], } bundle = component_snapshot_bundle( base_url, filename, reference_url, market_map_source, ) prefix_map = { "v0": "v0", "v1": "v1", "v2r": "v2r", "v3": "v3", "histgeo": "histgeo", "latent": "latent", "batchbias": "batchbias", "runoff_transfer": "runoff_transfer", "precinct": "precinct", } models = [] for key, prefix in prefix_map.items(): model = bundle["models"].get(key) if model: models.append((prefix, model)) available_model_prefixes.add(prefix) if v6.get("benchmark_completion_model"): models.append(( "benchmark_completion", v6["benchmark_completion_model"], )) available_model_prefixes.add( "benchmark_completion" ) models.append(("v6", v6)) available_model_prefixes.add("v6") for prefix, result in models: wp = result["winner_projection"] row[prefix + "_predicted_leader"] = result[ "predicted_leader" ] row[prefix + "_leader_correct"] = result[ "leader_correct" ] row[ prefix + "_winner_projected_share_points" ] = wp["projected_share_points"] row[ prefix + "_winner_share_error_points" ] = wp["share_error_points"] row[ prefix + "_winner_projected_margin_points" ] = wp["projected_margin_points"] row[ prefix + "_winner_margin_error_points" ] = wp["margin_error_points"] row[ prefix + "_candidate_share_mae_points" ] = result["candidate_share_mae_points"] v6d = v6.get("v6") or v6.get("v5") or v6.get("v4") or {} td = v6.get("temporal") or {} cb = v6.get("county_benchmarks") or {} path_bs = ( v6.get("path_benchmark_signal") or {} ) completion_bs = ( v6.get("completion_benchmark_signal") or v6.get("benchmark_signal") or {} ) bs = completion_bs consensus = v6d.get("crossover_consensus") or {} turning = ( v6.get("turning_point") or v6d.get("turning_point") or {} ) row["v6_model_confidence"] = v6d.get("model_confidence") row["v6_weights"] = v6d.get("weights") row["v6_components"] = v6d.get("components") row["v6_gate"] = v6d.get("gate") row["v6_horizon_validation"] = v6d.get( "horizon_validation" ) row["v6_raw_guard"] = v6d.get("raw_guard") row["v6_completion_referee"] = v6d.get( "completion_referee" ) row["v6_path_referee"] = v6d.get("path_referee") row["v6_reporting_path_profile"] = v6d.get( "reporting_path_profile" ) row["v6_path_benchmark_signal"] = path_bs row["v6_completion_benchmark_signal"] = completion_bs # Compatibility alias now means the final-oriented Completion Benchmark. row["v6_benchmark_signal"] = completion_bs row["v6_crossover_consensus"] = consensus row["v6_turning_point"] = turning row["v6_turning_point_actionable"] = bool( turning.get("actionable") ) row["v6_maturity_info"] = v6d.get( "maturity_info" ) row[ "v6_specialist_temporal_validation" ] = v6d.get( "specialist_temporal_validation" ) row["v6_county_benchmarks"] = { "projected_leader": cb.get("projected_leader"), "projected_runner_up": cb.get("projected_runner_up"), "projected_statewide_margin_points": cb.get( "projected_statewide_margin_points" ), "leader_weighted_expected_excess_points": cb.get( "leader_weighted_expected_excess_points" ), "leader_recent_weighted_excess_points": cb.get( "leader_recent_weighted_excess_points" ), "top_leverage_counties": cb.get( "top_leverage_counties" ), "most_pivotal_counties": cb.get( "most_pivotal_counties" ), } row["v6_monte_carlo"] = v6.get("monte_carlo") row["v6_temporal_confidence"] = td.get("confidence") row["v6_temporal_recent_votes"] = td.get("recent_votes") row["v6_temporal_trend"] = td.get("trend_label") row["raw_current_leader"] = raw_leader row["raw_current_margin_points"] = raw_margin row["v6_specialists"] = v6d.get("specialists") row["v6_reference_library"] = v6d.get("reference_library") row["v6_top_media_markets"] = ( ((v6.get("media_markets") or {}).get("markets") or [])[:8] ) final_leader = v6.get("final_leader") if final_leader: raw_opponent_share = max( ( share for name, share in checkpoint_shares.items() if name != final_leader ), default=0.0, ) raw_eventual_margin = 100.0 * ( checkpoint_shares.get(final_leader, 0.0) - raw_opponent_share ) row[ "raw_eventual_winner_margin_points" ] = raw_eventual_margin v6_margin = ( v6.get("winner_projection") or {} ).get("projected_margin_points") row["v6_lead_correction_points"] = ( v6_margin - raw_eventual_margin if v6_margin is not None else None ) mc = v6.get("monte_carlo") or {} row["v6_eventual_winner_mc_probability"] = ( (mc.get("win_probabilities") or {}).get( final_leader ) ) row[ "v6_path_benchmark_favors_eventual_winner" ] = ( path_bs.get("favored_candidate") == final_leader if path_bs.get("available") else None ) row[ "v6_benchmark_favors_eventual_winner" ] = ( completion_bs.get( "favored_candidate" ) == final_leader if completion_bs.get("available") else None ) row[ "v6_completion_benchmark_favors_eventual_winner" ] = row[ "v6_benchmark_favors_eventual_winner" ] row[ "v6_persistent_completion_actionable" ] = bool( completion_bs.get( "persistent_actionable_pre_maturity" ) and float( ( v6d.get("maturity_info") or {} ).get( "counted_fraction_estimate" ) or 0.0 ) >= 0.30 ) row[ "v6_turning_point_favors_eventual_winner" ] = ( turning.get("candidate") == final_leader if turning.get("available") else None ) row[ "v6_completion_benchmark_actionable" ] = bool( row.get( "v6_persistent_completion_actionable" ) or turning.get("actionable") ) row[ "v6_consensus_favors_eventual_winner" ] = ( consensus.get("candidate") == final_leader if consensus.get("available") else None ) if v6d.get("media_market_available"): any_market = True if resolved_market_source is None: resolved_market_source = v6d.get( "media_market_source" ) rows.append(row) summary = { "requested_tests": len(targets), "eligible_tests": sum( 1 for r in rows if r.get("eligible") ), } for prefix in ( "v0", "v1", "v2r", "v3", "histgeo", "latent", "batchbias", "runoff_transfer", "precinct", "benchmark_completion", "v6" ): if prefix in available_model_prefixes: summary[prefix] = summarize_batch(rows, prefix) else: summary[prefix] = None mc_probs = [ float(r["v6_eventual_winner_mc_probability"]) for r in rows if r.get("eligible") and r.get("v6_eventual_winner_mc_probability") is not None ] rank_rows = [ r.get("v6_candidate_rank_summary") or {} for r in rows if r.get("eligible") and r.get( "v6_candidate_rank_summary" ) ] total_rank_calls = sum( int(r.get("candidates_scored") or 0) for r in rank_rows ) total_exact_rank_calls = sum( int(r.get("exact_rank_calls") or 0) for r in rank_rows ) weighted_abs_rank_error = sum( float( r.get( "mean_absolute_rank_error" ) or 0.0 ) * int( r.get("candidates_scored") or 0 ) for r in rank_rows ) summary["candidate_rank_projection"] = { "checkpoint_rows": len(rank_rows), "candidate_rank_calls": total_rank_calls, "exact_rank_calls": total_exact_rank_calls, "exact_rank_accuracy": ( total_exact_rank_calls / total_rank_calls if total_rank_calls else None ), "mean_absolute_rank_error": ( weighted_abs_rank_error / total_rank_calls if total_rank_calls else None ), } summary["monte_carlo"] = { "eligible_tests": len(mc_probs), "mean_eventual_winner_probability": ( sum(mc_probs) / len(mc_probs) if mc_probs else None ), "eventual_winner_brier": ( sum((1.0 - p) ** 2 for p in mc_probs) / len(mc_probs) if mc_probs else None ), "eventual_winner_log_loss": ( sum(-math.log(max(1e-6, p)) for p in mc_probs) / len(mc_probs) if mc_probs else None ), "checkpoints_below_50_percent_for_eventual_winner": sum( 1 for p in mc_probs if p < 0.5 ), } path_benchmark_rows = [ r for r in rows if r.get("eligible") and ( r.get("v6_path_benchmark_signal") or {} ).get("available") ] completion_benchmark_rows = [ r for r in rows if r.get("eligible") and ( r.get( "v6_completion_benchmark_signal" ) or {} ).get("available") ] actionable_completion_rows = [ r for r in completion_benchmark_rows if r.get( "v6_completion_benchmark_actionable" ) ] summary["county_benchmarks"] = { "eligible_path_signal_checkpoints": len( path_benchmark_rows ), "eligible_completion_signal_checkpoints": len( completion_benchmark_rows ), "mean_path_signal_evidence": ( sum( float( ( r.get( "v6_path_benchmark_signal" ) or {} ).get("evidence") or 0.0 ) for r in path_benchmark_rows ) / len(path_benchmark_rows) if path_benchmark_rows else None ), "mean_completion_signal_evidence": ( sum( float( ( r.get( "v6_completion_benchmark_signal" ) or {} ).get("evidence") or 0.0 ) for r in completion_benchmark_rows ) / len(completion_benchmark_rows) if completion_benchmark_rows else None ), "path_favored_eventual_winner_rate": ( sum( 1 for r in path_benchmark_rows if r.get( "v6_path_benchmark_favors_eventual_winner" ) ) / len(path_benchmark_rows) if path_benchmark_rows else None ), "completion_favored_eventual_winner_rate": ( sum( 1 for r in completion_benchmark_rows if r.get( "v6_benchmark_favors_eventual_winner" ) ) / len(completion_benchmark_rows) if completion_benchmark_rows else None ), "actionable_completion_checkpoints": len( actionable_completion_rows ), "actionable_completion_favored_winner_rate": ( sum( 1 for r in actionable_completion_rows if r.get( "v6_benchmark_favors_eventual_winner" ) ) / len(actionable_completion_rows) if actionable_completion_rows else None ), "raw_wrong_completion_benchmark_right_checkpoints": sum( 1 for r in completion_benchmark_rows if not r.get("v0_leader_correct") and r.get( "v6_benchmark_favors_eventual_winner" ) ), "raw_wrong_actionable_completion_right_checkpoints": sum( 1 for r in actionable_completion_rows if not r.get("v0_leader_correct") and r.get( "v6_benchmark_favors_eventual_winner" ) ), "actionable_consensus_checkpoints": sum( 1 for r in completion_benchmark_rows if ( r.get( "v6_crossover_consensus" ) or {} ).get("eligible") ), "actionable_turning_point_checkpoints": sum( 1 for r in rows if r.get("eligible") and r.get("v6_turning_point_actionable") ), "actionable_turning_point_accuracy": ( sum( 1 for r in rows if r.get("eligible") and r.get( "v6_turning_point_actionable" ) and r.get( "v6_turning_point_favors_eventual_winner" ) ) / max( 1, sum( 1 for r in rows if r.get("eligible") and r.get( "v6_turning_point_actionable" ) ), ) if any( r.get("eligible") and r.get( "v6_turning_point_actionable" ) for r in rows ) else None ), # Backward-compatible V6.0 aliases now refer to Completion Benchmark. "eligible_signal_checkpoints": len( completion_benchmark_rows ), "mean_signal_evidence": ( sum( float( ( r.get( "v6_completion_benchmark_signal" ) or {} ).get("evidence") or 0.0 ) for r in completion_benchmark_rows ) / len(completion_benchmark_rows) if completion_benchmark_rows else None ), "favored_eventual_winner_rate": ( sum( 1 for r in completion_benchmark_rows if r.get( "v6_benchmark_favors_eventual_winner" ) ) / len(completion_benchmark_rows) if completion_benchmark_rows else None ), "raw_wrong_benchmark_right_checkpoints": sum( 1 for r in completion_benchmark_rows if not r.get("v0_leader_correct") and r.get( "v6_benchmark_favors_eventual_winner" ) ), } serious_rows = [ r for r in rows if r.get("eligible") and float( r.get( "actual_vote_counted_pct_backtest_only" ) or 0.0 ) >= 30.0 ] mature_rows = [ r for r in rows if r.get("eligible") and float( r.get( "actual_vote_counted_pct_backtest_only" ) or 0.0 ) >= 50.0 ] fifty_row = min( ( r for r in rows if r.get("eligible") ), key=lambda r: abs( float( r.get( "actual_vote_counted_pct_backtest_only" ) or 0.0 ) - 50.0 ), default=None, ) summary["v6_data_maturity"] = { "serious_threshold_percent": 30.0, "full_strength_threshold_percent": 50.0, "v6_30_plus": ( summarize_batch( serious_rows, "v6" ) if serious_rows else None ), "v6_50_plus": ( summarize_batch( mature_rows, "v6" ) if mature_rows else None ), "nearest_50_checkpoint": ( { "target_pct": fifty_row.get( "target_pct" ), "actual_vote_counted_pct": ( fifty_row.get( "actual_vote_counted_pct_backtest_only" ) ), "snapshot": fifty_row.get( "snapshot" ), "v6_leader_correct": fifty_row.get( "v6_leader_correct" ), "v6_predicted_leader": fifty_row.get( "v6_predicted_leader" ), "raw_current_leader": fifty_row.get( "raw_current_leader" ), "eventual_winner_mc_probability": ( fifty_row.get( "v6_eventual_winner_mc_probability" ) ), "completion_benchmark_favored": ( ( fifty_row.get( "v6_completion_benchmark_signal" ) or {} ).get( "favored_candidate" ) ), "completion_benchmark_actionable": ( fifty_row.get( "v6_completion_benchmark_actionable" ) ), "turning_point_candidate": ( ( fifty_row.get( "v6_turning_point" ) or {} ).get("candidate") ), "turning_point_actionable": ( fifty_row.get( "v6_turning_point_actionable" ) ), "turning_point_score": ( ( fifty_row.get( "v6_turning_point" ) or {} ).get("score") ), } if fifty_row else None ), } return { "schema_version": 33, "model": "V6_2_2_COMPARISON_BATCH", "source": { "race_url": base_url, "final_url": base_url + "latest.json", }, "race_name": final.get("election_name"), "final_votes_backtest_only": final_votes, "available_models": sorted(available_model_prefixes), "v2r_available": "v2r" in available_model_prefixes, "runoff_transfer_available": ( "runoff_transfer" in available_model_prefixes ), "latent_coalition_available": ( "latent" in available_model_prefixes ), "batch_bias_available": ( "batchbias" in available_model_prefixes ), "precinct_hierarchy_available": ( "precinct" in available_model_prefixes ), "v2r_baseline_url": ( round1_baseline_url(base_url) if "v2r" in available_model_prefixes else None ), "v3_reference_requested": normalize_reference_url( reference_url ), "media_market_requested": market_map_source, "media_market_available": any_market, "media_market_source": resolved_market_source, "media_market_auto_discovered": bool( resolved_market_source and not market_map_source ), "targets": targets, "rows": rows, "summary": summary, "diagnostics": { "tolerance_rule": ( "±2 through 10%; ±5 through 30%; ±7.5 after 30%." ), "backtest_warning": ( "The tested race latest.json is scoring-only. V6.2.2 uses " "only the selected snapshot and earlier dumps, plus optional " "pre-election references, same-cycle Round 1, and media-market " "mapping. Path Benchmark replays short reporting intervals; " "Completion Benchmark uses longer material-maturation windows. " "Path Benchmark remains 0% direct final weight. Completion " "Benchmark is the final-oriented benchmark layer, and V6.2.2 may " "also activate a Turning-Point Consensus when the newest material " "completion regime dominates the matured vote mass and receives " "independent confirmation." ), }, } def _first_matching_checkpoint( rows: List[Dict[str, Any]], predicate, ) -> Optional[Dict[str, Any]]: for row in rows: if row.get("eligible") and predicate(row): return { "target_pct": row.get("target_pct"), "actual_vote_counted_pct": row.get("actual_vote_counted_pct_backtest_only"), "snapshot": row.get("snapshot"), "snapshot_number": row.get("snapshot_number"), "timestamp": row.get("timestamp"), } return None def build_test_report( base_url: str, targets: List[float], reference_url: Optional[str] = None, market_map_source: Optional[str] = None, ) -> Dict[str, Any]: batch = batch_test_percentages( base_url, targets, reference_url, market_map_source ) rows = [ row for row in batch.get("rows") or [] if row.get("eligible") ] final = fetch_json(slug_base(base_url) + "latest.json") final_candidates = candidate_map(final) final_shares = shares_from_votes(final_candidates) final_ranked = rank(final_candidates, final_shares) final_leader = final_ranked[0] if final_ranked else None final_runner = ( final_ranked[1] if len(final_ranked) > 1 else None ) final_margin = ( 100.0 * ( final_shares.get(final_leader, 0.0) - final_shares.get(final_runner, 0.0) ) if final_leader and final_runner else None ) stable_correct = None for index, row in enumerate(rows): if row.get("v6_leader_correct") and all( later.get("v6_leader_correct") for later in rows[index:] ): stable_correct = { "target_pct": row.get("target_pct"), "actual_vote_counted_pct": row.get( "actual_vote_counted_pct_backtest_only" ), "snapshot": row.get("snapshot"), "snapshot_number": row.get("snapshot_number"), "timestamp": row.get("timestamp"), } break milestones = { "first_correct_winner": _first_matching_checkpoint( rows, lambda row: bool(row.get("v6_leader_correct")), ), "first_correct_winner_that_stayed_correct": ( stable_correct ), "first_actionable_turning_point_raw_wrong_turning_right": ( _first_matching_checkpoint( rows, lambda row: ( bool(final_leader) and row.get("raw_current_leader") != final_leader and row.get( "v6_turning_point_actionable" ) and row.get( "v6_turning_point_favors_eventual_winner" ) ), ) ), "first_useful_crossover_raw_wrong_v60_correct": ( _first_matching_checkpoint( rows, lambda row: ( bool(final_leader) and row.get("raw_current_leader") != final_leader and row.get("v6_predicted_leader") == final_leader ), ) ), "first_probability_crossover_raw_wrong_mc_right": ( _first_matching_checkpoint( rows, lambda row: ( bool(final_leader) and row.get("raw_current_leader") != final_leader and float( row.get( "v6_eventual_winner_mc_probability" ) or 0.0 ) > 0.5 ), ) ), "first_path_benchmark_crossover_raw_wrong_path_right": ( _first_matching_checkpoint( rows, lambda row: ( bool(final_leader) and row.get("raw_current_leader") != final_leader and row.get( "v6_path_benchmark_favors_eventual_winner" ) and float( ( row.get( "v6_path_benchmark_signal" ) or {} ).get("evidence") or 0.0 ) >= 0.25 ), ) ), "first_directional_completion_benchmark_crossover": ( _first_matching_checkpoint( rows, lambda row: ( bool(final_leader) and row.get("raw_current_leader") != final_leader and row.get( "v6_benchmark_favors_eventual_winner" ) and bool( ( row.get( "v6_completion_benchmark_signal" ) or {} ).get("directional") ) ), ) ), "first_actionable_completion_benchmark_crossover": ( _first_matching_checkpoint( rows, lambda row: ( bool(final_leader) and row.get("raw_current_leader") != final_leader and row.get( "v6_benchmark_favors_eventual_winner" ) and row.get( "v6_completion_benchmark_actionable" ) ), ) ), "first_benchmark_crossover_raw_wrong_benchmark_right": ( _first_matching_checkpoint( rows, lambda row: ( bool(final_leader) and row.get("raw_current_leader") != final_leader and row.get( "v6_benchmark_favors_eventual_winner" ) and row.get( "v6_completion_benchmark_actionable" ) ), ) ), "first_independent_consensus_raw_wrong_consensus_right": ( _first_matching_checkpoint( rows, lambda row: ( bool(final_leader) and row.get("raw_current_leader") != final_leader and ( row.get("v6_crossover_consensus") or {} ).get("eligible") and ( row.get("v6_crossover_consensus") or {} ).get("candidate") == final_leader ), ) ), "first_correct_at_or_after_30_percent": ( _first_matching_checkpoint( rows, lambda row: ( float( row.get( "actual_vote_counted_pct_backtest_only" ) or 0.0 ) >= 30.0 and bool( row.get("v6_leader_correct") ) ), ) ), "checkpoint_nearest_50_percent": min( ( { "target_pct": row.get("target_pct"), "actual_vote_counted_pct": row.get( "actual_vote_counted_pct_backtest_only" ), "snapshot": row.get("snapshot"), "snapshot_number": row.get( "snapshot_number" ), "timestamp": row.get("timestamp"), "v6_leader_correct": row.get( "v6_leader_correct" ), "v6_predicted_leader": row.get( "v6_predicted_leader" ), "raw_current_leader": row.get( "raw_current_leader" ), "completion_benchmark_favored": ( ( row.get( "v6_completion_benchmark_signal" ) or {} ).get( "favored_candidate" ) ), "completion_benchmark_actionable": ( row.get( "v6_completion_benchmark_actionable" ) ), "turning_point_candidate": ( ( row.get( "v6_turning_point" ) or {} ).get("candidate") ), "turning_point_actionable": row.get( "v6_turning_point_actionable" ), "turning_point_score": ( ( row.get( "v6_turning_point" ) or {} ).get("score") ), "eventual_winner_mc_probability": ( row.get( "v6_eventual_winner_mc_probability" ) ), } for row in rows if row.get("eligible") ), key=lambda item: abs( float( item.get( "actual_vote_counted_pct" ) or 0.0 ) - 50.0 ), default=None, ), "first_within_5_points": _first_matching_checkpoint( rows, lambda row: ( row.get("v6_winner_margin_error_points") is not None and abs( float( row[ "v6_winner_margin_error_points" ] ) ) <= 5.0 ), ), "first_within_2_points": _first_matching_checkpoint( rows, lambda row: ( row.get("v6_winner_margin_error_points") is not None and abs( float( row[ "v6_winner_margin_error_points" ] ) ) <= 2.0 ), ), "first_within_1_point": _first_matching_checkpoint( rows, lambda row: ( row.get("v6_winner_margin_error_points") is not None and abs( float( row[ "v6_winner_margin_error_points" ] ) ) <= 1.0 ), ), } standalone_keys = [ ("v0", "v0"), ("v1", "v1"), ("v2r", "v2r"), ("v3", "v3"), ("historical_candidate_geo", "histgeo"), ("latent_coalition", "latent"), ("batch_bias", "batchbias"), ("runoff_transfer", "runoff_transfer"), ("precinct_hierarchy", "precinct"), ("completion_benchmark", "benchmark_completion"), ] checkpoints = [] for row in rows: out = { "target_pct": row.get("target_pct"), "actual_vote_counted_pct": row.get( "actual_vote_counted_pct_backtest_only" ), "snapshot": row.get("snapshot"), "snapshot_number": row.get("snapshot_number"), "timestamp": row.get("timestamp"), "raw_current_leader": row.get( "raw_current_leader" ), "raw_current_margin_points": row.get( "raw_current_margin_points" ), "raw_eventual_winner_margin_points": row.get( "raw_eventual_winner_margin_points" ), } for display_key, prefix in standalone_keys: if row.get(prefix + "_predicted_leader") is None: out[display_key] = None else: out[display_key] = { "leader": row.get( prefix + "_predicted_leader" ), "margin": row.get( prefix + "_winner_projected_margin_points" ), "error": row.get( prefix + "_winner_margin_error_points" ), } out["v6_0"] = { "leader": row.get("v6_predicted_leader"), "margin": row.get( "v6_winner_projected_margin_points" ), "error": row.get( "v6_winner_margin_error_points" ), "leader_correct": row.get("v6_leader_correct"), "model_confidence": row.get( "v6_model_confidence" ), "weights": row.get("v6_weights"), "lead_correction_points": row.get( "v6_lead_correction_points" ), "components": row.get("v6_components"), "gate": row.get("v6_gate"), "horizon_validation": row.get( "v6_horizon_validation" ), "raw_guard": row.get("v6_raw_guard"), "completion_referee": row.get( "v6_completion_referee" ), "path_referee": row.get( "v6_path_referee" ), "path_benchmark_signal": row.get( "v6_path_benchmark_signal" ), "completion_benchmark_signal": row.get( "v6_completion_benchmark_signal" ), "benchmark_signal": row.get( "v6_benchmark_signal" ), "completion_benchmark_actionable": row.get( "v6_completion_benchmark_actionable" ), "maturity_info": row.get( "v6_maturity_info" ), "specialist_temporal_validation": row.get( "v6_specialist_temporal_validation" ), "crossover_consensus": row.get( "v6_crossover_consensus" ), "turning_point": row.get( "v6_turning_point" ), "turning_point_actionable": row.get( "v6_turning_point_actionable" ), "county_benchmarks": row.get( "v6_county_benchmarks" ), "reporting_path_profile": row.get( "v6_reporting_path_profile" ), "monte_carlo": row.get("v6_monte_carlo"), "eventual_winner_mc_probability": row.get( "v6_eventual_winner_mc_probability" ), "candidate_projection_breakdown": row.get( "v6_candidate_projection_breakdown" ), "candidate_rank_summary": row.get( "v6_candidate_rank_summary" ), } out["v6_1"] = out["v6_0"] out["v6_2"] = out["v6_0"] out["v6_2_1"] = out["v6_0"] out["temporal"] = { "confidence": row.get( "v6_temporal_confidence" ), "weight": 0.0, "recent_votes": row.get( "v6_temporal_recent_votes" ), "trend": row.get("v6_temporal_trend"), } out["media_markets"] = row.get( "v6_top_media_markets" ) out["reference_library"] = row.get( "v6_reference_library" ) checkpoints.append(out) return { "schema_version": 33, "model": "AFI_NEEDLE_V6_2_2_TEST_REPORT", "generated_at_utc": ( datetime.utcnow().isoformat(timespec="seconds") + "Z" ), "race": { "name": batch.get("race_name"), "race_url": slug_base(base_url), "final_leader": final_leader, "final_runner_up": final_runner, "final_margin_points": final_margin, "final_votes": batch.get( "final_votes_backtest_only" ), }, "inputs": { "targets": targets, "reference_requested": normalize_reference_url( reference_url ), "media_market_source": batch.get( "media_market_source" ), }, "summary": batch.get("summary"), "milestones": milestones, "checkpoints": checkpoints, "batch": batch, "notes": [ "latest.json is used only as the backtest answer key.", "Every V6.2.2 checkpoint can use only that snapshot and earlier dumps.", "Same-cycle Round 1 is a legitimate pre-runoff input.", "Path Benchmark is scored on short next-reporting intervals and has zero direct final weight.", "Completion Benchmark is scored only on material county-maturation windows and is the only benchmark layer eligible for final influence.", "Specialist final trust combines spatial/held-out validation with temporal projection survival.", "V6.2.2 treats roughly 30% estimated counted as serious-data territory and 50%+ as full-strength final-completion territory.", "Completion validation excludes fake equal-share fallbacks.", "Monte Carlo is explicitly centered on the deterministic V6.2.2 projection.", "Batch checkpoints include every candidate's current share/rank, V6.2.2 projected share/rank, and scoring-only final share/rank so hidden movement in multi-candidate races can be audited.", ], } def render_test_report_text( report: Dict[str, Any] ) -> str: race = report.get("race") or {} summary = (report.get("summary") or {}).get("v6") or {} milestones = report.get("milestones") or {} def checkpoint_text(value): if not value: return "—" return ( f"{float(value.get('actual_vote_counted_pct') or 0):.2f}% counted " f"(target {float(value.get('target_pct') or 0):.0f}%, " f"snapshot {value.get('snapshot')})" ) lines = [ "AFI NEEDLE V6.2.2 TEST REPORT", "=" * 32, f"Race: {race.get('name') or '—'}", f"Final winner: {race.get('final_leader') or '—'}", f"Final margin: {float(race.get('final_margin_points') or 0):+.3f}", f"Final votes: {int(race.get('final_votes') or 0):,}", "", "V6.2.2 SUMMARY", f"Winner accuracy: {100.0 * float(summary.get('leader_accuracy') or 0):.1f}%", f"Margin MAE: {float(summary.get('mean_absolute_winner_margin_error_points') or 0):.3f}", f"Winner-share MAE: {float(summary.get('mean_absolute_winner_share_error_points') or 0):.3f}", f"All-candidate MAE: {float(summary.get('mean_all_candidate_share_mae_points') or 0):.3f}", f"Within ±1: {100.0 * float(summary.get('within_1_point') or 0):.1f}%", f"Within ±2: {100.0 * float(summary.get('within_2_points') or 0):.1f}%", f"Within ±5: {100.0 * float(summary.get('within_5_points') or 0):.1f}%", "", "MILESTONES", "First correct winner: " + checkpoint_text( milestones.get("first_correct_winner") ), "First correct winner that stayed correct: " + checkpoint_text( milestones.get( "first_correct_winner_that_stayed_correct" ) ), "First deterministic crossover (raw wrong / V6 correct): " + checkpoint_text( milestones.get( "first_useful_crossover_raw_wrong_v60_correct" ) ), "First probability crossover (raw wrong / MC >50): " + checkpoint_text( milestones.get( "first_probability_crossover_raw_wrong_mc_right" ) ), "First Path Benchmark crossover (diagnostic only): " + checkpoint_text( milestones.get( "first_path_benchmark_crossover_raw_wrong_path_right" ) ), "First directional Completion Benchmark crossover: " + checkpoint_text( milestones.get( "first_directional_completion_benchmark_crossover" ) ), "First ACTIONABLE Completion Benchmark crossover: " + checkpoint_text( milestones.get( "first_actionable_completion_benchmark_crossover" ) ), "First ACTIONABLE Turning-Point crossover: " + checkpoint_text( milestones.get( "first_actionable_turning_point_raw_wrong_turning_right" ) ), "First independent expert consensus crossover: " + checkpoint_text( milestones.get( "first_independent_consensus_raw_wrong_consensus_right" ) ), "First within ±5: " + checkpoint_text( milestones.get("first_within_5_points") ), "First within ±2: " + checkpoint_text( milestones.get("first_within_2_points") ), "First within ±1: " + checkpoint_text( milestones.get("first_within_1_point") ), "", "CHECKPOINTS", ( "Counted | Raw | V6.2.2 | Error | RawGuard | " "Completion Bench | Bench Ev. | Bench Excess | " "Stage | Consensus | MC Winner | Top Experts" ), ] for row in report.get("checkpoints") or []: v6 = row.get("v6_2") or row.get("v6_1") or row.get("v6_0") or {} weights = v6.get("weights") or {} weight_text = ", ".join( f"{key}={100.0 * float(value):.0f}%" for key, value in sorted( weights.items(), key=lambda item: item[1], reverse=True, )[:5] ) raw_margin = row.get("raw_current_margin_points") v6_margin = v6.get("margin") error = v6.get("error") guard = v6.get("raw_guard") or {} bench = ( v6.get("completion_benchmark_signal") or v6.get("benchmark_signal") or {} ) maturity = v6.get("maturity_info") or {} consensus = v6.get("crossover_consensus") or {} mc_prob = v6.get( "eventual_winner_mc_probability" ) lines.append( "{counted:6.2f}% | {raw_leader} {raw_margin:>7} | " "{v6_leader} {v6_margin:>7} | {error:>7} | " "{guard:5.0f}% | {bench_candidate:12} | {bench_ev:5.0f}% | " "{bench_excess:>7} | {stage:9} | {consensus:12} | {mc:>6} | {weights}".format( counted=float( row.get("actual_vote_counted_pct") or 0 ), raw_leader=row.get( "raw_current_leader" ) or "—", raw_margin=( f"{float(raw_margin):+.2f}" if raw_margin is not None else "—" ), v6_leader=v6.get("leader") or "—", v6_margin=( f"{float(v6_margin):+.2f}" if v6_margin is not None else "—" ), error=( f"{float(error):+.2f}" if error is not None else "—" ), guard=100.0 * float( guard.get("release_fraction") or 0.0 ), bench_candidate=str( bench.get("favored_candidate") or "—" )[:12], bench_ev=100.0 * float( bench.get("evidence") or 0.0 ), bench_excess=( f"{float(bench.get('favored_weighted_excess_points')):+.2f}" if bench.get( "favored_weighted_excess_points" ) is not None else "—" ), stage=str( maturity.get("stage") or "—" )[:9], consensus=( str(consensus.get("candidate") or "—")[:12] if consensus.get("eligible") else "—" ), mc=( f"{100.0 * float(mc_prob):.1f}%" if mc_prob is not None else "—" ), weights=weight_text, ) ) return "\n".join(lines) + "\n" def run_training_battery( manifest_source: str, default_targets: Optional[List[float]] = None, ) -> Dict[str, Any]: """ Automatic multi-race historical battery. Manifest format: { "races": [ {"race_url": "...", "baseline_url": "...", "market_map": "...", "label": "..."} ], "targets": [1,3,5,10,20,30,40,50,60,70,80,90] } This function performs no parameter fitting. It is an out-of-sample evaluation harness so tuning can be done on one manifest and checked on a separate holdout manifest. """ manifest = load_json_source(manifest_source) if not isinstance(manifest, dict): raise ValueError("Training battery manifest must be a JSON object.") races = manifest.get("races") or [] if not isinstance(races, list) or not races: raise ValueError("Training battery manifest contains no races.") targets = manifest.get("targets") or default_targets or [ 1, 3, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90 ] race_results = [] aggregate_rows = [] for index, item in enumerate(races): if isinstance(item, str): item = {"race_url": item} if not isinstance(item, dict) or not item.get("race_url"): continue try: result = batch_test_percentages( item["race_url"], [float(x) for x in targets], item.get("baseline_url"), item.get("market_map"), ) race_results.append({ "label": item.get("label") or result.get("race_name") or f"Race {index+1}", "race_url": item["race_url"], "summary": result.get("summary"), "available_models": result.get("available_models"), "status": "ok", }) for row in result.get("rows") or []: if row.get("eligible"): aggregate_rows.append(row) except Exception as exc: race_results.append({ "label": item.get("label") or f"Race {index+1}", "race_url": item.get("race_url"), "status": "error", "error": str(exc), }) prefixes = [ "v0", "v1", "v2r", "v3", "histgeo", "latent", "batchbias", "runoff_transfer", "precinct", "v6" ] aggregate = {} for prefix in prefixes: if any( row.get(prefix + "_predicted_leader") is not None for row in aggregate_rows ): aggregate[prefix] = summarize_batch(aggregate_rows, prefix) else: aggregate[prefix] = None mc_probs = [ float(row["v6_eventual_winner_mc_probability"]) for row in aggregate_rows if row.get("v6_eventual_winner_mc_probability") is not None ] aggregate["monte_carlo"] = { "eligible_tests": len(mc_probs), "mean_eventual_winner_probability": ( sum(mc_probs) / len(mc_probs) if mc_probs else None ), "brier": ( sum((1.0 - p) ** 2 for p in mc_probs) / len(mc_probs) if mc_probs else None ), "log_loss": ( sum(-math.log(max(1e-6, p)) for p in mc_probs) / len(mc_probs) if mc_probs else None ), } reporting_profiles = build_historical_reporting_profiles(races) saved_reporting_profiles = {} if reporting_profiles: REPORTING_PROFILE_DIR.mkdir(parents=True, exist_ok=True) for state, profile in reporting_profiles.items(): profile_path = REPORTING_PROFILE_DIR / f"{state.lower()}_reporting_profile.json" save_json(profile_path, profile) saved_reporting_profiles[state] = str(profile_path) return { "schema_version": 2, "model": "AFI_NEEDLE_V6_2_2_HISTORICAL_BATTERY", "generated_at_utc": datetime.utcnow().isoformat(timespec="seconds") + "Z", "targets": targets, "races_requested": len(races), "races_completed": sum(1 for r in race_results if r.get("status") == "ok"), "races": race_results, "aggregate": aggregate, "reporting_profiles": reporting_profiles, "saved_reporting_profiles": saved_reporting_profiles, "method_note": ( "Use separate tuning and holdout manifests. The battery aggregates " "winner accuracy, margin/share error, and Monte Carlo calibration " "without leaking tested-race future snapshots into any checkpoint." ), } def _write_statewide_battery_status( status_path: Optional[str], payload: Dict[str, Any], ) -> None: if not status_path: return try: path = Path(status_path) path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix( path.suffix + ".tmp" ) tmp.write_text( json.dumps( payload, indent=2, ensure_ascii=False, ), encoding="utf-8", ) tmp.replace(path) except Exception: # Status reporting must never kill the actual battery. pass def run_statewide_checkpoint_battery( manifest_source: str, ) -> Dict[str, Any]: """ Test every supplied statewide race at one common backtest checkpoint. The PHP Test Lab discovers the state's district-00 race directories and writes a small manifest: { "state": "MI", "round": "primary", "target_pct": 30, "races": [ { "race_url": ".../Governor/00/republican/", "office": "Governor", "party": "republican", "label": "Governor — Republican", "baseline_url": ".../mi_republican_library.json", "market_map": ".../mi_media_markets.js" } ] } Each race is independently replayed at the snapshot nearest target_pct using the normal checkpoint tolerance. No tested-race final data is exposed to the projection; latest.json is still scoring-only. """ manifest = load_json_source(manifest_source) if not isinstance(manifest, dict): raise ValueError( "Statewide battery manifest must be a JSON object." ) races = manifest.get("races") or [] status_path = ( str( manifest.get("status_path") or "" ).strip() or None ) job_id = ( str( manifest.get("job_id") or "" ).strip() or None ) if not isinstance(races, list) or not races: raise ValueError( "Statewide battery manifest contains no races." ) target = float(manifest.get("target_pct") or 0.0) if not (0.0 < target <= 100.0): raise ValueError( "Statewide battery target_pct must be between 0 and 100." ) race_results: List[Dict[str, Any]] = [] eligible_results: List[Dict[str, Any]] = [] _write_statewide_battery_status( status_path, { "schema_version": 1, "job_id": job_id, "status": "running", "state": manifest.get("state"), "round": manifest.get("round"), "target_pct": target, "races_requested": len(races), "races_completed": 0, "races_failed": 0, "current_race": None, "started_at_utc": ( datetime.utcnow().isoformat( timespec="seconds" ) + "Z" ), "output_path": manifest.get( "output_path" ), "report_name": manifest.get( "report_name" ), "text_output_path": manifest.get( "text_output_path" ), "text_report_name": manifest.get( "text_report_name" ), }, ) total_rank_calls = 0 exact_rank_calls = 0 weighted_abs_rank_error = 0.0 for index, item in enumerate(races): if isinstance(item, str): item = {"race_url": item} if not isinstance(item, dict) or not item.get("race_url"): continue race_url = str(item["race_url"]) label = ( item.get("label") or f"Statewide Race {index + 1}" ) office = item.get("office") party = item.get("party") _write_statewide_battery_status( status_path, { "schema_version": 1, "job_id": job_id, "status": "running", "state": manifest.get("state"), "round": manifest.get("round"), "target_pct": target, "races_requested": len(races), "races_completed": index, "races_failed": sum( 1 for r in race_results if r.get("status") == "error" ), "current_race": label, "output_path": manifest.get( "output_path" ), "report_name": manifest.get( "report_name" ), "text_output_path": manifest.get( "text_output_path" ), "text_report_name": manifest.get( "text_report_name" ), }, ) try: batch = batch_test_percentages( race_url, [target], item.get("baseline_url"), item.get("market_map"), # Battery-only performance mode. Monte Carlo does not alter # deterministic winner/share/margin projections. monte_carlo_simulations=400, benchmark_replay_intervals=3, ) rows = batch.get("rows") or [] row = rows[0] if rows else None if not row: race_results.append({ "label": label, "office": office, "party": party, "race_url": race_url, "status": "error", "error": "No checkpoint row was produced.", }) continue final = fetch_json( slug_base(race_url) + "latest.json" ) final_candidates = candidate_map(final) final_shares = shares_from_votes( final_candidates ) final_ranked = rank( final_candidates, final_shares ) final_leader = ( final_ranked[0] if final_ranked else None ) final_runner = ( final_ranked[1] if len(final_ranked) > 1 else None ) final_margin = ( 100.0 * ( final_shares.get( final_leader, 0.0 ) - final_shares.get( final_runner, 0.0 ) ) if final_leader and final_runner else None ) audit = ( row.get( "v6_candidate_projection_breakdown" ) or [] ) projected_sorted = sorted( audit, key=lambda r: ( r.get("projected_rank") if r.get("projected_rank") is not None else 999 ), ) projected_leader = ( projected_sorted[0].get("name") if projected_sorted else row.get("v6_predicted_leader") ) projected_runner = ( projected_sorted[1].get("name") if len(projected_sorted) > 1 else None ) projected_margin = None if len(projected_sorted) > 1: projected_margin = ( float( projected_sorted[0].get( "projected_share_points" ) or 0.0 ) - float( projected_sorted[1].get( "projected_share_points" ) or 0.0 ) ) rank_summary = ( row.get( "v6_candidate_rank_summary" ) or {} ) scored_ranks = int( rank_summary.get( "candidates_scored" ) or 0 ) exact_ranks = int( rank_summary.get( "exact_rank_calls" ) or 0 ) mean_rank_error = float( rank_summary.get( "mean_absolute_rank_error" ) or 0.0 ) total_rank_calls += scored_ranks exact_rank_calls += exact_ranks weighted_abs_rank_error += ( mean_rank_error * scored_ranks ) result = { "label": label, "office": office, "party": party, "race_url": race_url, "race_name": ( batch.get("race_name") or final.get("election_name") or label ), "status": ( "ok" if row.get("eligible") else "ineligible" ), "target_pct": target, "actual_vote_counted_pct": ( row.get( "actual_vote_counted_pct_backtest_only" ) ), "checkpoint_distance_points": ( row.get( "checkpoint_distance_points" ) ), "checkpoint_tolerance_points": ( row.get( "checkpoint_tolerance_points" ) ), "snapshot": row.get("snapshot"), "snapshot_number": ( row.get("snapshot_number") ), "timestamp": row.get("timestamp"), "final_leader": final_leader, "final_runner_up": final_runner, "final_margin_points": final_margin, "final_votes": sum( int(c.get("votes") or 0) for c in final_candidates.values() ), "raw_current_leader": ( row.get("raw_current_leader") ), "raw_current_margin_points": ( row.get( "raw_current_margin_points" ) ), "raw_leader_correct": ( row.get("raw_current_leader") == final_leader if row.get("eligible") else None ), "v6_predicted_leader": ( row.get("v6_predicted_leader") ), "v6_projected_runner_up": ( projected_runner ), "v6_projected_top_margin_points": ( projected_margin ), "v6_leader_correct": ( row.get("v6_leader_correct") ), "v6_eventual_winner_margin_error_points": ( row.get( "v6_winner_margin_error_points" ) ), "v6_absolute_margin_error_points": ( abs( float( row.get( "v6_winner_margin_error_points" ) or 0.0 ) ) if row.get( "v6_winner_margin_error_points" ) is not None else None ), "v6_winner_share_error_points": ( row.get( "v6_winner_share_error_points" ) ), "v6_candidate_share_mae_points": ( row.get( "v6_candidate_share_mae_points" ) ), "v6_eventual_winner_mc_probability": ( row.get( "v6_eventual_winner_mc_probability" ) ), "v6_model_confidence": ( row.get("v6_model_confidence") ), "candidate_rank_summary": ( rank_summary ), "candidate_projection_breakdown": ( audit ), "turning_point": ( row.get("v6_turning_point") ), "completion_benchmark_signal": ( row.get( "v6_completion_benchmark_signal" ) ), "path_benchmark_signal": ( row.get( "v6_path_benchmark_signal" ) ), "crossover_consensus": ( row.get( "v6_crossover_consensus" ) ), "turning_point_trust_reallocation": ( ( row.get("v6_gate") or {} ).get( "turning_point_trust_reallocation" ) ), "reference_requested": ( batch.get( "v3_reference_requested" ) ), "media_market_source": ( batch.get( "media_market_source" ) ), } race_results.append(result) if row.get("eligible"): eligible_results.append(result) except Exception as exc: race_results.append({ "label": label, "office": office, "party": party, "race_url": race_url, "status": "error", "error": str(exc), }) _write_statewide_battery_status( status_path, { "schema_version": 1, "job_id": job_id, "status": "running", "state": manifest.get("state"), "round": manifest.get("round"), "target_pct": target, "races_requested": len(races), "races_completed": index + 1, "races_failed": sum( 1 for r in race_results if r.get("status") == "error" ), "current_race": None, "last_completed_race": label, "output_path": manifest.get( "output_path" ), "report_name": manifest.get( "report_name" ), "text_output_path": manifest.get( "text_output_path" ), "text_report_name": manifest.get( "text_report_name" ), }, ) completed = len(eligible_results) correct = sum( 1 for r in eligible_results if r.get("v6_leader_correct") ) raw_correct = sum( 1 for r in eligible_results if r.get("raw_leader_correct") ) margin_errors = [ float(r["v6_absolute_margin_error_points"]) for r in eligible_results if r.get( "v6_absolute_margin_error_points" ) is not None ] candidate_maes = [ float(r["v6_candidate_share_mae_points"]) for r in eligible_results if r.get( "v6_candidate_share_mae_points" ) is not None ] winner_share_errors = [ abs( float( r["v6_winner_share_error_points"] ) ) for r in eligible_results if r.get( "v6_winner_share_error_points" ) is not None ] mc_probs = [ float( r[ "v6_eventual_winner_mc_probability" ] ) for r in eligible_results if r.get( "v6_eventual_winner_mc_probability" ) is not None ] raw_wrong_v6_right = [ r["label"] for r in eligible_results if ( not r.get("raw_leader_correct") and r.get("v6_leader_correct") ) ] raw_right_v6_wrong = [ r["label"] for r in eligible_results if ( r.get("raw_leader_correct") and not r.get("v6_leader_correct") ) ] v6_wrong = [ r["label"] for r in eligible_results if not r.get("v6_leader_correct") ] summary = { "races_requested": len(races), "races_with_eligible_checkpoint": ( completed ), "races_ineligible_or_failed": ( len(races) - completed ), "v6_correct_races": correct, "v6_wrong_races": completed - correct, "v6_winner_accuracy": ( correct / completed if completed else None ), "raw_correct_races": raw_correct, "raw_winner_accuracy": ( raw_correct / completed if completed else None ), "every_eligible_race_correct": bool( completed > 0 and correct == completed ), "v6_margin_mae_points": ( sum(margin_errors) / len(margin_errors) if margin_errors else None ), "v6_winner_share_mae_points": ( sum(winner_share_errors) / len(winner_share_errors) if winner_share_errors else None ), "v6_all_candidate_share_mae_points": ( sum(candidate_maes) / len(candidate_maes) if candidate_maes else None ), "candidate_rank_calls": ( total_rank_calls ), "candidate_exact_rank_calls": ( exact_rank_calls ), "candidate_exact_rank_accuracy": ( exact_rank_calls / total_rank_calls if total_rank_calls else None ), "candidate_mean_absolute_rank_error": ( weighted_abs_rank_error / total_rank_calls if total_rank_calls else None ), "mean_eventual_winner_mc_probability": ( sum(mc_probs) / len(mc_probs) if mc_probs else None ), "raw_wrong_v6_right_count": len( raw_wrong_v6_right ), "raw_wrong_v6_right_races": ( raw_wrong_v6_right ), "raw_right_v6_wrong_count": len( raw_right_v6_wrong ), "raw_right_v6_wrong_races": ( raw_right_v6_wrong ), "v6_wrong_race_labels": v6_wrong, } result = { "schema_version": 1, "model": ( "AFI_NEEDLE_V6_2_2_STATEWIDE_CHECKPOINT_BATTERY" ), "generated_at_utc": ( datetime.utcnow().isoformat( timespec="seconds" ) + "Z" ), "state": manifest.get("state"), "round": manifest.get("round"), "target_pct": target, "summary": summary, "races": race_results, "method_note": ( "All supplied contested statewide races are evaluated at one common " "target percentage of eventual vote. Each race uses the nearest " "historical snapshot within the ordinary checkpoint tolerance. " "latest.json is scoring-only." ), } _write_statewide_battery_status( status_path, { "schema_version": 1, "job_id": job_id, "status": "complete", "state": manifest.get("state"), "round": manifest.get("round"), "target_pct": target, "races_requested": len(races), "races_completed": len(race_results), "races_failed": sum( 1 for r in race_results if r.get("status") == "error" ), "current_race": None, "completed_at_utc": ( datetime.utcnow().isoformat( timespec="seconds" ) + "Z" ), "output_path": manifest.get( "output_path" ), "report_name": manifest.get( "report_name" ), "text_output_path": manifest.get( "text_output_path" ), "text_report_name": manifest.get( "text_report_name" ), "summary": summary, }, ) return result def render_statewide_battery_text( report: Dict[str, Any] ) -> str: summary = report.get("summary") or {} races = report.get("races") or [] def pct(value, digits=1): if value is None: return "—" return f"{100.0 * float(value):.{digits}f}%" def pts(value, digits=2, signed=False): if value is None: return "—" number = float(value) if signed: return f"{number:+.{digits}f} pts" return f"{number:.{digits}f} pts" lines = [ "AFI NEEDLE V6.2.2 STATEWIDE SNAPSHOT REPORT", "=" * 47, f"State: {report.get('state') or '—'}", f"Round: {report.get('round') or '—'}", f"Target checkpoint: {float(report.get('target_pct') or 0):.1f}% of eventual vote", f"Generated: {report.get('generated_at_utc') or '—'}", "", "STATEWIDE SUMMARY", "-" * 17, ( "Eligible races: " f"{int(summary.get('races_with_eligible_checkpoint') or 0)} / " f"{int(summary.get('races_requested') or 0)}" ), ( "V6.2.2 correct races: " f"{int(summary.get('v6_correct_races') or 0)} / " f"{int(summary.get('races_with_eligible_checkpoint') or 0)}" ), f"V6.2.2 winner accuracy: {pct(summary.get('v6_winner_accuracy'))}", f"Raw-board winner accuracy: {pct(summary.get('raw_winner_accuracy'))}", f"Margin MAE: {pts(summary.get('v6_margin_mae_points'))}", f"Winner-share MAE: {pts(summary.get('v6_winner_share_mae_points'))}", f"All-candidate share MAE: {pts(summary.get('v6_all_candidate_share_mae_points'))}", ( "Exact candidate-rank accuracy: " f"{pct(summary.get('candidate_exact_rank_accuracy'))} " f"({int(summary.get('candidate_exact_rank_calls') or 0)} / " f"{int(summary.get('candidate_rank_calls') or 0)})" ), ( "Mean absolute rank error: " + ( f"{float(summary.get('candidate_mean_absolute_rank_error')):.3f}" if summary.get("candidate_mean_absolute_rank_error") is not None else "—" ) ), ( "Mean eventual-winner MC probability: " f"{pct(summary.get('mean_eventual_winner_mc_probability'))}" ), ( "Raw wrong / Needle right: " f"{int(summary.get('raw_wrong_v6_right_count') or 0)}" ), ( "Raw right / Needle wrong: " f"{int(summary.get('raw_right_v6_wrong_count') or 0)}" ), ( "Every eligible race correct: " + ("YES" if summary.get("every_eligible_race_correct") else "NO") ), ] if summary.get("raw_wrong_v6_right_races"): lines.extend([ "", "RAW WRONG / NEEDLE RIGHT", "-" * 24, ]) for label in summary["raw_wrong_v6_right_races"]: lines.append(f"- {label}") if summary.get("v6_wrong_race_labels"): lines.extend([ "", "MISSED RACES", "-" * 12, ]) for label in summary["v6_wrong_race_labels"]: lines.append(f"- {label}") lines.extend([ "", "RACE-BY-RACE", "=" * 12, ]) for index, race in enumerate(races, start=1): status = race.get("status") or "unknown" lines.extend([ "", f"{index}. {race.get('label') or race.get('race_name') or 'Race'}", "-" * max( 8, len( f"{index}. {race.get('label') or race.get('race_name') or 'Race'}" ) ), f"Status: {status}", ]) if status != "ok": lines.append( f"Error / reason: {race.get('error') or 'No eligible checkpoint inside tolerance.'}" ) if race.get("actual_vote_counted_pct") is not None: lines.append( "Nearest actual checkpoint: " f"{float(race['actual_vote_counted_pct']):.2f}%" ) continue actual = race.get("actual_vote_counted_pct") lines.extend([ ( "Checkpoint: " f"{float(actual):.2f}% counted" if actual is not None else "Checkpoint: —" ), f"Snapshot: {race.get('snapshot') or '—'}", f"Final winner: {race.get('final_leader') or '—'}", f"Final runner-up: {race.get('final_runner_up') or '—'}", f"Raw call: {race.get('raw_current_leader') or '—'}" + ( " [CORRECT]" if race.get("raw_leader_correct") else " [WRONG]" ), f"V6.2.2 call: {race.get('v6_predicted_leader') or '—'}" + ( " [CORRECT]" if race.get("v6_leader_correct") else " [WRONG]" ), ( "Projected top-two margin: " + pts( race.get("v6_projected_top_margin_points") ) ), ( "Final top-two margin: " + pts(race.get("final_margin_points")) ), ( "Absolute margin error: " + pts( race.get("v6_absolute_margin_error_points") ) ), ( "Winner-share error: " + pts( abs( float( race.get("v6_winner_share_error_points") or 0.0 ) ) if race.get("v6_winner_share_error_points") is not None else None ) ), ( "All-candidate share MAE: " + pts( race.get("v6_candidate_share_mae_points") ) ), ( "Eventual-winner MC probability: " + pct( race.get("v6_eventual_winner_mc_probability") ) ), ]) rank_summary = race.get("candidate_rank_summary") or {} if rank_summary: lines.extend([ ( "Exact rank calls: " f"{int(rank_summary.get('exact_rank_calls') or 0)} / " f"{int(rank_summary.get('candidates_scored') or 0)} " f"({pct(rank_summary.get('exact_rank_accuracy'))})" ), ( "Mean absolute rank error: " + ( f"{float(rank_summary.get('mean_absolute_rank_error')):.3f}" if rank_summary.get("mean_absolute_rank_error") is not None else "—" ) ), ]) candidates = race.get("candidate_projection_breakdown") or [] if candidates: lines.extend([ "", "Candidate projections:", ]) ordered = sorted( candidates, key=lambda row: ( row.get("projected_rank") if row.get("projected_rank") is not None else 999, str(row.get("name") or ""), ), ) for candidate in ordered: current_rank = candidate.get("current_rank") projected_rank = candidate.get("projected_rank") final_rank = candidate.get("final_rank_backtest_only") current_share = candidate.get("current_share_points") projected_share = candidate.get("projected_share_points") final_share = candidate.get("final_share_points_backtest_only") share_error = candidate.get("share_error_points_backtest_only") hidden = candidate.get("hidden_movement") rank_correct = candidate.get("rank_call_correct_backtest_only") movement = candidate.get("projected_rank_movement") if movement is None or int(movement) == 0: movement_text = "no rank move" elif int(movement) > 0: movement_text = f"up {int(movement)}" else: movement_text = f"down {abs(int(movement))}" lines.append( " - " f"{candidate.get('name') or '—'}: " f"current #{current_rank if current_rank is not None else '—'} " f"({float(current_share):.2f}%" if current_share is not None else " - " f"{candidate.get('name') or '—'}: current #" f"{current_rank if current_rank is not None else '—'} (—" ) # Replace the partially-built line above with one coherent row. lines[-1] = ( " - " f"{candidate.get('name') or '—'} | " f"current #{current_rank if current_rank is not None else '—'} " f"{f'{float(current_share):.2f}%' if current_share is not None else '—'} | " f"projected #{projected_rank if projected_rank is not None else '—'} " f"{f'{float(projected_share):.2f}%' if projected_share is not None else '—'} | " f"{movement_text} | " f"final #{final_rank if final_rank is not None else '—'} " f"{f'{float(final_share):.2f}%' if final_share is not None else '—'} | " f"share err {pts(share_error, signed=True)} | " f"rank {'CORRECT' if rank_correct else 'WRONG'}" + (" | HIDDEN MOVEMENT" if hidden else "") ) lines.extend([ "", "NOTES", "=" * 5, "- latest.json is scoring-only.", "- Each race uses only the selected historical snapshot and earlier dumps.", "- The common checkpoint is selected independently per race using the normal tolerance.", "- Uncontested races are excluded from statewide winner accuracy.", "- Candidate final ranks/shares are scoring-only backtest fields.", "- Statewide battery Monte Carlo uses the reduced battery sampling count for speed; deterministic projections are unchanged.", ]) return "\n".join(lines) + "\n" def _live_probability_rating(probability: float) -> str: p = max(0.0, min(1.0, float(probability or 0.0))) if p >= 0.99: return "NEAR CERTAIN" if p >= 0.95: return "OVERWHELMING FAVORITE" if p >= 0.80: return "STRONGLY FAVORED" if p >= 0.65: return "FAVORED" if p >= 0.55: return "SLIGHT EDGE" return "TOSS-UP" def _live_forecast_status(v6: Dict[str, Any], top_probability: float) -> str: maturity = v6.get("maturity_info") or {} stage = str(maturity.get("stage") or "early").casefold() turning = v6.get("turning_point") or {} completion = v6.get("completion_referee") or {} quality = float(v6.get("weighted_final_quality") or 0.0) completion_evidence = float(completion.get("evidence") or 0.0) if stage == "early": return "EARLY" if turning.get("actionable"): return "VOLATILE" if stage == "exploratory": return "DEVELOPING" if stage == "serious": if completion_evidence >= 0.60 and quality >= 0.55 and top_probability >= 0.65: return "STABILIZING" return "DEVELOPING" if stage == "mature": if completion_evidence >= 0.65 and quality >= 0.60: return "STABLE" return "MATURE" return "DEVELOPING" def build_live_forecast_payload( race_url: str, snapshot_name: str, baseline_url: Optional[str] = None, market_map_source: Optional[str] = None, monte_carlo_simulations: int = 1200, ) -> Dict[str, Any]: """ Run the production V6.2.2 projection on an immutable live snapshot and return a small frontend-safe payload. Final-result/scoring fields from the Test Lab are deliberately omitted. """ result = project_v4( race_url, snapshot_name, baseline_url, market_map_source, monte_carlo_simulations=max(1000, int(monte_carlo_simulations or 1200)), ) v6 = result.get("v6") or {} mc = result.get("monte_carlo") or {} win_probabilities = mc.get("win_probabilities") or {} projection = result.get("projection") or [] candidate_rows = [] for row in projection: name = str(row.get("name") or "").strip() if not name: continue probability = max(0.0, min(1.0, float(win_probabilities.get(name) or 0.0))) candidate_rows.append({ "name": name, "party": row.get("party"), "color": row.get("color"), "current_votes": int(row.get("current_votes") or 0), "current_share_points": 100.0 * float(row.get("current_share") or 0.0), "projected_share_points": 100.0 * float(row.get("projected_share") or 0.0), "win_probability": probability, "win_probability_percent": 100.0 * probability, "contention_eligible": probability >= 0.01, }) candidate_rows.sort( key=lambda row: (row["win_probability"], row["projected_share_points"]), reverse=True, ) # The public Tug always shows at least the top two candidates when two or # more candidates exist. The 1% contention threshold applies only beyond # that minimum field. for index, row in enumerate(candidate_rows): above_threshold = float(row.get("win_probability") or 0.0) >= 0.01 minimum_two = index < 2 row["contention_eligible"] = bool(minimum_two or above_threshold) row["contention_reason"] = ( "probability_threshold" if above_threshold else ("minimum_two" if minimum_two else "below_threshold") ) top_probability = float(candidate_rows[0]["win_probability"]) if candidate_rows else 0.0 probability_leader = candidate_rows[0]["name"] if candidate_rows else None deterministic_leader = result.get("predicted_leader") deterministic_runner = None projected_sorted = sorted( candidate_rows, key=lambda row: row["projected_share_points"], reverse=True, ) if len(projected_sorted) > 1: deterministic_runner = projected_sorted[1]["name"] maturity = v6.get("maturity_info") or {} turning = v6.get("turning_point") or {} source_snapshot = fetch_json(slug_base(race_url) + snapshot_name) source_candidates = candidate_map(source_snapshot) source_votes = sum(int(c.get("votes") or 0) for c in source_candidates.values()) generated_at = datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") return { "schema_version": 1, "product": "AFI VoteFlow Live Forecast", "model": v6.get("version") or result.get("model") or "V6.2.2", "generated_at_utc": generated_at, "source": { "race_url": race_url, "snapshot": snapshot_name, "snapshot_timestamp": source_snapshot.get("last_updated"), "votes_reported": source_votes, "source_percent_reporting": source_snapshot.get("percent_reporting"), "estimated_vote_counted_pct": maturity.get("counted_percent_estimate"), "estimated_vote_counted_source": maturity.get("source"), }, "forecast": { "leader": deterministic_leader, "runner_up": deterministic_runner, "probability_leader": probability_leader, "projected_margin_points": result.get("projected_margin_points"), "race_rating": _live_probability_rating(top_probability), "status": _live_forecast_status(v6, top_probability), "turning_point": bool(turning.get("actionable")), "turning_point_candidate": turning.get("candidate") if turning.get("actionable") else None, }, "contention": { "min_candidates": 2, "enter_probability": 0.01, "exit_probability": 0.005, "exit_updates": 3, "max_candidates": 4, }, "candidates": candidate_rows, "diagnostics": { "maturity_stage": maturity.get("stage"), "maturity_source": maturity.get("source"), "reference_url": v6.get("reference_url"), "reference_library": v6.get("reference_library"), "monte_carlo_simulations": mc.get("simulations"), "model_confidence_internal": v6.get("model_confidence"), "model_confidence_public": False, }, } def main() -> None: p = argparse.ArgumentParser(description="AFI VoteFlow Needle model runner") p.add_argument("--race-url") p.add_argument("--snapshot") p.add_argument("--batch-percentages") p.add_argument("--baseline-url", help="Optional prior/reference final JSON, local normalized JSON, or race directory") p.add_argument("--market-map", help="Optional media-market JSON/JS URL or local path override; otherwise engine/media_markets/_media_markets.js is auto-discovered") p.add_argument("--output") p.add_argument( "--live-forecast", action="store_true", help="Emit a frontend-safe live forecast payload without backtest scoring fields", ) p.add_argument( "--live-mc-simulations", type=int, default=1200, help="Monte Carlo simulations for --live-forecast (minimum 1000)", ) p.add_argument("--test-report", action="store_true", help="Generate a standardized V6.2.2 backtest report") p.add_argument("--text-output", help="Optional human-readable test-report TXT output path") p.add_argument("--training-manifest", help="Run the V6.2.2 multi-race historical battery from a JSON manifest") p.add_argument( "--statewide-battery-manifest", help=( "Run every supplied statewide race at one common target percentage" ), ) p.add_argument( "--model", default="compare", choices=["v0", "v1", "v2r", "v3", "v4", "v6", "compare"] ) # Historical-reference importer modes. p.add_argument("--wiki-scan-url") p.add_argument("--wiki-import-url") p.add_argument("--wiki-table-id", type=int) p.add_argument("--wiki-state") p.add_argument("--wiki-race-url") # Official state-results file importer modes. p.add_argument("--official-scan-file") p.add_argument("--official-import-file") p.add_argument("--official-selection-id") p.add_argument("--official-state") p.add_argument("--official-race-url") p.add_argument( "--official-election-date", help=( "Optional election-date override for official reports that do not " "contain a reliable date, notably Iowa SOS detail.txt " "(example: 2022-06-07) and South Carolina fixed-width detail " "(this 2022 primary file: 2022-06-14)" ), ) p.add_argument( "--official-import-all-statewide", action="store_true", help="Import every multi-candidate statewide race detected in an official-results file", ) p.add_argument( "--official-output-dir", help="Destination directory for --official-import-all-statewide", ) args = p.parse_args() if args.live_forecast: if not args.race_url: raise SystemExit("--race-url is required for live-forecast mode") if not args.snapshot: raise SystemExit("--snapshot is required for live-forecast mode") if not args.output: raise SystemExit("--output is required for live-forecast mode") result = build_live_forecast_payload( args.race_url, args.snapshot, args.baseline_url, args.market_map, args.live_mc_simulations, ) save_json(Path(args.output), result) print(json.dumps(result, indent=2)) return if args.training_manifest: if not args.output: raise SystemExit("--output is required for training-battery mode") result = run_training_battery(args.training_manifest) save_json(Path(args.output), result) print(json.dumps(result, indent=2)) return if args.statewide_battery_manifest: if not args.output: raise SystemExit( "--output is required for statewide-battery mode" ) manifest_for_status = None try: manifest_for_status = load_json_source( args.statewide_battery_manifest ) result = run_statewide_checkpoint_battery( args.statewide_battery_manifest ) save_json(Path(args.output), result) if args.text_output: text_path = Path(args.text_output) text_path.parent.mkdir( parents=True, exist_ok=True, ) text_path.write_text( render_statewide_battery_text( result ), encoding="utf-8", ) print(json.dumps(result, indent=2)) return except Exception as exc: status_path = None job_id = None if isinstance( manifest_for_status, dict ): status_path = ( manifest_for_status.get( "status_path" ) ) job_id = ( manifest_for_status.get( "job_id" ) ) _write_statewide_battery_status( status_path, { "schema_version": 1, "job_id": job_id, "status": "failed", "error": str(exc), "failed_at_utc": ( datetime.utcnow().isoformat( timespec="seconds" ) + "Z" ), "output_path": args.output, }, ) raise if args.official_import_all_statewide: if not args.official_import_file: raise SystemExit( "--official-import-file is required for bulk statewide import mode" ) if not args.official_state: raise SystemExit( "--official-state is required for bulk statewide import mode" ) if not args.official_output_dir: raise SystemExit( "--official-output-dir is required for bulk statewide import mode" ) result = import_all_statewide_official_references( args.official_import_file, args.official_state, args.official_output_dir, args.official_race_url or args.race_url, election_date_override=( args.official_election_date ), ) if args.output: save_json(Path(args.output), result) print(json.dumps(result, indent=2)) return if args.official_scan_file: if not args.output: raise SystemExit("--output is required for official-results scan mode") result = scan_official_results_file( args.official_scan_file, args.official_state, election_date_override=( args.official_election_date ), ) save_json(Path(args.output), result) print(json.dumps(result, indent=2)) return if args.official_import_file: if not args.output: raise SystemExit("--output is required for official-results import mode") if not args.official_selection_id: raise SystemExit("--official-selection-id is required for official-results import mode") if not args.official_state: raise SystemExit("--official-state is required for official-results import mode") result = import_official_results_reference( args.official_import_file, args.official_selection_id, args.official_state, args.official_race_url or args.race_url, election_date_override=( args.official_election_date ), ) save_json(Path(args.output), result) print(json.dumps(result, indent=2)) return if args.wiki_scan_url: if not args.output: raise SystemExit("--output is required for Wikipedia scan mode") result = scan_wikipedia_article(args.wiki_scan_url, args.wiki_state) save_json(Path(args.output), result) print(json.dumps(result, indent=2)) return if args.wiki_import_url: if not args.output: raise SystemExit("--output is required for Wikipedia import mode") if args.wiki_table_id is None: raise SystemExit("--wiki-table-id is required for Wikipedia import mode") if not args.wiki_state: raise SystemExit("--wiki-state is required for Wikipedia import mode") result = import_wikipedia_reference( args.wiki_import_url, args.wiki_table_id, args.wiki_state, args.wiki_race_url or args.race_url, ) save_json(Path(args.output), result) print(json.dumps(result, indent=2)) return if args.test_report: if not args.race_url: raise SystemExit("--race-url is required for test-report mode") if not args.batch_percentages: raise SystemExit("--batch-percentages is required for test-report mode") if not args.output: raise SystemExit("--output is required for test-report mode") targets = [] for raw in args.batch_percentages.split(","): raw = raw.strip() if raw: value = float(raw) if 0 < value <= 100: targets.append(value) if not targets: raise SystemExit("No valid batch percentages supplied") result = build_test_report(args.race_url, targets, args.baseline_url, args.market_map) save_json(Path(args.output), result) if args.text_output: text_path = Path(args.text_output) text_path.parent.mkdir(parents=True, exist_ok=True) text_path.write_text(render_test_report_text(result), encoding="utf-8") print(json.dumps(result, indent=2)) return if not args.race_url: raise SystemExit("--race-url is required for model mode") if not args.output: raise SystemExit("--output is required for model mode") if args.batch_percentages: targets = [] for raw in args.batch_percentages.split(","): raw = raw.strip() if raw: value = float(raw) if 0 < value <= 100: targets.append(value) if not targets: raise SystemExit("No valid batch percentages supplied") result = batch_test_percentages( args.race_url, targets, args.baseline_url, args.market_map, ) elif args.snapshot: if args.model == "v0": result = project_v0(args.race_url, args.snapshot) elif args.model == "v1": result = project_v1(args.race_url, args.snapshot) elif args.model == "v2r": result = project_v2r(args.race_url, args.snapshot) if result is None: raise SystemExit("V2R is only available for a runoff with a Round 1 baseline.") elif args.model == "v3": result = project_v3(args.race_url, args.snapshot, args.baseline_url) elif args.model in ("v4", "v6"): result = project_v4( args.race_url, args.snapshot, args.baseline_url, args.market_map, ) else: result = { "schema_version": 31, "model": "V6_2_2_FULL_COMPARISON", "v0": project_v0(args.race_url, args.snapshot), "v1": project_v1(args.race_url, args.snapshot), "v3": project_v3(args.race_url, args.snapshot, args.baseline_url), "histgeo": project_historical_candidate_geo( args.race_url, args.snapshot, args.baseline_url ), "latent": project_latent_coalition_factors( args.race_url, args.snapshot, args.baseline_url, v3_result=project_v3(args.race_url, args.snapshot, args.baseline_url), ), "batchbias": project_batch_bias_model( args.race_url, args.snapshot, project_v3(args.race_url, args.snapshot, args.baseline_url), ), "runoff_transfer": project_runoff_transfer_expert( args.race_url, args.snapshot ) if round1_baseline_url(args.race_url) else None, "precinct": project_precinct_hierarchy_model( args.race_url, args.snapshot ), "v6": project_v4( args.race_url, args.snapshot, args.baseline_url, args.market_map, ), } if round1_baseline_url(args.race_url): try: v2r = project_v2r(args.race_url, args.snapshot) if v2r: result["v2r"] = v2r except Exception as exc: result["v2r_error"] = str(exc) else: raise SystemExit("Provide --snapshot or --batch-percentages") save_json(Path(args.output), result) print(json.dumps(result, indent=2)) if __name__ == '__main__': main()