#!/usr/bin/env python3
"""
PulsCar study: 12-volt battery and charging-system failures in NHTSA complaints and recalls.

Re-run:  python3 analysis.py <dir with the NHTSA zips> <output dir>
Needs only the Python 3 standard library. Input files (free, from NHTSA):
  https://static.nhtsa.gov/odi/ffdd/cmpl/COMPLAINTS_RECEIVED_2015-2019.zip
  https://static.nhtsa.gov/odi/ffdd/cmpl/COMPLAINTS_RECEIVED_2020-2024.zip
  https://static.nhtsa.gov/odi/ffdd/cmpl/COMPLAINTS_RECEIVED_2025-2026.zip
  https://static.nhtsa.gov/odi/ffdd/rcl/FLAT_RCL_POST_2010.zip
The script prints the SHA-256 of each input so a re-run can be matched to ours
(NHTSA rebuilds these files daily; later downloads will contain newer complaints).
Model years of recalled vehicles are NOT taken from these files: they come from each
campaign's Part 573 report (see part573_reports.csv, fetch573.py and part573.py).
"""
import csv, hashlib, io, os, random, re, statistics, sys, zipfile
from collections import Counter, defaultdict

SRC = sys.argv[1] if len(sys.argv) > 1 else '.'
OUT = sys.argv[2] if len(sys.argv) > 2 else 'out'
os.makedirs(OUT, exist_ok=True)
CMPL_ZIPS = ['COMPLAINTS_RECEIVED_2015-2019.zip', 'COMPLAINTS_RECEIVED_2020-2024.zip', 'COMPLAINTS_RECEIVED_2025-2026.zip']
RCL_ZIP = 'FLAT_RCL_POST_2010.zip'

def sha256(p):
    h = hashlib.sha256()
    with open(p, 'rb') as f:
        for b in iter(lambda: f.read(1 << 20), b''):
            h.update(b)
    return h.hexdigest()

# ------------------------------------------------------------------ rules (case-insensitive)
# Step 1. The text mentions a battery, an alternator, the charging system or a voltage regulator.
MENTION = re.compile(r"\bbatter(y|ies)\b|\balternators?\b|\bcharging system\b|\bvoltage regulator\b", re.I)
# Step 2. Exclusions: high-voltage / EV traction battery / key fob context unless the text also says 12 volt;
# motorcycles.
HV = re.compile(r"hybrid batter|high[- ]voltage|\bhv batter|traction batter|lithium|li-ion|charging station|supercharg|"
                r"charge port|state of charge|\brange\b|kwh|electric vehicle|\bev\b|\bphev\b|\bbev\b|plug-in|battery pack|"
                r"drive battery|propulsion|\bfob\b|remote batter|key batter", re.I)
V12 = re.compile(r"\b12[- ]?v(olts?)?\b|\b12v\b|auxiliary batter|low[- ]voltage batter|\b12[- ]volt", re.I)
MOTO = re.compile(r"motorcycle|harley|scooter|\bbike\b|goldwing|ninja", re.I)
# Step 3. The text describes the 12-volt battery or charging system failing.
FAIL = re.compile(r"""
 \b(dead|weak|drained|discharged|failed|failing|bad)\s+(12[- ]?v(olt)?\s+)?batter(y|ies)\b
|\bbatter(y|ies)\s+(was\s+|is\s+|had\s+|went\s+|goes\s+|keeps\s+|kept\s+|would\s+|has\s+)?(completely\s+|totally\s+)?(dead|died|dies|dying|drained|drains|draining|discharged|failed|fails|weak)\b
|\balternators?\s+(had\s+|has\s+)?(failed|fails|failing|went\s+(out|bad)|go(es)?\s+(out|bad)|died|quit|stopped\s+(working|charging)|was\s+(bad|faulty|defective|not\s+charging|dead|shot|out|failing)|is\s+(bad|faulty|not\s+charging|failing|dead)|needed\s+to\s+be\s+replaced|needs\s+to\s+be\s+replaced|replaced)
|\b(new|replaced?\s+the|replace\s+the|replacing\s+the|bad|faulty|defective|failed|failing|dead)\s+alternators?\b
|\b(not|wasn'?t|isn'?t|stopped|no\s+longer)\s+charging\b
|\bcharging\s+system\s+(failure|failed|fault|malfunction|warning|light|message|problem|issue)
|\b(service|check)\s+(battery\s+)?charging\s+system
|\bbattery\s+(warning\s+)?(light|lamp|indicator|icon|symbol)|\bcharging\s+(warning\s+)?(light|lamp|indicator)
|\blow\s+(battery\s+)?voltage\b|\bvoltage\s+(dropped|drops|dropping|drop|was\s+low|is\s+low)
|\bjump[- ]?start(ed|ing|s)?\b|\bjumped\s+(it|the\s+(car|battery|vehicle|truck)|my\s+(car|battery))|\bjump\s+(it|the\s+(car|battery|vehicle|truck)|my\s+(car|battery))
""", re.I | re.X)
# A match is ignored if the 45 characters before it, within the same sentence, are hypothetical
# ("I am afraid the battery might die"), or if never/not/no stands at most two words before it
# ("has never tested at low voltage").
HYPO = re.compile(r"\b(afraid|worr\w*|concern\w*|fear\w*|might|may|in case|risk\w*)\b[^.]*$", re.I)
NEG = re.compile(r"\b(never|not|no)\s+(\w+\s+){0,2}$", re.I)

def fail_matches(d):
    real = 0
    for m in FAIL.finditer(d):
        before = d[max(0, m.start() - 45):m.start()]
        if HYPO.search(before) or NEG.search(before):
            continue
        real += 1
    return real

# Where it happened. ON THE ROAD: one sentence both places the car in use on the road and says it
# died, stalled, shut off or lost power. Checked per sentence so a no-start in one sentence and
# "while driving" in another do not combine.
DRIVE = re.compile(r"""(while|when|as)\s+(i\s+was\s+|we\s+were\s+|she\s+was\s+|he\s+was\s+|they\s+were\s+|my\s+\w+\s+was\s+|the\s+contact\s+was\s+|it\s+was\s+|being\s+)?(driv(ing|en)|in\s+motion|on\s+the\s+(highway|freeway|interstate|road|expressway|parkway)|travel?ling|going\s+(about\s+|approximately\s+|around\s+)?\d+|merging|accelerating|turning|stopped\s+at|at\s+a\s+(stop\s*light|red\s+light|light|stop\s+sign|traffic\s+light))
|\b(at|going|approximately|about|around)\s+\d{2}\s?(mph|miles\s+(per|an)\s+hour)|\bon\s+the\s+(highway|freeway|interstate|expressway)\b|\bin\s+(heavy\s+)?traffic\b|\bin\s+the\s+middle\s+of\s+(the\s+)?(road|street|intersection|highway|freeway|traffic)""", re.I | re.X)
DIED = re.compile(r"\b(died|dies|stall(ed|s|ing)?|shut\s+(off|down)|shuts\s+(off|down)|turned\s+off|cut\s+(off|out)|lost\s+(all\s+)?(motive\s+|drive\s+)?(power|electrical)(?!\s+steering)|loss\s+of\s+(all\s+|motive\s+|drive\s+)?power(?!\s+steering)|loses\s+(motive\s+)?power(?!\s+steering)|went\s+dead|quit\s+running|stopped\s+running|engine\s+(stopped|quit))\b", re.I)
# NO-START: would not start / crank / turn over, dead battery, jump start.
NOSTART = re.compile(r"(would\s?n[o']?t|did\s?n[o']?t|won'?t|will\s+not|would\s+not|failed\s+to|fail\s+to|fails\s+to|could\s?n[o']?t|can\s?n[o']?t|cannot|unable\s+to|not)\s+(re)?(start|crank|turn\s+over)|\bno[- ](re)?start\b|\bno\s+crank\b|\bdead\s+batter|\bbatter(y|ies)\s+(was\s+|is\s+|went\s+)?(completely\s+|totally\s+)?dead\b|\bjump[- ]?start", re.I)

def sentences(d):
    return re.split(r'(?<=[.!?])\s+|\n+', d)

def on_road(d):
    return any(DRIVE.search(s) and DIED.search(s) for s in sentences(d))

# Warnings, inside the on-road group
BATT_WARN = re.compile(r"\bbattery\s+(warning\s+|saver\s+)?(light|lamp|indicator|icon|symbol|message|warning)|\bcharging\s+(system\s+)?(warning\s+)?(light|lamp|indicator|message|warning)|\b(service|check)\s+(battery\s+)?charging\s+system|\balternator\s+(warning\s+)?(light|lamp|message)|\blow\s+(battery\s+)?voltage\s+(warning|light|message)|\bbattery\s+saver", re.I)
NO_WARN = re.compile(r"\b(no|never|not\s+any|zero)\s+(prior\s+)?(dash(board)?\s+|check\s+engine\s+|warning\s+|battery\s+|engine\s+|error\s+)*(warning\s+)?(lights?|lamps?|codes?|indicators?|messages?|warnings?|alerts?)\b|\bwithout\s+(any\s+)?(prior\s+)?warnings?\b|\b(warning|battery|check\s+engine)\s+lights?\s+(did\s+not|didn'?t|never|failed\s+to)\s+(come|came|illuminate|turn|go|light|appear)", re.I)
ANY_WARN = re.compile(r"\b(warnings?|check engine|lights? (came|come|comes|went|turned|flash\w*|illuminat\w*)|illuminat\w*|messages?|indicators?|lamps?|codes?|chimes?|alerts?)\b", re.I)
# What else the owner says went with it (on-road group)
LOST = {
    'power_steering': re.compile(r"power\s+steering|steering\s+(went|became|got|was|locked|stiff|hard|heavy)|hard\s+to\s+steer|could\s?n[o']?t\s+steer|unable\s+to\s+steer", re.I),
    'brakes': re.compile(r"\bbrakes?\b", re.I),
    'lights_or_dash_went_out': re.compile(r"(head)?lights\s+(went|go|turned|shut)\s+(out|off|dark)|dash(board)?\s+(went|go|turned)\s+(dark|black|out|blank)|(all|everything)\s+(electrical\s+)?(went|shut)\s+(dark|out|off)|lost\s+all\s+(electrical|power|lights)|no\s+(head)?lights|hazards?\s+(lights?\s+)?(did\s?n[o']?t|would\s?n[o']?t|could\s?n[o']?t|not)\s+work", re.I),
    'christmas_tree_many_lights': re.compile(r"christmas\s+tree|all\s+(the\s+)?(warning\s+)?lights\s+(came|lit|went|turned|flash)|multiple\s+(warning\s+)?lights|every\s+(warning\s+)?light", re.I),
    'transmission_or_shifting': re.compile(r"transmission|shift(ing|ed)?\b|stuck\s+in\s+(gear|park)|limp\s+mode", re.I),
}
TIME_AFTER = re.compile(r"\b(\d+|a\s+few|few|several|one|two|three|four|five|ten|fifteen|twenty|thirty)\s+(more\s+)?(minutes?|mins?|miles?|seconds?|blocks?)\s+(later|after|down\s+the\s+road)", re.I)
VOLT = re.compile(r"(?<![\d.])(\d{1,2}\.\d{1,2})\s?(v\b|volts?\b|vdc\b)", re.I)
ALT = re.compile(r"\balternators?\b", re.I)
# powertrain from model name only (same heuristic as the published noise study; FUEL_TYPE is mostly blank)
ELECTRIC = re.compile(r"\b(EV\d?|ELECTRIC|LIGHTNING|BOLT|LEAF|MODEL [3SXY]|MACH-E|IONIQ 5|IONIQ 6|EV6|EV9|ID\.4|I3|TAYCAN|R1T|R1S|LYRIQ|HUMMER EV|SOLTERRA|BZ4X|ARIYA|NIRO EV|KONA ELECTRIC|POLESTAR|GV60|BLAZER EV|EQUINOX EV|PROLOGUE|ZDX|I4|IX|EQS|EQE|RZ)\b")
EV_MAKES = {'TESLA', 'RIVIAN', 'LUCID', 'POLESTAR', 'FISKER', 'VINFAST'}
HYBRID = re.compile(r"HYBRID|PHEV|PLUG-?IN|PRIUS|HEV\b|4XE")
COLD = {'MN','WI','MI','ND','SD','ME','VT','NH','NY','PA','OH','IL','IA','MA','MT','WY','ID','NE','CT','RI','IN','AK','CO','UT'}
HOT = {'FL','TX','AZ','NV','LA','MS','AL','GA','SC','HI','NM','OK','AR','CA'}

def powertrain(make, model):
    m = (model or '').upper()
    if make in EV_MAKES or ELECTRIC.search(m): return 'electric'
    if HYBRID.search(m): return 'hybrid_or_plug_in'
    return 'other'

def to_int(x):
    try: return int(x)
    except Exception: return 0

# ------------------------------------------------------------------ load complaints
print('SOURCE FILES (SHA-256)')
for z in CMPL_ZIPS + [RCL_ZIP]:
    p = os.path.join(SRC, z)
    print(f'  {z}  {os.path.getsize(p)} bytes  {sha256(p)}')

recs = {}
raw_rows = 0
for z in CMPL_ZIPS:
    zf = zipfile.ZipFile(os.path.join(SRC, z))
    with zf.open(zf.namelist()[0]) as f:
        for line in io.TextIOWrapper(f, encoding='latin-1'):
            r = line.rstrip('\r\n').split('\t')
            if len(r) < 46: continue
            raw_rows += 1
            if r[45].strip() != 'V': continue
            odi = r[1]
            if odi in recs:
                recs[odi]['comps'].add(r[11]); continue
            recs[odi] = dict(odi=odi, make=r[3], model=r[4], year=r[5], crash=r[6], faildate=r[7], fire=r[8],
                             injured=to_int(r[9]), deaths=to_int(r[10]), comps={r[11]}, state=r[13], ldate=r[16],
                             miles=to_int(r[17]), descr=r[19], speed=to_int(r[31]) if len(r) > 31 else 0,
                             towed=r[48] if len(r) > 48 else '', inc_state=r[49] if len(r) > 49 else '')
allc = list(recs.values())
ld = sorted(c['ldate'] for c in allc)
print(f'rows {raw_rows}, distinct vehicle complaints {len(allc)}, received {ld[0]}..{ld[-1]}')

# ------------------------------------------------------------------ classify
n_mention = n_after_excl = 0
fs = []
for c in allc:
    d = c['descr'] or ''
    if not MENTION.search(d): continue
    n_mention += 1
    if HV.search(d) and not V12.search(d): continue
    if MOTO.search(d): continue
    n_after_excl += 1
    if fail_matches(d) == 0: continue
    c['road'] = on_road(d)
    c['nostart'] = (not c['road']) and bool(NOSTART.search(d))
    c['setting'] = 'on_the_road' if c['road'] else ('no_start_or_dead_when_parked' if c['nostart'] else 'other_or_unclear')
    c['alt'] = bool(ALT.search(d))
    c['pt'] = powertrain(c['make'], c['model'])
    fs.append(c)
road = [c for c in fs if c['road']]
ns = [c for c in fs if c['nostart']]
oth = [c for c in fs if c['setting'] == 'other_or_unclear']
pct = lambda a, b: round(100.0 * a / b, 1) if b else ''
print(f'mention {n_mention}; after exclusions {n_after_excl}; failure described {len(fs)}; '
      f'on road {len(road)} ({pct(len(road), len(fs))}%); no-start {len(ns)} ({pct(len(ns), len(fs))}%); other {len(oth)}')

def w(name, header, rows):
    with open(os.path.join(OUT, name), 'w', newline='') as f:
        wr = csv.writer(f); wr.writerow(header); wr.writerows(rows)

# 1. funnel
w('summary.csv', ['step', 'complaints', 'pct_of_failure_set'], [
    ['distinct vehicle complaints received ' + ld[0] + '-' + ld[-1], len(allc), ''],
    ['mention battery / alternator / charging system / voltage regulator', n_mention, ''],
    ['after EV traction-battery, key-fob and motorcycle exclusions', n_after_excl, ''],
    ['describe a 12-volt battery or charging failure (failure set)', len(fs), 100.0],
    ['  on the road: died, stalled, shut off or lost power while in use', len(road), pct(len(road), len(fs))],
    ['  no start / dead battery when parked', len(ns), pct(len(ns), len(fs))],
    ['  other or unclear', len(oth), pct(len(oth), len(fs))],
])

# 1b. how NHTSA filed them: component codes across the whole file and inside the failure set
def has(c, prefix): return any(x.startswith(prefix) for x in c['comps'])
comp_rows = [
    ['whole file', 'ELECTRICAL SYSTEM:12V/24V/48V BATTERY (incl. cables)', sum(1 for c in allc if has(c, 'ELECTRICAL SYSTEM:12V/24V/48V BATTERY'))],
    ['whole file', 'ELECTRICAL SYSTEM:ALTERNATOR/GENERATOR/REGULATOR', sum(1 for c in allc if has(c, 'ELECTRICAL SYSTEM:ALTERNATOR'))],
    ['whole file', 'ELECTRICAL SYSTEM:STARTER ASSEMBLY', sum(1 for c in allc if has(c, 'ELECTRICAL SYSTEM:STARTER'))],
]
top = Counter()
for c in fs:
    for x in {y.split(':')[0].strip() for y in c['comps']}: top[x] += 1
for k, v in top.most_common(8):
    comp_rows.append(['failure set, top-level system (a complaint can carry several)', k, v])
comp_rows.append(['failure set', 'carries the 12V battery or alternator component', sum(1 for c in fs if has(c, 'ELECTRICAL SYSTEM:12V/24V/48V BATTERY') or has(c, 'ELECTRICAL SYSTEM:ALTERNATOR'))])
w('component_codes.csv', ['scope', 'component', 'complaints'], comp_rows)

# 2. setting by group
def setting_rows(label, key):
    g = defaultdict(list)
    for c in fs: g[key(c)].append(c)
    out = []
    for k in sorted(g, key=lambda k: -len(g[k])):
        L = g[k]; r = sum(1 for c in L if c['road']); n = sum(1 for c in L if c['nostart'])
        out.append([label, k, len(L), r, pct(r, len(L)), n, pct(n, len(L))])
    return out
def my_band(c):
    y = to_int(c['year'])
    if not 1990 <= y <= 2027: return 'unknown'
    return '2010_or_older' if y <= 2010 else '2011-2015' if y <= 2015 else '2016-2020' if y <= 2020 else '2021_or_newer'
rows = []
rows += setting_rows('alternator named in text', lambda c: 'alternator named' if c['alt'] else 'alternator not named')
rows += setting_rows('powertrain from model name', lambda c: c['pt'])
rows += setting_rows('model year', my_band)
rows += setting_rows('year received', lambda c: c['ldate'][:4])
w('setting_by_group.csv', ['grouping', 'group', 'failure_complaints', 'on_the_road', 'pct_on_the_road', 'no_start', 'pct_no_start'], rows)

# 3. month of incident, normalised by all complaints with an incident in the same month
def month(c):
    f = c['faildate'] or ''
    return f[4:6] if len(f) == 8 and '2014' <= f[:4] <= '2026' and '01' <= f[4:6] <= '12' else None
def month_table(subset, states=None):
    a, b = Counter(), Counter()
    for c in allc:
        if states and c['state'] not in states: continue
        m = month(c)
        if m: a[m] += 1
    for c in subset:
        if states and c['state'] not in states: continue
        m = month(c)
        if m: b[m] += 1
    return a, b
rows = []
for label, sub, st in [('all failure complaints', fs, None), ('on the road', road, None), ('no start', ns, None),
                       ('all failure complaints, cold states', fs, COLD), ('all failure complaints, hot states', fs, HOT)]:
    a, b = month_table(sub, st)
    for m in sorted(a):
        rows.append([label, m, b[m], a[m], round(100.0 * b[m] / a[m], 3)])
w('by_incident_month.csv', ['subset', 'incident_month', 'complaints_in_subset', 'all_complaints_same_month', 'per_100_complaints'], rows)

# 4. on-road outcomes and warnings
def share(L, f): k = sum(1 for c in L if f(c)); return k, pct(k, len(L))
rows = []
for label, L in [('on the road', road), ('no start', ns), ('whole failure set', fs), ('all complaints', allc)]:
    sp = [c['speed'] for c in L if 1 <= c['speed'] <= 149]
    rows.append([label, len(L), *share(L, lambda c: c['towed'] == 'Y'), *share(L, lambda c: c['crash'] == 'Y'),
                 *share(L, lambda c: c['fire'] == 'Y'), *share(L, lambda c: c['injured'] > 0),
                 len(sp), statistics.median(sp) if sp else ''])
w('outcomes_by_setting.csv', ['setting', 'complaints', 'towed', 'pct_towed', 'crash', 'pct_crash', 'fire', 'pct_fire',
                              'injury_reported', 'pct_injury', 'speed_given_1_149', 'median_speed_mph'], rows)
bw = [c for c in road if BATT_WARN.search(c['descr'])]
nw = [c for c in road if NO_WARN.search(c['descr'])]
aw = [c for c in road if ANY_WARN.search(c['descr'])]
ta = [c for c in road if TIME_AFTER.search(c['descr']) and BATT_WARN.search(c['descr'])]
rows = [['names a battery or charging warning (light, message, battery saver)', len(bw), pct(len(bw), len(road))],
        ['says outright there was no warning', len(nw), pct(len(nw), len(road))],
        ['mentions any warning, light, message or code at all', len(aw), pct(len(aw), len(road))],
        ['names a battery or charging warning AND gives a time or distance after something', len(ta), pct(len(ta), len(road))]]
for k, rx in LOST.items():
    n = sum(1 for c in road if rx.search(c['descr'])); rows.append([f'mentions {k}', n, pct(n, len(road))])
w('on_the_road_details.csv', ['on-the-road complaints that...', 'complaints', 'pct_of_on_the_road'], rows)

# 5. voltages owners quote
vals = []
for c in fs:
    for m in VOLT.finditer(c['descr']):
        v = float(m.group(1))
        if 5.0 <= v <= 16.5: vals.append((c['odi'], v))
bands = [('below 11.8', lambda v: v < 11.8), ('11.8 to 11.99', lambda v: 11.8 <= v < 12.0), ('12.0 to 12.39', lambda v: 12.0 <= v < 12.4),
         ('12.4 to 12.99', lambda v: 12.4 <= v < 13.0), ('13.0 to 13.19', lambda v: 13.0 <= v < 13.2),
         ('13.2 to 15.0', lambda v: 13.2 <= v <= 15.0), ('above 15.0', lambda v: v > 15.0)]
w('owner_quoted_voltages.csv', ['band_volts', 'readings', 'complaints_quoting_one_or_more'],
  [[b, sum(1 for _, v in vals if f(v)), len({o for o, v in vals if f(v)})] for b, f in bands] +
  [['all readings 5.0-16.5 V', len(vals), len({o for o, _ in vals})]])

# 6. by make (makes with >= 150 failure complaints); counts follow sales and filing habits, not reliability
g = defaultdict(list)
for c in fs: g[c['make']].append(c)
w('by_make.csv', ['make', 'failure_complaints', 'on_the_road', 'pct_on_the_road', 'alternator_named', 'pct_alternator_named'],
  [[m, len(L), sum(c['road'] for c in L), pct(sum(c['road'] for c in L), len(L)), sum(c['alt'] for c in L), pct(sum(c['alt'] for c in L), len(L))]
   for m, L in sorted(g.items(), key=lambda x: -len(x[1])) if len(L) >= 150])

# 7. vehicle age and mileage
rows = []
for label, L in [('on the road', road), ('no start', ns), ('alternator named', [c for c in fs if c['alt']])]:
    ages = [to_int(c['ldate'][:4]) - to_int(c['year']) for c in L if 1990 <= to_int(c['year']) <= 2027]
    ages = [a for a in ages if 0 <= a <= 40]
    mi = [c['miles'] for c in L if 101 <= c['miles'] <= 399999]
    rows.append([label, len(L), len(ages), statistics.median(ages) if ages else '', len(mi), statistics.median(mi) if mi else ''])
w('age_and_mileage.csv', ['setting', 'complaints', 'with_model_year', 'median_age_years', 'with_mileage', 'median_miles'], rows)

# ------------------------------------------------------------------ recalls
TARGET = re.compile(r'^ELECTRICAL SYSTEM:(12V/24V/48V BATTERY|ALTERNATOR/GENERATOR/REGULATOR|STARTER ASSEMBLY)')
camps = {}
zf = zipfile.ZipFile(os.path.join(SRC, RCL_ZIP))
with zf.open(zf.namelist()[0]) as f:
    for line in io.TextIOWrapper(f, encoding='latin-1'):
        r = line.rstrip('\r\n').split('\t')
        if len(r) < 23 or r[10] != 'V' or not TARGET.match(r[6]): continue
        c = camps.setdefault(r[1], dict(comp=set(), mfr=r[7], rcdate=r[15], potaff=to_int(r[11]), cons=r[20], defect=r[19]))
        c['comp'].add(r[6].split(':')[1].strip())
FIRE_RX = re.compile(r"\bfire\b", re.I)
STALL_RX = re.compile(r"stall|loss of (motive|drive|propulsion) power|los(e|es|s|ing) (of )?(motive |drive )?power|shut(s)? ?(down|off)|power loss|loss of (all )?electrical|engine (to )?(stop|shut)|vehicle (to )?(shut|stop)", re.I)
NOSTART_RX = re.compile(r"not (re)?start|no[- ]start|fail(s|ure)? to (re)?start|unable to (re)?start|inability to (re)?start|cannot be (re-?)?started|prevent the (vehicle|engine) from (re)?starting", re.I)
rows = []
cls = Counter(); pop = Counter()
for k, c in sorted(camps.items()):
    fire = bool(FIRE_RX.search(c['cons'])); stall = bool(STALL_RX.search(c['cons'])); nost = bool(NOSTART_RX.search(c['cons']))
    label = 'fire' if fire and not stall else 'stall_or_loss_of_power' if stall and not fire else 'fire_and_stall' if fire and stall else 'no_start' if nost else 'other'
    cls[label] += 1; pop[label] += c['potaff']
    rows.append([k, ' + '.join(sorted(c['comp'])), c['mfr'], c['rcdate'], c['potaff'], label, c['cons'].strip()])
w('recall_campaigns.csv', ['nhtsa_campaign', 'component_group', 'manufacturer_filing', 'part573_received', 'potentially_affected',
                           'consequence_class', 'consequence_text_as_filed'], rows)
w('recalls_by_consequence.csv', ['consequence_class', 'campaigns', 'vehicles_counted_once_per_campaign'],
  [[k, cls[k], pop[k]] for k in ['fire', 'fire_and_stall', 'stall_or_loss_of_power', 'no_start', 'other']] +
  [['total', sum(cls.values()), sum(pop.values())]])
yc = Counter(c['rcdate'][:4] for c in camps.values()); yp = Counter()
for c in camps.values(): yp[c['rcdate'][:4]] += c['potaff']
w('recalls_by_year.csv', ['part573_received_year', 'campaigns', 'vehicles_counted_once_per_campaign'], [[y, yc[y], yp[y]] for y in sorted(yc)])
print('recall campaigns', len(camps), dict(cls), 'population', sum(pop.values()))

# ------------------------------------------------------------------ audit samples (fixed seed, for hand reading)
random.seed(20260923)
with open(os.path.join(OUT, 'audit_samples.txt'), 'w') as f:
    for label, L in [('FAILURE SET', fs), ('ON THE ROAD', road), ('NOT ON THE ROAD', [c for c in fs if not c['road']])]:
        f.write(f'===== {label}: 40 random complaints =====\n')
        for c in random.sample(L, 40):
            f.write(f"--- ODI {c['odi']} {c['make']} {c['model']} {c['year']} setting={c['setting']}\n{c['descr'][:900]}\n")
print('done')