#!/usr/bin/env python3
"""
Reads NHTSA Part 573 Safety Recall Reports (text extracted with `pdftotext -layout`) for the
12-volt battery / alternator / starter campaigns and writes, per campaign, the vehicles and model
years exactly as the manufacturer listed them, the population, the stated safety risk and the
answer to the form's field "Identification of Any Warning that can Occur".

Re-run:  python3 part573.py <dir with RCLRPT-*.txt> <index json from the fetch step> <output dir>
Documents: https://static.nhtsa.gov/odi/rcl/<year>/RCLRPT-<campaign>-<id>.PDF, found through
https://api.nhtsa.gov/safetyIssues/byNhtsaId?filter=recalls&nhtsaId=<campaign>.
Where a campaign has several reports (amendments), the one with the latest Submission Date is used.
Model years here are the manufacturer's own "Vehicle N :" lines, never the recalls-by-vehicle API,
which attributes a campaign's full year span to every model in it.
"""
import csv, json, os, re, sys
from datetime import datetime

DOCS, INDEX, OUT = sys.argv[1], sys.argv[2], sys.argv[3]
idx = json.load(open(INDEX))

def parse(txt):
    t = txt.replace('\f', '\n')
    sub = re.search(r"Submission Date\s*:\s*([A-Z]{3} \d{1,2}, \d{4})", t, re.I)
    pop = re.search(r"Total number of potentially involved\s*:\s*([\d,]+)", t, re.I) or re.search(r"Number of potentially involved\s*:\s*([\d,]+)", t)
    veh = []
    for m in re.finditer(r"^\s*Vehicle\s+\d+\s*:\s*(.+?)\s*$", t, re.M):
        v = re.sub(r"\s+", " ", m.group(1)).strip()
        if v and v not in veh: veh.append(v)
    def block(start, stops):
        m = re.search(start, t, re.I | re.S)
        if not m: return ''
        rest = t[m.end():]
        end = min([x.start() for x in (re.search(s, rest, re.I) for s in stops) if x] or [len(rest)])
        s = rest[:end]
        s = re.sub(r"The information contained in this report was submitted pursuant to 49 CFR §\s*573", " ", s)
        s = re.sub(r"Part 573 Safety Recall Report\s+\S+(\s+Page \d+)?", " ", s)
        s = re.sub(r"Page \d+ of \d+", " ", s)
        s = re.sub(r"^\s*that can Occur\s*:", " ", s, flags=re.I | re.M)
        return re.sub(r"\s+", " ", s).strip(' :')
    warn = block(r"Identification of Any Warning", [r"Involved Components", r"Supplier Identification", r"Chronology", r"Component Manufacturer"])
    warn = re.sub(r"^\s*that can Occur\s*:\s*", "", warn, flags=re.I).strip()
    risk = block(r"Description of the Safety Risk[^:]*:", [r"Description of the Cause", r"Identification of Any Warning"])
    return dict(sub=sub.group(1) if sub else '', pop=pop.group(1).replace(',', '') if pop else '', veh=veh, warn=warn, risk=risk)


# Warning field, classified by rule (first match wins):
#  not_reported  - blank, "NR", "None provided", "No information provided"
#  no_warning    - None / there is no warning / will not receive a warning / without (any) warning
#  dashboard     - names a light, lamp, telltale, indicator, message, MIL, chime, gauge or display
#  other_signs   - anything else: smoke, smell, noise, hard starting, flicker, visible damage on inspection
def warn_class(w):
    w = re.sub(r"\bPage \d+\b", "", w or "").strip(" .")
    if not w or re.fullmatch(r"(NR|N/?A|none provided|no information provided.*)", w, re.I): return 'not_reported'
    if re.match(r"(none\b|no warning|there (is|are|will be) no\b|the (customer|driver) (will|would) not receive|the customer will receive no|no known warning)", w, re.I) \
       or re.search(r"without (any )?(prior )?warning|no warning (that )?(will|prior|associated)|there is no (known )?warning", w, re.I):
        if not re.search(r"light|lamp|telltale|message|indicator|MIL\b|chime|gauge|display|smoke|smell|odor|noise|visual|notice", w, re.I):
            return 'no_warning'
    if re.search(r"light|lamp|telltale|indicator|message|\bMIL\b|chime|gauge|display|warning (symbol|icon)|voltage meter|Service Battery|Battery Saver", w, re.I):
        if not re.search(r"no vehicle diagnostic warnings", w, re.I): return 'dashboard'
    return 'other_signs'

rows = []
for camp, info in sorted(idx.items()):
    best = None
    for u in info.get('docs', []):
        p = os.path.join(DOCS, os.path.basename(u) + '.txt')
        if not os.path.exists(p) or 'RCLRPT' not in p: continue
        d = parse(open(p, errors='replace').read())
        try: when = datetime.strptime(d['sub'].title(), '%b %d, %Y')
        except Exception: when = datetime.min
        d['doc'] = u; d['when'] = when
        if best is None or when > best['when']: best = d
    if best is None:
        rows.append([camp, '', '', '', '', '', '', 'no RCLRPT text report found'])
        continue
    rows.append([camp, best['sub'], best['pop'], ' | '.join(best['veh']), best['risk'][:500], best['warn'][:500], warn_class(best['warn']), best['doc']])

with open(os.path.join(OUT, 'part573_reports.csv'), 'w', newline='') as f:
    w = csv.writer(f)
    w.writerow(['nhtsa_campaign', 'report_submission_date', 'population_in_report', 'vehicles_as_listed_in_report',
                'safety_risk_as_filed', 'warning_that_can_occur_as_filed', 'warning_class', 'document'])
    w.writerows(rows)
print(len(rows), 'campaigns;', sum(1 for r in rows if r[1]), 'with a parsed text report')
from collections import Counter
print(Counter(r[6] for r in rows if r[1]))