#!/usr/bin/env python3
"""
Downloads the Part 573 Safety Recall Reports for every 12-volt battery / alternator / starter campaign
in NHTSA's recall flat file and extracts their text with `pdftotext -layout` (poppler).

Re-run:  python3 fetch573.py <dir with FLAT_RCL_POST_2010.zip> <docs dir>
Writes <docs dir>/p573_index.json (campaign -> report URLs) for part573.py.
Documents are found through https://api.nhtsa.gov/safetyIssues/byNhtsaId?filter=recalls&nhtsaId=<campaign>;
RCLRPT-* files are the Part 573 reports (RCDNN-* older defect notices are used only when no RCLRPT exists).
"""
import io, json, os, re, subprocess, sys, time, urllib.request, zipfile

SRC, DOCS = sys.argv[1], sys.argv[2]
os.makedirs(DOCS, exist_ok=True)
TARGET = re.compile(r'^ELECTRICAL SYSTEM:(12V/24V/48V BATTERY|ALTERNATOR/GENERATOR/REGULATOR|STARTER ASSEMBLY)')
camps = {}
z = zipfile.ZipFile(os.path.join(SRC, 'FLAT_RCL_POST_2010.zip'))
with z.open(z.namelist()[0]) as f:
    for line in io.TextIOWrapper(f, encoding='latin-1'):
        r = line.rstrip('\r\n').split('\t')
        if len(r) < 23 or r[10] != 'V' or not TARGET.match(r[6]): continue
        camps[r[1]] = r[15]

def urls_of(o, out):
    if isinstance(o, dict):
        for k, v in o.items():
            if k == 'url' and isinstance(v, str): out.append(v)
            else: urls_of(v, out)
    elif isinstance(o, list):
        for x in o: urls_of(x, out)
    return out

index = {}
for camp, received in sorted(camps.items()):
    try:
        d = json.load(urllib.request.urlopen(
            f"https://api.nhtsa.gov/safetyIssues/byNhtsaId?filter=recalls&nhtsaId={camp[:6]}", timeout=60))
    except Exception as e:
        index[camp] = {'err': str(e)}; continue
    urls = urls_of(d, [])
    docs = [u for u in urls if '/RCLRPT-' in u] or [u for u in urls if '/RCDNN-' in u]
    index[camp] = {'rcdate': received, 'docs': docs}
    for u in docs:
        p = os.path.join(DOCS, os.path.basename(u))
        if not os.path.exists(p):
            try: urllib.request.urlretrieve(u, p)
            except Exception as e: print('failed', u, e); continue
        if not os.path.exists(p + '.txt'):
            subprocess.run(['pdftotext', '-layout', p, p + '.txt'], capture_output=True)
    time.sleep(0.3)
json.dump(index, open(os.path.join(DOCS, 'p573_index.json'), 'w'), indent=1)
print(len(index), 'campaigns;', sum(1 for v in index.values() if v.get('docs')), 'with a report')