society-ethics/annotated-encyclical

1#!/usr/bin/env python32"""Fetch the original Vatican encyclical pages and extract each footnote's HTML3(text + hyperlinks), which the plain-text `encyclical.<lang>.txt` sources drop.4 5Writes footnotes.json: { "<lang>": { "<n>": "<sanitized html>", ... }, ... }6build.py reads this to render footnotes with their original links restored.7 8Re-run after the source text changes; it is network-bound (vatican.va)."""9import json, re, sys, urllib.request10 11LANGS = ['en', 'it', 'es', 'fr', 'de', 'pt', 'pl', 'ar']12URL = 'https://www.vatican.va/content/leo-xiv/{}/encyclicals/documents/20260515-magnifica-humanitas.html'13 14# footnote definition anchor: <a name="_ftn12" href="#_ftnref12" ...>...</a>15# The Vatican HTML is buggy: anchors may be empty, the visible [N] label may sit16# outside the anchor, and `name` and the label sometimes disagree (duplicates).17# So we read BOTH the name number and the visible label, then reconcile (below).18ANCHOR = re.compile(r'<a\s+name="_ftn(\d+)"\s+href="#_ftnref\d+"[^>]*>(.*?)</a>', re.I)19LABEL = re.compile(r'\[?\s*(\d+)\s*\]')                 # a [N] / N] label20LEAD_LABEL = re.compile(r'^\s*(?:\[\s*\d+\s*\]|\d+\s*\])\s*')   # strip a stray leading label21A_OPEN = re.compile(r'<a\b[^>]*?href="([^"]+)"[^>]*>', re.I)22KEEP_TAG = re.compile(r'</?(?:i|a)\b[^>]*>', re.I)      # tags we keep (a/i)23ANY_TAG = re.compile(r'<[^>]+>')24EMPTY_I = re.compile(r'<i>\s*</i>', re.I)25ELEM_END = re.compile(r'</(?:p|span|div|li)>', re.I)26 27 28def fetch(lang):29    req = urllib.request.Request(URL.format(lang), headers={'User-Agent': 'Mozilla/5.0'})30    return urllib.request.urlopen(req, timeout=60).read().decode('utf-8', 'replace')31 32 33def sanitize(raw):34    """Keep only <i> and <a href> tags; drop a stray leading [N]; tidy whitespace."""35    raw = raw.replace('\n', ' ').replace('\t', ' ')36 37    def fix_a(m):38        href = m.group(1)39        if href.startswith('/'):40            href = 'https://www.vatican.va' + href41        return f'<a href="{href}" target="_blank" rel="noopener">'42    raw = A_OPEN.sub(fix_a, raw)43    # drop every tag that isn't a kept <i>/<a>44    raw = ANY_TAG.sub(lambda m: m.group(0) if KEEP_TAG.fullmatch(m.group(0)) else '', raw)45    raw = EMPTY_I.sub('', raw)46    raw = raw.replace('\xa0', ' ').replace('&nbsp;', ' ')47    raw = LEAD_LABEL.sub('', raw)              # remove a label that leaked outside the anchor48    raw = re.sub(r'\s+', ' ', raw).strip()49    raw = re.sub(r'\s+([,.;:])', r'\1', raw)   # no space before punctuation50    return raw.strip()51 52 53def extract(html):54    # collect each definition with its name-number, visible-label-number and content55    raw = []56    for m in ANCHOR.finditer(html):57        name_n = int(m.group(1))58        chunk = html[m.end():]59        cut = ELEM_END.search(chunk)60        if cut:61            chunk = chunk[:cut.start()]62        # the visible [N] label is either inside the anchor or leaks just after it63        lbl = LABEL.search(m.group(2)) or LABEL.match(chunk.replace('\xa0', ' ').lstrip())64        label_n = int(lbl.group(1)) if lbl else None65        raw.append({'name': name_n, 'label': label_n, 'html': sanitize(chunk)})66 67    # Two-pass reconciliation: lock the definitions where name==label, then assign68    # each conflicting/missing-label one to whichever number is still free.69    fns, taken = {}, set()70    pend = []71    for d in raw:72        if d['label'] is not None and d['name'] == d['label'] and d['name'] not in taken:73            fns[d['name']] = d['html']; taken.add(d['name'])74        else:75            pend.append(d)76    for d in pend:77        for cand in (d['label'], d['name']):           # prefer the visible label, then name78            if cand is not None and cand not in taken:79                fns[cand] = d['html']; taken.add(cand); break80    return fns81 82 83def main():84    out = {}85    for lang in LANGS:86        html = fetch(lang)87        fns = extract(html)88        nums = sorted(fns)89        missing = [i for i in range(1, (max(nums) if nums else 0) + 1) if i not in fns]90        linked = sum(1 for v in fns.values() if '<a ' in v)91        print(f"{lang}: {len(fns)} footnotes, {linked} with links, missing={missing}")92        out[lang] = {str(k): fns[k] for k in nums}93    json.dump(out, open('footnotes.json', 'w', encoding='utf-8'), ensure_ascii=False, indent=0)94    print(f"Wrote footnotes.json ({len(out)} languages)")95 96 97if __name__ == '__main__':98    sys.exit(main())99