PACWIN2027/Form_Summary_Extraction
0
1# -*- coding: utf-8 -*-
2"""Web front end for the electoral PDF extractor.
3
4Upload a Form 20, electoral roll or summary sheet and get an Excel workbook
5back. Extraction runs on a worker thread so the browser can poll for progress
6instead of holding a request open for the length of a 250-page document.
7
8 uvicorn app:app --host 0.0.0.0 --port 8000
9"""
10
11import os
12import shutil
13import tempfile
14import threading
15import time
16import traceback
17import uuid
18
19from fastapi import BackgroundTasks, FastAPI, File, Form, HTTPException, UploadFile
20from fastapi.responses import FileResponse, HTMLResponse
21from fastapi.staticfiles import StaticFiles
22
23from extractor import pipeline
24
25BASE_DIR = os.path.dirname(os.path.abspath(__file__))
26STATIC_DIR = os.path.join(BASE_DIR, 'static')
27WORK_DIR = os.environ.get('WORK_DIR', os.path.join(tempfile.gettempdir(),
28 'electoral-extractor'))
29MAX_UPLOAD_MB = int(os.environ.get('MAX_UPLOAD_MB', '80'))
30JOB_TTL_SECONDS = int(os.environ.get('JOB_TTL_SECONDS', str(6 * 3600)))
31
32app = FastAPI(title='Electoral PDF Extractor', version='1.0.0')
33
34_jobs = {}
35_lock = threading.Lock()
36
37
38# --- job plumbing -----------------------------------------------------------
39def _new_job(filename):
40 job_id = uuid.uuid4().hex
41 with _lock:
42 _jobs[job_id] = {
43 'id': job_id,
44 'filename': filename,
45 'status': 'queued',
46 'message': 'Queued',
47 'log': [],
48 'created': time.time(),
49 }
50 return job_id
51
52
53def _update(job_id, **fields):
54 with _lock:
55 job = _jobs.get(job_id)
56 if job:
57 job.update(fields)
58
59
60def _log(job_id, message):
61 with _lock:
62 job = _jobs.get(job_id)
63 if job:
64 job['log'].append(message)
65 job['message'] = message
66
67
68def _sweep():
69 """Drop finished jobs and their files once they age out."""
70 cutoff = time.time() - JOB_TTL_SECONDS
71 with _lock:
72 stale = [k for k, v in _jobs.items() if v['created'] < cutoff]
73 for key in stale:
74 job = _jobs.pop(key)
75 shutil.rmtree(job.get('workdir', ''), ignore_errors=True)
76
77
78def _extract(job_id, pdf_path, workdir, doc_type, translate, start_page, separator):
79 _update(job_id, status='running')
80 try:
81 result = pipeline.run(
82 pdf_path,
83 doc_type=doc_type or None,
84 translate=translate,
85 start_page=start_page,
86 area_separator=separator,
87 output_dir=workdir,
88 progress=lambda m: _log(job_id, m),
89 )
90 _update(job_id,
91 status='done',
92 message='Done - %d record(s) extracted' % result['record_count'],
93 doc_type=result['doc_type'],
94 doc_type_label=result['doc_type_label'],
95 pages=result['pages'],
96 record_count=result['record_count'],
97 warnings=result['warnings'],
98 columns=result['preview_columns'],
99 preview=[[('' if c is None else c) for c in row]
100 for row in result['preview_rows']],
101 output_path=result['output_path'])
102 except Exception as exc: # surfaced to the browser
103 _update(job_id, status='error', message=str(exc) or exc.__class__.__name__,
104 detail=traceback.format_exc(limit=4))
105
106
107# --- routes -----------------------------------------------------------------
108@app.get('/', response_class=HTMLResponse)
109def index():
110 with open(os.path.join(STATIC_DIR, 'index.html'), encoding='utf-8') as fh:
111 return HTMLResponse(fh.read())
112
113
114@app.get('/api/health')
115def health():
116 from extractor import ocr
117 return {'status': 'ok', 'ocr_available': ocr.available(),
118 'ocr_languages': ocr.languages(),
119 'doc_types': [{'value': t, 'label': pipeline.DOC_TYPE_LABELS[t]}
120 for t in pipeline.DOC_TYPES]}
121
122
123@app.post('/api/extract')
124async def extract(background: BackgroundTasks,
125 file: UploadFile = File(...),
126 doc_type: str = Form(''),
127 translate: bool = Form(True),
128 start_page: int = Form(0),
129 area_separator: str = Form('semicolon')):
130 if not (file.filename or '').lower().endswith('.pdf'):
131 raise HTTPException(400, 'Please upload a PDF file.')
132 if doc_type and doc_type not in pipeline.DOC_TYPES:
133 raise HTTPException(400, 'Unknown document type %r' % doc_type)
134
135 job_id = _new_job(file.filename)
136 workdir = os.path.join(WORK_DIR, job_id)
137 os.makedirs(workdir, exist_ok=True)
138 pdf_path = os.path.join(workdir, 'input.pdf')
139
140 size = 0
141 limit = MAX_UPLOAD_MB * 1024 * 1024
142 with open(pdf_path, 'wb') as out:
143 while True:
144 chunk = await file.read(1024 * 1024)
145 if not chunk:
146 break
147 size += len(chunk)
148 if size > limit:
149 shutil.rmtree(workdir, ignore_errors=True)
150 with _lock:
151 _jobs.pop(job_id, None)
152 raise HTTPException(413, 'File is larger than %d MB.' % MAX_UPLOAD_MB)
153 out.write(chunk)
154
155 _update(job_id, workdir=workdir, size=size)
156 separator = '\n' if area_separator == 'newline' else '; '
157 background.add_task(_extract, job_id, pdf_path, workdir, doc_type,
158 translate, start_page or None, separator)
159 background.add_task(_sweep)
160 return {'job_id': job_id}
161
162
163@app.get('/api/jobs/{job_id}')
164def job_status(job_id):
165 with _lock:
166 job = _jobs.get(job_id)
167 if job is None:
168 raise HTTPException(404, 'Unknown job.')
169 return {k: v for k, v in job.items() if k not in ('workdir', 'output_path')}
170
171
172@app.get('/api/jobs/{job_id}/download')
173def download(job_id):
174 with _lock:
175 job = _jobs.get(job_id)
176 if job is None or job.get('status') != 'done':
177 raise HTTPException(404, 'No workbook is ready for this job.')
178 path = job['output_path']
179 stem = os.path.splitext(os.path.basename(job['filename']))[0]
180 return FileResponse(
181 path,
182 media_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
183 filename='%s_extracted.xlsx' % stem)
184
185
186app.mount('/static', StaticFiles(directory=STATIC_DIR), name='static')
187