PACWIN2027/Form_Summary_Extraction
0
1# -*- coding: utf-8 -*-
2"""Summary-sheet parser.
3
4From page 4 onward a Punjab summary sheet is one long ruled table:
5
6 | 1 part no | 2 polling station | 3 sections in the part | 4 aux | 5 M | 6 F | 7 TG | 8 total |
7
8A booth occupies one ruled row but may hold several *text* lines inside it - one
9per section, each with its own voter counts, followed by a booth total line.
10Sections also wrap mid-name, and a booth may straddle a page break.
11
12This module folds all of that back into exactly one record per booth, with every
13section listed and numbered inside a single ``Area_Included`` cell.
14"""
15
16import re
17
18SECTION_START = re.compile(r'^\s*(\d{1,3})\s*[:.\)]\s*(.*)$')
19INT = re.compile(r'^\d[\d,]*$')
20
21#: Column indices in the 8-column body table.
22C_PART, C_STATION, C_SECTIONS, C_AUX, C_MALE, C_FEMALE, C_THIRD, C_TOTAL = range(8)
23
24DEFAULT_START_PAGE = 4
25
26
27class Booth(object):
28 def __init__(self, part_no):
29 self.part_no = part_no
30 self.station_lines = []
31 self.section_lines = []
32 self.aux_lines = []
33 self.counts = ['', '', '', ''] # male, female, third gender, total
34 self.pages = []
35
36 # -- assembly ------------------------------------------------------------
37 def sections(self):
38 """Wrapped section text folded into one entry per numbered section."""
39 out = []
40 for line in self.section_lines:
41 m = SECTION_START.match(line)
42 if m:
43 out.append(m.group(2).strip())
44 elif out:
45 out[-1] = (out[-1] + ' ' + line).strip()
46 elif line:
47 out.append(line)
48 return [_tidy(s) for s in out if _tidy(s)]
49
50 def station(self):
51 # A booth split across a page break reprints its station name on the
52 # continuation page, so identical lines are collapsed.
53 seen, lines = set(), []
54 for line in self.station_lines:
55 if line not in seen:
56 seen.add(line)
57 lines.append(line)
58 return _tidy(' '.join(lines))
59
60 def aux(self):
61 seen = []
62 for a in self.aux_lines:
63 a = _tidy(a)
64 if a and a not in seen:
65 seen.append(a)
66 return '; '.join(seen)
67
68
69def _tidy(text):
70 text = re.sub(r'\s+', ' ', text or '').strip()
71 text = re.sub(r'\s+([,.;:])', r'\1', text)
72 return text.strip(' ,;')
73
74
75def parse(pages, start_page=DEFAULT_START_PAGE):
76 """Parse ``pages`` (list of :class:`~extractor.pdfsource.Page`) into booths.
77
78 Returns ``(booths, info)`` where ``info`` carries the sheet's own grand
79 total (useful as a checksum) and any page that looked like a booth table but
80 could not be read.
81 """
82 booths = []
83 by_part = {}
84 skipped = []
85 grand_total = None
86
87 for page in pages:
88 if page.number < start_page:
89 continue
90 table = _body_table(page)
91 if table is None:
92 if page.has_text_layer and 'ਸੈਕਸ਼ਨ' in page.text():
93 skipped.append(page.number)
94 continue
95
96 first_body_row = True
97 for row in table:
98 if _is_header(row):
99 continue
100 part = row.col(C_PART).strip()
101 if part and not INT.match(part):
102 continue
103
104 if not part and not row.col(C_STATION) and not row.col(C_SECTIONS):
105 # Counts with no booth attached. At the top of a page it is the
106 # total of a booth whose cell was cut by the page break; anywhere
107 # else it is the sheet's grand total, printed once at the end.
108 counts = _row_counts(row)
109 if first_body_row and booths:
110 booths[-1].counts = counts or booths[-1].counts
111 else:
112 grand_total = counts or grand_total
113 first_body_row = False
114 continue
115 first_body_row = False
116
117 booth = None
118 if part:
119 booth = by_part.get(part)
120 if booth is None:
121 booth = Booth(part)
122 by_part[part] = booth
123 booths.append(booth)
124 elif booths:
125 booth = booths[-1] # continuation row with no part number
126 if booth is None:
127 continue
128
129 booth.station_lines.extend(row.lines(C_STATION))
130 booth.section_lines.extend(row.lines(C_SECTIONS))
131 booth.aux_lines.extend(row.lines(C_AUX))
132 if page.number not in booth.pages:
133 booth.pages.append(page.number)
134
135 counts = _row_counts(row)
136 if counts:
137 booth.counts = counts
138
139 return booths, {'skipped_pages': skipped, 'grand_total': grand_total}
140
141
142#: Header row of the delivered sheet, matching the reference workbook.
143COLUMNS = ['SR NO.', 'POLLING STATION NAME', 'AREA INCLUDED',
144 'MALE', 'FEMALE', 'THIRD GENDER', 'TOTAL']
145
146#: Extra provenance columns kept on a second sheet.
147SOURCE_COLUMNS = ['SR NO.', 'PART NO.', 'POLLING STATION (PUNJABI)',
148 'AREA INCLUDED (PUNJABI)', 'AUXILIARY', 'PDF PAGES']
149
150
151def number_areas(areas, separator='; '):
152 """Render every section of one booth into a single numbered cell."""
153 return separator.join('%d. %s' % (i, a) for i, a in enumerate(areas, 1))
154
155
156def to_rows(booths, translator=None, separator='; '):
157 """Build the delivered rows and the Punjabi provenance rows.
158
159 ``translator`` may be ``None``, in which case the Punjabi text is emitted
160 as-is instead of being blanked.
161 """
162 if translator is not None:
163 # Sections first: "ਪਿੰਡ <name>" gives Google enough context to
164 # transliterate the village rather than translate it. The resulting
165 # name pairs are then protected while the station names - which have no
166 # such marker - are translated.
167 sections = [s for b in booths for s in b.sections()]
168 translator.translate_all(sections)
169 translator.protected.update(_village_glossary(sections, translator))
170 translator.translate_all([b.station() for b in booths])
171
172 def en(text):
173 return translator.get(text) if translator is not None else text
174
175 rows, source_rows = [], []
176 for i, b in enumerate(booths, 1):
177 areas = [en(a) for a in b.sections()]
178 male, female, third, total = b.counts
179 rows.append([
180 i,
181 en(b.station()),
182 number_areas(areas, separator),
183 _int(male), _int(female), _int(third), _int(total),
184 ])
185 source_rows.append([
186 i, _int(b.part_no), b.station(),
187 number_areas(b.sections(), separator), b.aux(),
188 ', '.join(str(p) for p in b.pages),
189 ])
190 return rows, source_rows
191
192
193PUNJABI_VILLAGE = re.compile('ਪਿੰਡ\\s+([^,]{2,40})')
194ENGLISH_VILLAGE = re.compile(r'Village\s+([^,]{2,40})', re.I)
195
196#: Village names that are also ordinary Punjabi words are the ones Google gets
197#: wrong, but very short names produce false substitutions, so skip them.
198MIN_GLOSSARY_LEN = 3
199
200
201def _village_glossary(sections, translator):
202 """Learn ``{punjabi village: english village}`` from translated sections."""
203 glossary = {}
204 for punjabi in sections:
205 src = PUNJABI_VILLAGE.search(punjabi)
206 dst = ENGLISH_VILLAGE.search(translator.get(punjabi))
207 if not src or not dst:
208 continue
209 name = src.group(1).strip()
210 english = dst.group(1).strip()
211 if len(name) >= MIN_GLOSSARY_LEN and english and name not in glossary:
212 glossary[name] = english
213 return glossary
214
215
216def validate(booths, info):
217 """Human-readable warnings about anything that did not reconcile."""
218 warnings = []
219 for b in booths:
220 m, f, t, total = [_int(x) or 0 for x in b.counts]
221 if m + f + t != total:
222 warnings.append('Part %s: %d + %d + %d does not equal the printed total %d'
223 % (b.part_no, m, f, t, total))
224 if not b.sections():
225 warnings.append('Part %s: no sections/areas found' % b.part_no)
226
227 grand = info.get('grand_total')
228 if grand:
229 summed = [sum(_int(b.counts[i]) or 0 for b in booths) for i in range(4)]
230 printed = [_int(x) or 0 for x in grand]
231 if summed != printed:
232 warnings.append(
233 'Booth rows add up to %s but the sheet prints a grand total of %s '
234 '(difference %s) - check the source document.'
235 % (summed, printed, [p - s for p, s in zip(printed, summed)]))
236
237 for page in info.get('skipped_pages', []):
238 warnings.append('Page %d looked like a booth table but could not be read' % page)
239 return warnings
240
241
242def _int(value):
243 try:
244 return int(str(value).replace(',', '').strip())
245 except (TypeError, ValueError):
246 return '' if value in (None, '') else value
247
248
249def _body_table(page):
250 """The 8-column booth table on this page, if there is one."""
251 for table in page.tables():
252 if table and max(len(r.cols) for r in table) >= 8:
253 return table
254 return None
255
256
257def _is_header(row):
258 part = row.col(C_PART).strip()
259 if part == '1' and row.col(C_STATION).strip() == '2':
260 return True # the '1 2 3 4 5 6 7 8' ruler row
261 joined = ' '.join(row.col(i) for i in range(min(len(row.cols), 8)))
262 return any(k in joined for k in ('ਪੋਲਿੰਗ ਸਟੇਸ਼ਨ ਭਵਨ', 'ਸੈਕਸ਼ਨਾਂ ਦੇ ਵੇਰਵੇ',
263 'ਵੋਟਰਾਂ ਦੀ ਗਿਣਤੀ', 'ਪੁਰਸ਼'))
264
265
266def _row_counts(row):
267 """Booth-level Male/Female/Third/Total for this row.
268
269 When a booth lists several sections the counts column holds one line per
270 section followed by the booth total, so the last line is the booth figure.
271 Sections with no electors are printed without any number at all, which is
272 why the count lines are read positionally from the bottom rather than
273 zipped against the section list.
274 """
275 cols = []
276 for idx in (C_MALE, C_FEMALE, C_THIRD, C_TOTAL):
277 nums = [l.replace(',', '') for l in row.lines(idx) if INT.match(l.strip())]
278 cols.append(nums)
279 if not any(cols):
280 return None
281 return [c[-1] if c else '' for c in cols]
282