gauravmeena0708/epfo-circulars
0
1from bs4 import BeautifulSoup as bs, NavigableString2import requests3from urllib.parse import urljoin4import json5from datetime import datetime6import os7import argparse # For command-line arguments8
9# --- Configuration ---
10# If tesseract is not in your PATH, you might need to specify its location
11# For example, on Windows:
12# pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe'
13
14CIRCULAR_DATA_FILE = "circular-data.json"15
16HEADERS = {
17 'Host': 'www.epfindia.gov.in',
18 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:55.0) Gecko/20100101 Firefox/55.0',
19 'Accept': 'text/html, */*; q=0.01',
20 'Accept-Language': 'en-US,en;q=0.5',
21 'Accept-Encoding': 'gzip, deflate, br',
22 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
23 'X-Requested-With': 'XMLHttpRequest',
24 'Referer': 'https://www.epfindia.gov.in/site_en/Contact_office_wise.php?id=MHEM',
25 'Connection': 'keep-alive',
26 'Upgrade-Insecure-Requests': '1'
27}
28
29def generate_year_params():
30 """Dynamically generates financial year parameters up to the current date."""
31 current_date = datetime.now()
32 current_year = current_date.year
33 current_month = current_date.month
34
35 # Financial year in India starts in April
36 if current_month >= 4:
37 latest_start_year = current_year
38 else:
39 latest_start_year = current_year - 1
40
41 params = []
42 # Oldest explicitly listed year before 'Old Circulars' is 2009-2010
43 for year in range(latest_start_year, 2008, -1):
44 params.append(f"yr={year}-{year+1}")
45
46 params.append("yr=Old+Circulars")
47 return params
48
49YEAR_PARAMS = generate_year_params()
50
51# --- Utility Functions ---
52def load_json_file(filepath):
53 """Loads a JSON file if it exists, otherwise returns an empty dictionary."""
54 if os.path.exists(filepath):
55 try:
56 with open(filepath, 'r', encoding='utf-8') as f:
57 return json.load(f)
58 except json.JSONDecodeError:
59 print(f"Warning: Could not decode JSON from {filepath}. Starting fresh.")
60 return {}
61 return {}
62
63def save_json_file(data, filepath):
64 """Saves data to a JSON file."""
65 with open(filepath, 'w', encoding='utf-8') as f:
66 json.dump(data, f, ensure_ascii=False, indent=4)
67 print(f"Data saved to {filepath}")
68
69# --- Main Data Fetching Logic ---70def fetch_circular_metadata():
71 """Fetches circular metadata from EPFO website and saves to circular-data.json."""
72 parsed_circulars_data = []
73 print("Starting to fetch circular metadata...")
74
75 for q_param in YEAR_PARAMS:
76 current_page_url = f'https://www.epfindia.gov.in/site_en/get_cir_content.php?{q_param}'
77 print(f" Requesting URL: {current_page_url}")
78
79 try:
80 r = requests.get(current_page_url, headers=HEADERS, timeout=20)
81 r.raise_for_status()
82 soup = bs(r.text, 'html.parser')
83 print(f" Status Code: {r.status_code} for {q_param}")
84
85 table_rows = soup.find_all('tr')
86 data_rows = table_rows[1:] if table_rows and table_rows[0].find('th') else table_rows
87
88 for row_idx, row in enumerate(data_rows):
89 cells = row.find_all('td')
90 if len(cells) < 4:
91 # print(f" Skipping row {row_idx+1} in {q_param} due to insufficient cells ({len(cells)}).")
92 continue
93
94 serial_no = cells[0].get_text(strip=True)
95 subject_cell = cells[1]
96 title_parts = [content.strip() for content in subject_cell.contents if isinstance(content, NavigableString) and content.strip()]
97 title = " ".join(title_parts).split('Circular No.')[0].split('No.')[0].strip() # Basic title cleaning
98
99 circular_no_date_raw = ""
100 after_first_br = False
101 temp_circular_parts = []
102 for content in subject_cell.contents:
103 if content.name == 'br':
104 if not after_first_br:
105 after_first_br = True
106 continue
107 else: # Second br or end of relevant part
108 break
109 if after_first_br:
110 if isinstance(content, NavigableString):
111 text_content = content.strip()
112 if text_content:
113 temp_circular_parts.append(text_content)
114 elif content.name == 'a' and temp_circular_parts: # Link after some text
115 break
116 circular_no_date_raw = " ".join(filter(None, temp_circular_parts))
117
118
119 circular_no = ""
120 date_of_circular = ""
121 delimiter_dated = " dated "
122 delimiter_date = " date " # some entries use "date" instead of "dated"
123
124 actual_delimiter = None
125 if delimiter_dated in circular_no_date_raw.lower(): # Check lower case
126 actual_delimiter = delimiter_dated
127 elif delimiter_date in circular_no_date_raw.lower():
128 actual_delimiter = delimiter_date
129
130 if actual_delimiter:
131 # Find the actual delimiter with original casing for split
132 delimiter_pos = circular_no_date_raw.lower().find(actual_delimiter)
133 original_delimiter = circular_no_date_raw[delimiter_pos : delimiter_pos + len(actual_delimiter)]
134
135 parts = circular_no_date_raw.split(original_delimiter, 1)
136 circular_no = parts[0].strip()
137 if len(parts) > 1:
138 date_of_circular = parts[1].strip()
139 else:
140 circular_no = circular_no_date_raw.strip()
141
142 # Further clean title from circular number if any residue
143 if circular_no and title.endswith(circular_no): # simple check
144 title = title[:-len(circular_no)].strip()
145
146
147 def get_pdf_link(cell, base_url):
148 link_tag = cell.find('a')
149 if link_tag and link_tag.has_attr('href'):
150 relative_link = link_tag['href']
151 return urljoin(base_url, relative_link)
152 return None
153
154 hindi_pdf_link = get_pdf_link(cells[2], current_page_url)
155 english_pdf_link = get_pdf_link(cells[3], current_page_url)
156
157 circular_data = {
158 "serial_no": serial_no,
159 "title": title,
160 "circular_no": circular_no,
161 "date": date_of_circular,
162 "hindi_pdf_link": hindi_pdf_link,
163 "english_pdf_link": english_pdf_link
164 }
165 parsed_circulars_data.append(circular_data)
166 print(f" Successfully processed {q_param}")
167
168 except requests.exceptions.RequestException as e:
169 print(f" Error fetching {q_param}: {e}")
170 except Exception as e:
171 print(f" An error occurred during parsing for {q_param}: {e}")
172
173 if parsed_circulars_data:
174 save_json_file(parsed_circulars_data, CIRCULAR_DATA_FILE)
175 else:
176 print("No circular metadata was extracted.")
177 print("Finished fetching circular metadata.")
178
179
180# --- Main Execution ---
181if __name__ == "__main__":182 parser = argparse.ArgumentParser(description="Fetch EPFO circular metadata.")183 parser.add_argument(184 "--action",185 choices=['fetch', 'all'],186 default='fetch',187 help="'fetch' updates circular-data.json. 'all' is kept as a backwards-compatible alias."188 )189 args = parser.parse_args()190 191 if args.action in {'fetch', 'all'}:192 fetch_circular_metadata()193
194 print("\nScript finished.")
195