acl-crown-analysis/acl-crown-analysis-code
0
1import os
2import csv
3import yaml
4import time
5import requests
6import concurrent.futures
7from pathlib import Path
8
9API_KEY = "YOUR_API_KEY_HERE"
10
11BASE_DIR = Path(__file__).resolve().parent.parent
12DATA_DIR = BASE_DIR / "data"
13DATA_FALSE_DIR = BASE_DIR / "data-false"
14CONFIG_FILE = BASE_DIR / "config" / "venues_top.yaml"
15
16HEADERS = {"x-api-key": API_KEY}
17
18def request_with_retry(url, params=None, max_retries=5):
19 for i in range(max_retries):
20 try:
21 response = requests.get(url, params=params, headers=HEADERS, timeout=15)
22 if response.status_code == 200:
23 return response
24 elif response.status_code == 429:
25 wait_time = 5 * (i + 1)
26 print(f"Rate limit exceeded (429). Waiting {wait_time}s... (Attempt {i+1}/{max_retries})")
27 time.sleep(wait_time)
28 continue
29 elif 500 <= response.status_code < 600:
30 print(f"Server error ({response.status_code}). Retrying... (Attempt {i+1}/{max_retries})")
31 time.sleep(2)
32 continue
33 else:
34 return response
35 except requests.exceptions.RequestException as e:
36 print(f"Network error: {e}. Retrying... (Attempt {i+1}/{max_retries})")
37 time.sleep(2)
38
39 print(f"Failed to fetch {url} after {max_retries} attempts.")
40 return None
41
42def load_config():
43 if not CONFIG_FILE.exists():
44 print(f"Config file not found: {CONFIG_FILE}")
45 return set(), set()
46
47 with open(CONFIG_FILE, 'r', encoding='utf-8') as f:
48 config = yaml.safe_load(f)
49
50 top_confs = set(conf.lower() for conf in config.get('top_conferences', []))
51 top_journals = set(jour.lower() for jour in config.get('top_journals', []))
52
53 return top_confs, top_journals
54
55def search_paper(title):
56 url = "https://api.semanticscholar.org/graph/v1/paper/search"
57 params = {
58 "query": title,
59 "limit": 1,
60 "fields": "paperId,title,year"
61 }
62
63 response = request_with_retry(url, params)
64
65 if response and response.status_code == 200:
66 data = response.json()
67 if data.get('data'):
68 return data['data'][0]
69 elif response:
70 print(f"Error searching paper: {response.status_code} - {response.text}")
71
72 return None
73
74def get_citations(paper_id):
75 citations = []
76 offset = 0
77 limit = 1000
78 total = 0
79
80 detail_url = f"https://api.semanticscholar.org/graph/v1/paper/{paper_id}"
81 r = request_with_retry(detail_url, params={"fields": "citationCount"})
82
83 if r and r.status_code == 200:
84 total = r.json().get('citationCount', 0)
85 else:
86 return [], 0
87
88 if total == 0:
89 return [], 0
90
91 print(f"Fetching {total} citations for paper {paper_id}...")
92
93 while True:
94 url = f"https://api.semanticscholar.org/graph/v1/paper/{paper_id}/citations"
95 params = {
96 "fields": "year,venue",
97 "offset": offset,
98 "limit": limit
99 }
100
101 response = request_with_retry(url, params)
102
103 if response and response.status_code == 200:
104 data = response.json()
105 batch = data.get('data', [])
106 if not batch:
107 break
108
109 citations.extend(batch)
110 offset += len(batch)
111
112 if offset >= total or len(batch) < limit:
113 break
114
115 time.sleep(1.0)
116 else:
117 print(f"Error fetching citations page.")
118 break
119
120 return citations, total
121
122def is_top_venue(venue_name, top_list):
123 if not venue_name:
124 return False
125 v = venue_name.lower()
126 for top in top_list:
127 if top in v:
128 return True
129 return False
130
131def process_single_row(row, top_confs, top_journals):
132 title = row.get('title')
133 if not title:
134 return None, None
135
136 paper_info = search_paper(title)
137
138 if paper_info:
139 paper_id = paper_info['paperId']
140 row['paperId'] = paper_id
141
142 citations, count = get_citations(paper_id)
143 row['citationCount'] = count
144
145 top_conf_count = 0
146 top_journal_count = 0
147 year_counts = {y: 0 for y in range(2014, 2025)}
148
149 for cit in citations:
150 citing_paper = cit.get('citingPaper', {})
151 if not citing_paper:
152 continue
153
154 year = citing_paper.get('year')
155 if year and 2014 <= year <= 2024:
156 year_counts[year] += 1
157
158 venue = citing_paper.get('venue')
159 if is_top_venue(venue, top_confs):
160 top_conf_count += 1
161 elif is_top_venue(venue, top_journals):
162 top_journal_count += 1
163
164 row['top_conf_citations'] = top_conf_count
165 row['top_journal_citations'] = top_journal_count
166 for year in range(2014, 2025):
167 row[f'citations_{year}'] = year_counts[year]
168
169 return row, True
170 else:
171 return row, False
172
173def process_files():
174 top_confs, top_journals = load_config()
175
176 if not DATA_DIR.exists():
177 print(f"Data directory not found: {DATA_DIR}")
178 return
179
180 files = [f for f in os.listdir(DATA_DIR) if f.endswith('.csv')]
181
182 for filename in files:
183 file_path = DATA_DIR / filename
184 false_file_path = DATA_FALSE_DIR / filename.replace('.csv', '.txt')
185
186 DATA_FALSE_DIR.mkdir(parents=True, exist_ok=True)
187
188 print(f"\nProcessing file: {filename}")
189
190 not_found_papers = []
191 fieldnames = []
192
193 with open(file_path, 'r', encoding='utf-8') as f:
194 reader = csv.DictReader(f)
195 fieldnames = reader.fieldnames
196 rows = list(reader)
197
198 if false_file_path.exists():
199 with open(false_file_path, 'r', encoding='utf-8') as f:
200 not_found_papers = [line.strip() for line in f.readlines()]
201
202 total_rows = len(rows)
203 batch_size = 10
204 max_workers = 3
205
206 indices_to_process = []
207 for i, row in enumerate(rows):
208 if row.get('title') and not (row.get('paperId') and row.get('citationCount')):
209 indices_to_process.append(i)
210
211 print(f"Total rows: {total_rows}, Rows to process: {len(indices_to_process)}")
212
213 for i in range(0, len(indices_to_process), batch_size):
214 batch_indices = indices_to_process[i:i + batch_size]
215
216 print(f"Processing batch {i//batch_size + 1}/{(len(indices_to_process) + batch_size - 1)//batch_size} (Rows {batch_indices[0]+1}-{batch_indices[-1]+1})...")
217
218 with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
219 future_to_index = {
220 executor.submit(process_single_row, rows[idx], top_confs, top_journals): idx
221 for idx in batch_indices
222 }
223
224 for future in concurrent.futures.as_completed(future_to_index):
225 idx = future_to_index[future]
226 try:
227 updated_row, found = future.result()
228 if updated_row:
229 rows[idx] = updated_row
230 title = updated_row.get('title')
231
232 if found:
233 print(f" [✓] Found: {title[:40]}...")
234 if title in not_found_papers:
235 not_found_papers.remove(title)
236 else:
237 print(f" [x] Not Found: {title[:40]}...")
238 if title not in not_found_papers:
239 not_found_papers.append(title)
240 except Exception as exc:
241 print(f"Row {idx} generated an exception: {exc}")
242
243 print(f"Saving progress after batch...")
244 with open(file_path, 'w', encoding='utf-8', newline='') as f:
245 writer = csv.DictWriter(f, fieldnames=fieldnames)
246 writer.writeheader()
247 writer.writerows(rows)
248
249 if not_found_papers:
250 with open(false_file_path, 'w', encoding='utf-8') as f:
251 for t in not_found_papers:
252 f.write(f"{t}\n")
253
254 time.sleep(1)
255
256if __name__ == "__main__":
257 process_files()
258