semaj83/ctmatch

1 2from typing import List, Tuple3 4TREC_REL_PATH =  "/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/trec_21_judgments.txt"5KZ_REL_PATH =  "/Users/jameskelly/Documents/cp/ctmatch/data/kz_data/qrels-clinical_trials.txt"6 7TREC_RELLED_TOPIC_PATH = "/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/processed_trec_data/processed_trec21_topics.jsonl" 8KZ_RELLED_TOPIC_PATH = '/Users/jameskelly/Documents/cp/ctmatch/data/kz_data/processed_kz_data/processed_kz_topics.jsonl' 9 10KZ_DOC_PATH = '/Users/jameskelly/Documents/cp/ctmatch/data/kz_data/clinicaltrials.gov-16_dec_2015.zip'11KZ_PROCESSED_DOC_PATH = '/Users/jameskelly/Documents/cp/ctmatch/data/kz_data/processed_kz_data/processed_kz_docs.jsonl'12 13TREC_ML_PATH = '/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/trec_data.jsonl'14KZ_ML_PATH = '/Users/jameskelly/Documents/cp/ctmatch/data/kz_data/kz_data.jsonl'15 16 17def get_data_tuples(trec_or_kz: str = 'trec') -> List[Tuple[str, str]]:18	if trec_or_kz == 'trec':19		return get_trec_doc_data_tuples(), get_trec_topic_data_tuples()20	return get_kz_doc_data_tuples(), get_kz_topic_data_tuples()21 22 23 24# --------------------------------------------------------------------------------------------------------------- #25# data from TREC clinical track 2021 & 202226# --------------------------------------------------------------------------------------------------------------- #27 28 29def get_trec_doc_data_tuples() -> List[Tuple[str]]:30	trec22_pt1_docs = '/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/trec_docs_21/ClinicalTrials.2021-04-27.part1.zip'31	trec_pt1_target = '/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/processed_trec_data/processed_trec22_docs_part1.jsonl'32 33	trec22_pt2_docs = '/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/trec_docs_21/ClinicalTrials.2021-04-27.part2.zip'34	trec_pt2_target = '/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/processed_trec_data/processed_trec22_docs_part2.jsonl'35 36	trec22_pt3_docs = '/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/trec_docs_21/ClinicalTrials.2021-04-27.part3.zip'37	trec_pt3_target = '/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/processed_trec_data/processed_trec22_docs_part3.jsonl'38 39	trec22_pt4_docs = '/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/trec_docs_21/ClinicalTrials.2021-04-27.part4.zip'40	trec_pt4_target = '/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/processed_trec_data/processed_trec22_docs_part4.jsonl'41 42	trec22_pt5_docs = '/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/trec_docs_21/ClinicalTrials.2021-04-27.part5.zip'43	trec_pt5_target = '/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/processed_trec_data/processed_trec22_docs_part5.jsonl'44 45	trec_doc_data_tuples = [46		(trec22_pt1_docs, trec_pt1_target), 47		(trec22_pt2_docs, trec_pt2_target), 48		(trec22_pt3_docs, trec_pt3_target),49		(trec22_pt4_docs, trec_pt4_target),50		(trec22_pt5_docs, trec_pt5_target)51	]52 53	return trec_doc_data_tuples54 55 56def get_trec_topic_data_tuples() -> List[Tuple[str]]:57	trec21_topic_path = '/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/trec_21_topics.xml'58	trec21_topic_target = '/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/processed_trec_data/processed_trec21_topics.jsonl'59	trec22_topic_path = '/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/trec_22_topics.xml'60	trec22_topic_target = '/Users/jameskelly/Documents/cp/ctmatch/data/trec_data/processed_trec_data/processed_trec22_topics.jsonl'61 62	trec_topic_data_tuples = [63		(trec21_topic_path, trec21_topic_target), 64		(trec22_topic_path, trec22_topic_target)65	]66	return trec_topic_data_tuples67 68 69 70 71# --------------------------------------------------------------------------------------------------------------- #72# data from Koontz, et al. (2016)73# --------------------------------------------------------------------------------------------------------------- #74def get_kz_doc_data_tuples() -> List[Tuple[str]]:75	# kz_doc_data_tuples = []76	# for i in range(1, 18):77	# 	kz_doc_path = f'/Users/jameskelly/Documents/cp/ctmatch/data/kz_data/kz_doc_splits/kz_doc_split{i}.zip'78	# 	kz_doc_target = f'/Users/jameskelly/Documents/cp/ctmatch/data/kz_data/processed_kz_data/processed_kz_doc_split{i}.jsonl'79	# 	kz_doc_data_tuples.append((kz_doc_path, kz_doc_target))80	kz_docs = KZ_DOC_PATH81	kz_docs_target = KZ_PROCESSED_DOC_PATH82	return [(kz_docs, kz_docs_target)]83	84	#return kz_doc_data_tuples85 86def get_kz_topic_data_tuples() -> List[Tuple[str]]:87	kz_topic_desc_path = '/Users/jameskelly/Documents/cp/ctmatch/data/kz_data/topics-2014_2015-description.topics'88	kz_topic_target = '/Users/jameskelly/Documents/cp/ctmatch/data/kz_data/processed_kz_data/processed_kz_topics.jsonl'89	kz_topic_data_tuples = [90		(kz_topic_desc_path, kz_topic_target)91	]92	return kz_topic_data_tuples93