onconpc/onconpc-visualization
2
1"""2Author: Intae Moon3The following script provides features and labels for a supervised training of classifying tumors with known primaries based on samples'41. Clinical information (e.g. age and sex)52. Somatic mutation profile (i.e. frequence of mutated genes)63. Somatic copy number alteration events (i.e. genes associated with CNA : -2 (deep loss), -1 (single-copy loss), 0 (diploid), 1 (low-level gain), 2 (high-level amplification))74. Somatic mutation signatures.8"""9import os10import pickle11from typing import List, Mapping, Tuple12 13import numpy as np14import pandas as pd15from absl import app, flags16 17import utils18 19FLAGS = flags.FLAGS20_FILEPATH = './data/seq_panel_data'21flags.DEFINE_string('config', None, 'Cancer centers to incorporate for feature processing (genie, profile_dfci, or both)')22flags.DEFINE_string('filename_suffix', None, 'Filename suffix for processed files.')23 24def load_genie_data(filepath: str,25 panels: List[str],26 centers: List[str]) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]:27 """28 Load GENIE data.29 Args:30 filepath: Path to GENIE data.31 panels: List of panels to include.32 centers: List of centers to include.33 Returns:34 df_patients: DataFrame containing patient data.35 df_mutations: DataFrame containing mutation data.36 df_cna: DataFrame containing CNA data.37 """38 df_patients = pd.read_csv(os.path.join(filepath, "genie/data_clinical_patient_5.0-public.txt"), sep='\t', comment='#')39 df_samples = pd.read_csv(os.path.join(filepath, "genie/data_clinical_sample_5.0-public.txt"), sep='\t', comment='#')40 df_samples = df_samples[df_samples.SEQ_ASSAY_ID.isin(panels)]41 df_patients = pd.merge(df_patients, df_samples, how='right', on='PATIENT_ID')42 df_patients = df_patients[df_patients.CENTER.isin(centers)]43 44 df_mutations = pd.read_csv(os.path.join(filepath, "genie/data_mutations_extended_5.0-public.txt"), sep='\t',45 comment='#', low_memory=False)46 df_mutations = df_mutations[df_mutations.Center.isin(centers)]47 df_cna = pd.read_csv(os.path.join(filepath, "genie/data_CNA_5.0-public.txt"), sep='\t', comment='#')48 df_cna = df_cna.set_index('Hugo_Symbol').T49 df_cna.index.name = 'Tumor_Sample_Barcode'50 return df_patients, df_mutations, df_cna51 52def load_dfci_data(filepath: str) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]:53 """54 Load Profile DFCI data.55 Args:56 filepath: Path to Profile DFCI data.57 Returns:58 df_samples: DataFrame containing sample data.59 df_mutations: DataFrame containing mutation data.60 df_cna: DataFrame containing CNA data.61 """62 df_samples = pd.read_csv(os.path.join(filepath, "dfci/profile_samples_info"), sep='\t', comment='#')63 df_mutations = pd.read_csv(os.path.join(filepath, "dfci/profile_mutation_dfci"), sep='\t',64 comment='#', low_memory=False)65 df_mutations.dropna(subset=['CANONICAL_GENE'], inplace=True)66 df_mutations['UNIQUE_SAMPLE_ID'] = [float(idx[:-3]) for idx in df_mutations.UNIQUE_SAMPLE_ID.values]67 68 df_cna = pd.read_csv(os.path.join(filepath, "dfci/profile_cnv"), sep='\t', comment='#').set_index('Unnamed: 0')69 return df_samples, df_mutations, df_cna70 71def process_patient_data(df: pd.DataFrame,72 cancers_to_include_dic: Mapping[str, List[str]],73 cancer_type_column: str,74 new_cancer_type_column: str='CANCER_TYPE') -> pd.DataFrame:75 """76 Get relevant patient info DataFrame based on cancer types to include.77 Args:78 df: DataFrame containing patient data.79 cancers_to_include_dic: Dictionary containing cancer types to include.80 cancer_type_column: Column name for cancer type.81 new_cancer_type_column: New column name for cancer type.82 Returns:83 df_final: DataFrame containing processed patient data.84 """85 df_final = pd.DataFrame()86 for cancer_key, detailed_cancers in cancers_to_include_dic.items():87 df_partial = df[df[cancer_type_column].isin(detailed_cancers)].copy()88 # If cancer type consists of only one detailed cancer type, just include the detailed cacner type as main cancer type89 df_partial[new_cancer_type_column] = cancer_key if len(detailed_cancers) > 1 else detailed_cancers[0]90 df_final = pd.concat([df_final, df_partial])91 return df_final92 93def combine_feature_dfs(df_list: List[pd.DataFrame],94 feature_list: List[str]) -> pd.DataFrame:95 """96 Combine feature dfs.97 Args:98 df_list: List of feature dfs.99 feature_list: List of features to include.100 Returns:101 df: DataFrame containing combined features.102 """103 df = pd.DataFrame()104 for df_partial in df_list:105 df_partial = utils.zero_pad_missing_features(df_partial, feature_list)106 df = pd.concat([df, df_partial])107 return df108 109def main(argv):110 # Get flags.111 FLAGS = flags.FLAGS112 config = FLAGS.config113 filename_suffix = FLAGS.filename_suffix114 print('Loading data...')115 if config == 'genie':116 panels = ['DFCI-ONCOPANEL-1', 'DFCI-ONCOPANEL-2', 'DFCI-ONCOPANEL-3',117 'MSK-IMPACT468', 'MSK-IMPACT410', 'MSK-IMPACT341', 'VICC-01-T7', 'VICC-01-T5A']118 centers = ['DFCI', 'MSK', 'VICC']119 df_genie_patients, df_mutations_genie, df_cna_genie = load_genie_data(_FILEPATH, panels, centers)120 print('Obtaining mutation signatures via R function...')121 df_trinuc_feats_genie = utils.get_snv_in_trinuc_context(df_mutations_genie,122 sample_id_col='Tumor_Sample_Barcode',123 chromosome_col='Chromosome',124 start_pos_col='Start_Position',125 ref_allele_col='Reference_Allele',126 alt_allele_col='Tumor_Seq_Allele2',127 config=config)128 df_mut_sigs_genie = utils.obtain_mutation_signatures(df_trinuc_feats_genie)129 elif config == 'profile_dfci':130 df_samples_dfci, df_mutations_dfci, df_cna_dfci = load_dfci_data(_FILEPATH)131 print('Obtaining mutation signatures via R function...')132 df_trinuc_feats_dfci = utils.get_snv_in_trinuc_context(df_mutations_dfci,133 sample_id_col='UNIQUE_SAMPLE_ID',134 chromosome_col='CHROMOSOME',135 start_pos_col='POSITION',136 ref_allele_col='REF_ALLELE',137 alt_allele_col='ALT_ALLELE',138 config=config)139 df_mut_sigs_dfci = utils.obtain_mutation_signatures(df_trinuc_feats_dfci)140 elif config == 'both':141 # In this config, remove DFCI samples in GENIE, since we are directly using Profile DFCI data.142 panels = ['MSK-IMPACT468', 'MSK-IMPACT410', 'MSK-IMPACT341', 'VICC-01-T7', 'VICC-01-T5A']143 centers = ['MSK', 'VICC']144 df_genie_patients, df_mutations_genie, df_cna_genie = load_genie_data(_FILEPATH, panels, centers)145 df_samples_dfci, df_mutations_dfci, df_cna_dfci = load_dfci_data(_FILEPATH)146 print('Obtaining mutation signatures via R function...')147 df_trinuc_feats_genie = utils.get_snv_in_trinuc_context(df_mutations_genie,148 sample_id_col='Tumor_Sample_Barcode',149 chromosome_col='Chromosome',150 start_pos_col='Start_Position',151 ref_allele_col='Reference_Allele',152 alt_allele_col='Tumor_Seq_Allele2',153 config='genie')154 df_trinuc_feats_dfci = utils.get_snv_in_trinuc_context(df_mutations_dfci,155 sample_id_col='UNIQUE_SAMPLE_ID',156 chromosome_col='CHROMOSOME',157 start_pos_col='POSITION',158 ref_allele_col='REF_ALLELE',159 alt_allele_col='ALT_ALLELE',160 config='profile_dfci')161 df_mut_sigs_dfci = utils.obtain_mutation_signatures(df_trinuc_feats_dfci)162 df_mut_sigs_genie = utils.obtain_mutation_signatures(df_trinuc_feats_genie)163 else:164 raise ValueError('Invalid config.')165 print('Processing data...')166 if config in ['profile_dfci', 'both']:167 # Define CUP subtypes.168 cup_subtypes = ['Undifferentiated Malignant Neoplasm',169 'Poorly Differentiated Carcinoma, NOS',170 'Acinar Cell Carcinoma, NOS',171 'Adenocarcinoma, NOS',172 'Cancer of Unknown Primary, NOS',173 'Small Cell Carcinoma of Unknown Primary',174 'Neuroendocrine Tumor, NOS',175 'Squamous Cell Carcinoma, NOS',176 'Cancer of Unknown Primary',177 'Mixed Cancer Types',178 'Neuroendocrine Carcinoma, NOS']179 df_cup_samples_dfci = df_samples_dfci.loc[df_samples_dfci.PRIMARY_CANCER_DIAGNOSIS.isin(cup_subtypes)].copy()180 df_cup_samples_dfci.CANCER_TYPE = 'Cancer of Unknown Primary'181 # Get OncoTree-based cancer types to include.182 with open('./data/cancer_type_to_oncotree_subtypes_dict.pkl', 'rb') as handle:183 cancers_to_include_dic = pickle.load(handle)184 if config in ['genie', 'both']:185 # Process patient data based on cancer types to include.186 df_genie_patients_final = process_patient_data(df_genie_patients, cancers_to_include_dic, cancer_type_column='CANCER_TYPE_DETAILED')187 # Get final cancer types.188 cancer_types_final = list(np.unique(df_genie_patients_final.CANCER_TYPE.values))189 cancer_counts_genie = df_genie_patients_final.CANCER_TYPE.value_counts(sort=True)190 print('GENIE cancer counts')191 print(cancer_counts_genie) 192 # GENIE somatic feature processing.193 df_features_genie, df_labels_genie = utils.pre_process_features_genie(df_mutations_genie,194 df_cna_genie,195 df_mut_sigs_genie,196 df_genie_patients_final,197 cancer_types=cancer_types_final)198 if config in ['profile_dfci', 'both']:199 # Process patient data based on cancer types to include. 200 df_dfci_samples_final = process_patient_data(df_samples_dfci, cancers_to_include_dic, cancer_type_column='PRIMARY_CANCER_DIAGNOSIS')201 if config == 'profile_dfci':202 # Get final cancer types203 cancer_types_final = list(np.unique(df_dfci_samples_final.CANCER_TYPE.values))204 cancer_counts_dfci = df_dfci_samples_final.CANCER_TYPE.value_counts(sort=True)205 print('PROFILE DFCI cancer counts')206 print(cancer_counts_dfci)207 # DFCI somatic feature processing.208 df_samples_dfci_total = pd.concat([df_dfci_samples_final, df_cup_samples_dfci])209 # Change column names to match GENIE data.210 df_samples_dfci_total.rename(columns={"age_at_seq": "Age", "gender": "Sex"}, inplace=True)211 (df_features_dfci,212 df_labels_dfci,213 df_features_cup) = utils.pre_process_features_dfci(df_mutations_dfci,214 df_cna_dfci,215 df_mut_sigs_dfci,216 df_samples_dfci_total, 217 cancer_types_final,218 cup_samples_ids=df_cup_samples_dfci.UNIQUE_SAMPLE_ID.values)219 print('Exporting features and labels of interest...')220 # Load features onconpc pickle file221 with open('./data/features_onconpc.pkl', "rb") as fp:222 features_onconpc = pickle.load(fp)223 if config == 'both':224 # Combine features and labels across centers.225 df_features_combined = combine_feature_dfs([df_features_genie, df_features_dfci], features_onconpc)226 df_features_combined.columns = utils.standardize_feat_names(df_features_combined.columns)227 df_labels_combined = pd.concat([df_labels_genie, df_labels_dfci])228 df_features_cup = df_features_cup[features_onconpc]229 df_features_cup.columns = utils.standardize_feat_names(df_features_cup.columns)230 # Export combined (Profile & GENIE) features and labels.231 df_features_combined.to_csv(f'./data/features_combined_{filename_suffix}', sep='\t', index=True)232 df_labels_combined.to_csv(f'./data/labels_combined_{filename_suffix}', sep='\t', index=True)233 df_features_cup.to_csv(f'./data/features_combined_cup_{filename_suffix}', sep='\t', index=True)234 elif config == 'genie':235 df_features_genie = utils.zero_pad_missing_features(df_features_genie, features_onconpc)236 df_features_genie.columns = utils.standardize_feat_names(df_features_genie.columns)237 df_features_genie.to_csv(f'./data/features_genie_{filename_suffix}', sep='\t', index=True)238 df_labels_genie.to_csv(f'./data/labels_genie_{filename_suffix}', sep='\t', index=True)239 elif config == 'profile_dfci':240 df_features_dfci.columns = utils.standardize_feat_names(df_features_dfci.columns)241 df_features_cup.columns = utils.standardize_feat_names(df_features_cup.columns)242 df_features_dfci.to_csv(f'./data/features_dfci_{filename_suffix}', sep='\t', index=True)243 df_labels_dfci.to_csv(f'./data/labels_dfci_{filename_suffix}', sep='\t', index=True)244 df_features_cup.to_csv(f'./data/features_dfci_cup_{filename_suffix}', sep='\t', index=True)245 return246 247if __name__ == '__main__':248 # Define flags that are required249 flags.mark_flags_as_required(['config',250 'filename_suffix'])251 app.run(main)