DRDMsig/Data_Engineering
0
1import os2import json3import SimpleITK as sitk4import glob5import pandas as pd6 7def load_dicom_images(folder_path):8 reader = sitk.ImageSeriesReader()9 dicom_names = reader.GetGDCMSeriesFileNames(folder_path)10 reader.SetFileNames(dicom_names)11 image = reader.Execute()12 return image13 14def convert_windows_to_linux_path(windows_path):15 # Replace backslashes with forward slashes and remove the drive letter16 # Some meta files have windows paths, but the data is stored on a linux server17 linux_path = windows_path.replace('\\', '/')18 if ':' in linux_path:19 linux_path = linux_path.split(':', 1)[1]20 return linux_path21 22# =============================================================================23# ========================developed with TotalSegmentor========================24# =============================================================================25 26def read_table(file_path, split_str=';'):27 try:28 df = pd.read_excel(file_path, engine='openpyxl') 29 except:30 df = pd.read_csv(file_path, sep=split_str)31 return df32 33def load_nifti(image_path):34 return sitk.ReadImage(image_path)35 36def save_nifti(image, output_path, folder_path):37 output_dirpath = os.path.dirname(output_path)38 if not os.path.exists(output_dirpath):39 print(f"Creating directory {output_dirpath}")40 os.makedirs(output_dirpath)41 # Set metadata in the NIfTI file's header42 image.SetMetaData("FolderPath", folder_path)43 sitk.WriteImage(image, output_path)44 45def find_metadata_files(path, file_name='*meta*'):46 # for TotalSegmentor dataset47 search_pattern = os.path.join(path, '**', file_name)48 return glob.glob(search_pattern, recursive=True)49 50def get_img_path_from_folder(folder_path, img_type='.nii.gz', include_str=None, exclude_str='segmentation', is_sorted=True): 51 img_path = []52 for root, dirs, files in os.walk(folder_path):53 for file in files:54 if file.endswith(img_type) and (include_str is None or include_str in file) and (exclude_str is None or exclude_str not in file):55 img_path.append(os.path.join(root, file))56 if is_sorted:57 img_path.sort()58 return img_path59 60def get_unisize_resampler(ref_img, interpolator='linear', spacing=None, size=None):61 '''62 Resample the image to have isotropic spacing, following the steps:63 1. Find the minimum spacing64 2. Resample the image to have the minimum spacing65 3. Set the interpolator (linear for images, nearest for segmentation masks)66 4. Set the output spacing67 5. Return the resampler for resampling68 For example, if the input image has spacing [0.1, 0.1, 0.3], the output image will have spacing [0.1, 0.1, 0.1]69 '''70 # 讨论为什么重新写这个函数！！！71 if size is None:72 size = ref_img.GetSize()73 if spacing is None:74 spacing = ref_img.GetSpacing()75 min_spacing = min(spacing)76 if all([spc == min_spacing for spc in spacing]):77 return None78 else:79 # if 1:80 if interpolator == 'nearest':81 interpolator = sitk.sitkNearestNeighbor82 elif interpolator == 'linear':83 interpolator = sitk.sitkLinear84 resampler = sitk.ResampleImageFilter() 85 # new_spacing = [max_spacing] * len(spacing)86 # print(size)87 new_size = [int(round(old_sz * old_spc / min_spacing)) for old_sz, old_spc in zip(size, spacing)]88 new_size_xy=[new_size[0],new_size[1],new_size[2]] 89 # 讨论为什么重新写这个函数！！！ --- YHM Jachin90 new_size_spacing=[min_spacing,min_spacing,min_spacing]91 # 讨论为什么重新写这个函数！！！ --- YHM Jachin92 # resampler.SetSize(new_size)93 # resampler.SetOutputSpacing([min_spacing] * len(spacing))94 resampler.SetSize(new_size_xy)95 resampler.SetOutputSpacing(new_size_spacing)96 97 # print(new_size,new_size_xy)98 resampler.SetOutputOrigin(ref_img.GetOrigin())99 resampler.SetOutputDirection(ref_img.GetDirection())100 resampler.SetInterpolator(interpolator)101 resampler.SetDefaultPixelValue(ref_img.GetPixelIDValue())102 resampler.SetOutputPixelType(ref_img.GetPixelID())103 return resampler104 105def clamp_image(in_img,clamp_range):106 ''' 107 Clamp the image to the specified range108 '''109 clamp_filter = sitk.ClampImageFilter()110 clamp_filter.SetLowerBound(clamp_range[0])111 clamp_filter.SetUpperBound(clamp_range[1])112 return clamp_filter.Execute(in_img)113 114def get_synonyms_dict(dict_type='ROI'):115 '''116 Get the dictionary of synonyms for the specified dictionary type117 '''118 if dict_type == 'ROI':119 dict_synonyms = {120 'whole-body': ['whole-body', 'whole body', 'wholebody', 'whole body', 'whole-body', 'whole body', 'wholebody','polytrauma','head-neck-thorax-abdomen-pelvis-leg','head-neck-thorax-abdomen-pelvis'],121 'neck-thorax-abdomen-pelvis-leg': ['neck-thorax-abdomen-pelvis-leg','neck-thx-abd-pelvis-leg', 'angiography neck-thx-abd-pelvis-leg', 'neck thorax abdomen pelvis leg', 'neck and thorax and abdomen and pelvis and leg', 'neck, thorax, abdomen, pelvis & leg', 'neck/thorax/abdomen/pelvis/leg', 'neck, thorax, abdomen, pelvis and leg', 'neck thorax abdomen pelvis leg'],122 'neck-thorax-abdomen-pelvis': ['neck-thorax-abdomen-pelvis', 'neck-thx-abd-pelvis', 'neck thorax abdomen pelvis', 'neck and thorax and abdomen and pelvis', 'neck, thorax, abdomen & pelvis', 'neck/thorax/abdomen/pelvis', 'neck, thorax, abdomen and pelvis', 'neck thorax abdomen & pelvis'],123 'thorax-abdomen-pelvis-leg': ['thorax-abdomen-pelvis-leg','thx-abd-pelvis-leg', 'angiography thx-abd-pelvis-leg', 'thorax abdomen pelvis leg', 'thorax and abdomen and pelvis and leg', 'thorax, abdomen, pelvis & leg', 'thorax/abdomen/pelvis/leg', 'thorax, abdomen, pelvis and leg', 'thorax abdomen pelvis leg'],124 'neck-thorax-abdomen': ['neck-thorax-abdomen', 'neck-thorax-abdomen', 'neck thorax abdomen', 'neck and thorax and abdomen', 'neck, thorax, abdomen', 'neck/thorax/abdomen', 'neck, thorax, abdomen', 'neck thorax abdomen'],125 'head-neck-thorax-abdomen': ['head-neck-thorax-abdomen', 'head-neck-thorax-abdomen', 'head neck thorax abdomen', 'head and neck and thorax and abdomen', 'head, neck, thorax, abdomen', 'head/thorax/abdomen', 'head, thorax, abdomen', 'head thorax abdomen'],126 'head-neck-thorax': ['head-neck-thorax', 'head neck thorax', 'head and neck and thorax', 'head, neck, thorax', 'head/thorax', 'head, thorax', 'head thorax'],127 'thorax-abdomen-pelvis': ['thorax-abdomen-pelvis', 'thx-abd-pelvis', 'polytrauma', 'thorax abdomen pelvis', 'thorax and abdomen and pelvis', 'thorax, abdomen & pelvis', 'thorax/abdomen/pelvis', 'thorax, abdomen and pelvis', 'thorax abdomen & pelvis'],128 'abdomen-pelvis-leg': ['abdomen-pelvis-leg', 'angiography abdomen-pelvis-leg', 'abd-pelvis-leg', 'abdomen pelvis leg', 'abdomen and pelvis and leg', 'abdomen, pelvis & leg', 'abdomen/pelvis/leg', 'abdomen, pelvis, leg', 'abdomen pelvis leg'],129 'neck-thorax': ['neck-thorax', 'neck thorax', 'neck and thorax', 'neck, thorax', 'thorax-neck', 'thorax neck', 'thorax and neck', 'thorax, neck','thorax/neck'],130 'thorax-abdomen': ['thorax-abdomen', 'thorax abdomen', 'thorax and abdomen', 'thorax, abdomen'],131 'abdomen-pelvis': ['abdomen-pelvis', 'abdomen pelvis', 'abdomen and pelvis', 'abdomen & pelvis', 'abdomen/pelvis', 'abdomen-pelvis', 'abdomen pelvis', 'abdomen and pelvis', 'abdomen & pelvis', 'abdomen/pelvis'],132 'pelvis-leg': ['pelvis-leg', 'pelvis leg', 'pelvis and leg', 'pelvis, leg', 'pelvis/leg', 'pelvis-leg', 'pelvis leg', 'pelvis and leg', 'pelvis, leg', 'pelvis/leg'],133 'head-neck': ['head-neck', 'head neck', 'head and neck', 'head, neck', 'head/neck', 'head-neck', 'head neck', 'head and neck', 'head, neck', 'head/neck'],134 'abdomen': ['abdomen', 'abdominal', 'belly', 'stomach', 'tummy', 'gut', 'guts', 'viscera', 'bowels', 'intestines', 'gastrointestinal', 'digestive', 'peritoneum','gastric', 'liver', 'spleen', 'pancreas','kidney','lumbar','renal','hepatic','splenic','pancreatic','intervention'],135 'thorax': ['chest', 'thorax', 'breast', 'lung', 'heart','heart-thorakale aorta', 'heart-thorakale', 'mediastinum', 'pleura', 'bronchus', 'bronchi', 'trachea', 'esophagus', 'diaphragm', 'rib', 'sternum', 'clavicle', 'scapula', 'axilla', 'armpit','breast biopsy','thoracic','mammary','caeiothoracic','mediastinal','pleural','bronchial','bronchial tree','tracheal','esophageal','diaphragmatic','costal','sternal','clavicular','scapular','axillary','axillar','cardiac','pericardial','pericardiac','pericardium'],136 'head': ['head', 'headbasis', 'brain', 'skull', 'face','nose','ear','eye','mouth','jaw','cheek','chin','forehead','temporal','parietal','occipital','frontal','mandible','maxilla','mandibular','maxillary','nasal','orbital','orbita','ocular','auricular','otic','oral','buccal','labial','lingual','palatal'],137 'neck': ['neck', 'throat', 'cervical', 'thyroid', 'trachea', 'larynx', 'pharynx', 'esophagus','pharyngeal','laryngeal','cervical','thyroid','trachea','esophagus','carotid','jugular'],138 'hand': ['hand', 'finger', 'thumb', 'palm', 'wrist', 'knuckle', 'fingernail', 'phalanx', 'metacarpal', 'carpal', 'radius'],139 'arm': ['arm', 'forearm', 'upper arm', 'bicep', 'tricep', 'brachium', 'brachial', 'humerus', 'radius', 'ulna', 'elbow', 'shoulder', 'armpit''clavicle', 'scapula', 'acromion', 'acromioclavicular'],140 'leg': ['leg', 'felsenleg','thigh', 'calf', 'shin', 'knee', 'foot', 'ankle', 'toe', 'heel', 'sole', 'arch', 'instep', 'metatarsal', 'phalanx', 'tibia', 'fibula', 'femur', 'patella', 'kneecap','achilles tendon','achilles'],141 'pelvis': ['pelvis', 'hip', 'groin', 'buttock', 'gluteus', 'gluteal', 'ischium', 'pubis', 'sacrum', 'coccyx', 'acetabulum', 'iliac', 'iliac crest', 'iliac spine', 'iliac wing', 'sacroiliac', 'sacroiliac joint', 'sacroiliac ligament', 'sacroiliac spine', 'ureter', 'bladder', 'urethra', 'prostate', 'testicle', 'ovary', 'uterus',],142 'skeleton': ['skeleton','bone','spine', 'back', 'vertebra', 'sacrum', 'coccyx'],143 }144 elif dict_type == 'Label_tissue':145 dict_synonyms = {146 'liver': ['liver','hepatic'],147 'spleen': ['spleen','splenic'],148 'kidney': ['kidney','renal'],149 'pancreas': ['pancreas','pancreatic'],150 'stomach': ['stomach','gastric'],151 'intestine': ['large intestine', 'small intestine','large bowel','small bowel'],152 'gallbladder': ['gallbladder'],153 'adrenal_gland': ['adrenal_gland','adrenal gland'],154 'bladder': ['bladder'],155 'prostate': ['prostate'],156 'uterus': ['uterus'],157 'ovary': ['ovary'],158 'testicle': ['testicle'],159 'lymph_node': ['lymph_node','lymph node'],160 'bone': ['bone'],161 'lung': ['lung'],162 'heart': ['heart'],163 'esophagus': ['esophagus'],164 'muscle': ['muscle'],165 'fat': ['fat'],166 'skin': ['skin'],167 'vessel': ['vessel'],168 'tumor': ['tumor'],169 'other': ['other']170 }171 elif dict_type == 'Task':172 dict_synonyms = {173 'segmentation': ['segmentation', 'seg', 'mask'],174 'classification': ['classification', 'class', 'diagnosis','identify','identification'],175 'localization': ['localization', 'locate', 'location', 'position'],176 'registration': ['registration', 'register', 'align', 'alignment'],177 'detection': ['detection', 'detect', 'find', 'locate'],178 'quantification': ['quantification', 'quantify', 'measure', 'measurement'],179 }180 elif dict_type == 'Modality':181 dict_synonyms = {182 'CT': ['CT', 'computed tomography'],183 'MRI': ['MRI', 'MR', 'magnetic resonance imaging'],184 'PET': ['PET', 'positron emission tomography'],185 'US': ['US', 'ultrasound'],186 'X-ray': ['X-ray', 'radiography'],187 'SPECT': ['SPECT', 'single-photon emission computed tomlogy'],188 }189 else:190 raise ValueError(f"dict_type {dict_type} is not valid")191 return dict_synonyms192 193def replace_synonyms(text, dict_synonyms):194 '''195 Replace the synonyms in the text with the standard term196 '''197 if isinstance(text,str):198 for key, value in dict_synonyms.items():199 for v in value:200 if v.lower() in text.lower():201 return key202 Warning(f"Value {text} is not in the correct format")203 elif isinstance(text,list):204 text = [replace_synonyms(t, dict_synonyms) for t in text]205 elif isinstance(text,dict):206 for key in text.keys():207 # replace values in dict208 text[key] = replace_synonyms(text[key], dict_synonyms)209 # replace keys in dict210 for k in dict_synonyms.keys():211 text[dict_synonyms[k]] = text.pop(key)212 return text213 214# =============================================================================215 216class meta_data(object):217 '''218 This class is used to store the metadata of the dataset219 '''220 def __init__(self):221 self.config_format_path = os.path.join(os.path.dirname(__file__),'config_format.json') 222 with open(self.config_format_path, 'r') as file:223 self.config_format = json.load(file)224 self.config = {}225 for key in self.config_format.keys():226 if self.config_format[key]['required'] == True:227 self.config[key] = {}228 self.keytypes = self.find_all_keys_with_type()229 self.keytypes_flatten = self.flatten_json()230 self.ambiguity_keys = ['ROI', 'Label_tissue', 'Task', 'Modality']231 for key in self.ambiguity_keys:232 ambiguity_dict = get_synonyms_dict(key)233 self.config_format[key]['options'] = list(ambiguity_dict.keys())234 235 def get_ketytypes(self):236 return self.keytypes237 238 def get_keytypes_flatten(self):239 return self.keytypes_flatten240 241 def find_all_keys_with_type(self, data=None, parent_key=''):242 if data is None:243 data = self.config_format244 keys_with_type = {}245 if isinstance(data, dict):246 for key, value in data.items():247 full_key = f"{parent_key}.{key}" if parent_key else key248 if isinstance(value, dict) and 'type' in value:249 keys_with_type[full_key] = value['type']250 keys_with_type.update(self.find_all_keys_with_type(value, full_key))251 elif isinstance(data, list):252 for index, item in enumerate(data):253 full_key = f"{parent_key}[{index}]"254 keys_with_type.update(self.find_all_keys_with_type(item, full_key))255 return keys_with_type256 257 def flatten_json(self, data=None, parent_key='', sep='.'):258 if data is None:259 data = self.config_format260 items = {}261 if isinstance(data, dict):262 for key, value in data.items():263 new_key = f"{parent_key}{sep}{key}" if parent_key else key264 if isinstance(value, dict):265 items.update(self.flatten_json(value, new_key, sep=sep))266 elif isinstance(value, list):267 for i, item in enumerate(value):268 items.update(self.flatten_json(item, f"{new_key}[{i}]", sep=sep))269 else:270 items[new_key] = value271 elif isinstance(data, list):272 for i, item in enumerate(data):273 items.update(self.flatten_json(item, f"{parent_key}[{i}]", sep=sep))274 return items275 276 def req_check(self):277 self.unfilled_keys = []278 for key in self.config.keys():279 if self.config[key] == {}:280 self.unfilled_keys.append(key) 281 if len(self.unfilled_keys) == 0:282 return True283 else:284 return False285 286 def type_check(self, key, value):287 if key not in self.config_format.keys():288 print(key, "is not a valid key")289 return False290 291 if key == 'Modality':292 if value not in self.config_format[key]['options']:293 return False294 else:295 return True296 297 elif key == 'OriImg_path':298 if isinstance(value, str):299 return True300 else:301 return False302 303 elif key == 'Label_path' and isinstance(value, dict):304 for skey in value.keys():305 if skey in self.config_format[key]['keys']:306 for kk in value[skey]:307 if isinstance(value[skey][kk],str):308 pass309 # if kk in self.config_format[key]['value']['keys']:310 # if isinstance(value[skey][kk],str):311 # pass312 # else:313 # return False314 else:315 return False316 return True 317 318 elif key == 'ROI':319 if value not in self.config_format[key]['options']:320 return False321 else:322 return True323 324 elif key == 'Label_tissue' and isinstance(value, list):325 for i in value:326 if i not in self.config_format[key]['items']['options']:327 return False328 return True329 330 elif key =='Task' and isinstance(value, list):331 for i in value:332 if i not in self.config_format[key]['items']['options']:333 return False334 return True335 336 elif key == 'Spacing_mm':337 if isinstance(value, float):338 return True339 else:340 False341 342 # elif key == 'Size' and isinstance(value, list) and len(value) == 3 :343 elif key == 'Size' and isinstance(value, list) and len(value) >= 3 :344 return all(isinstance(item, int) for item in value)345 346 elif key == 'Dataset_name':347 if isinstance(value, str):348 return True349 else:350 return False351 elif key == 'ImgDict':352 if isinstance(value, dict):353 return True354 else:355 return False 356 elif key == 'Label_Dict':357 if isinstance(value, dict):358 return True359 else:360 return False 361 def add_extra_keyvalue(self, key, value):362 self.config[key] = value363 return True 364 365 def add_keyvalue(self, key, value):366 if key in self.ambiguity_keys:367 value = replace_synonyms(value, get_synonyms_dict(key))368 # print(key, value)369 if self.type_check(key, value):370 self.config[key] = value371 return True372 else:373 Warning(f"Value {value} is not in the correct format for key {key}")374 pass375 # print(f"Value {value} is not in the correct format for key {key}")376 377 def get_meta_data(self):378 if self.req_check():379 return self.config380 else:381 print("Not all required keys are filled", self.unfilled_keys)382 return False383 384 385 386if __name__ == '__main__':387 meta = meta_data()388 print(meta.get_keytypes_flatten())389 print(meta.get_ketytypes())390 meta.add_keyvalue('Modality', 'CT')391 meta.add_keyvalue('OriImg_path', 'C:/Users/jzheng/Desktop/CT')392 meta.add_keyvalue('Label_path', {'ROI': {'1': 'C:/Users/jzheng/Desktop/CT/1'}, 'Tissue': {'1': 'C:/Users/jzheng/Desktop/CT/1'}})393 meta.add_keyvalue('Spacing_mm', 1.5)394 meta.add_keyvalue('Size', [512, 512, 100])395 meta.add_keyvalue('Dataset_name', 'CT')396 meta.add_keyvalue('Label_tissue', ['1', '2', '3'])397 meta.add_keyvalue('Task', ['1', '2', '3'])398 print(meta.get_meta_data())399 meta.add_extra_key('extra', 'extra')400 print(meta.get_meta_data())401 print(meta.get_ketytypes())402 print(meta.get_keytypes_flatten)403 404 org_data_foler_path = '/home/jachin/data/Github/data/data_gen_def/DATASETS/TotalSegmentorCT_MRI/TS_CT'405 img_paths = get_img_path_from_folder(org_data_foler_path, img_type='.nii.gz', include_str='ct', exclude_str='segmentation') 406 print(img_paths)