DRDMsig/Data_Engineering
0
1#coding:utf-8
2'''
3write by ygq
4create on 2025-07-24
5update MnMs data clean
6https://github.com/openmedlab/Awesome-Medical-Dataset/blob/main/resources/M&Ms.md
7https://zhuanlan.zhihu.com/p/694831343
8
9来自 6 个国际医疗中心 的 340 名受试者 的 CMR 数据。
10覆盖 4 个主流 MRI 设备厂商（Siemens, Philips, GE, Canon）。
11数据集文件结构如下，数据集被组织成训练集、验证集和测试集三个主目录，其中训练集进一步分为有标注和无标注的子目录。每个有标注的子目录包含病人的成像文件以及相应的标注数据。
12M&Ms
13├── Training
14│ ├── Labeled
15│ │ ├── A0S9V9
16│ │ │ ├── A0S9V9_sa.nii.gz
17│ │ │ └── A0S9V9_sa_gt.nii.gz
18│ │ ├── A1D0Q7
19│ │ ├── A1D9Z7
20│ │ └── ...
21│ └── Unlabeled
22├── Validation
23├── Testing
24└── 211230_M&Ms_Dataset_information_diagnosis_opendataset.csv
25
26对训练集有标注的 150 例数据进行图像尺寸统计，size 的格式为 (x,y,z,frame)
27经验丰富的临床医生对心脏磁共振（CMR）图像进行了分割，参考了 ACDC 的标注标准，标注了左心室（LV）、右心室（RV）血池以及左心室心肌（MYO）的轮廓，标签分别为：1（LV）、2（MYO）和3（RV）。
28
29'''
30import os
31import glob
32import pandas as pd
33import SimpleITK as sitk
34import argparse
35import json
36from tqdm import tqdm
37from util import meta_data
38import util
39import numpy as np
40# from bert_helper import *
41
42
43
44meta_id_name='External code'
45meta_vendor_name='VendorName'
46meta_centre_name='Centre'
47meta_pathology_name='Pathology'
48meta_ed_name='ED'
49meta_es_name='ES'
50meta_age_name='Age'
51meta_sex_name='Sex'
52meta_height_name='Height'
53meta_weight_name='Weight'
54
55TASK_VALUE="segmentation"
56CLAMP_RANGE_CT = [-300,300]
57CLAMP_RANGE_MRI = None # MRI images threshold placeholder TBC...
58TARGET_VOXEL_SPACING=None
59
60LABEL_DICT={
61 "0":"backgroud",
62 "1":"LV",#左心室 Blood Pools
63 "2":"MYO",#左心室心肌
64 "3":"RV"#右心室 Blood Pools
65}
66
67# def find_metadata_files(path):
68# # for Cancer Image Archive (TCIA) dataset
69# search_pattern = os.path.join(path, '**', 'metadata.csv')
70# return glob.glob(search_pattern, recursive=True)
71
72def find_metadata_files(path):
73 # for Cancer Image Archive (TCIA) dataset
74 search_pattern = os.path.join(path, '*.csv')
75 return glob.glob(search_pattern, recursive=True)
76##added by yanguoqing on 20250527
77def find_image_dirs(path):
78 return os.listdir(path)
79
80##modify by yanguoqing on 20250527
81def load_dicom_images(folder_path):
82 reader = sitk.ImageSeriesReader()
83 dicom_names = reader.GetGDCMSeriesFileNames(folder_path)
84 reader.SetFileNames(dicom_names)
85 image = reader.Execute()
86 return dicom_names,image
87
88##added by yanguoqing on 20250527
89def load_dicom_tag(imgs):
90 reader = sitk.ImageFileReader()
91 # dicom_names = reader.GetGDCMSeriesFileNames(folder_path)
92 reader.SetFileName(imgs)
93 reader.ReadImageInformation() # 仅读取元信息，不加载像素数据
94 # metadata_keys = reader.GetMetaDataKeys()
95 tag=reader.Execute()
96 return tag
97
98def load_nrrd(fp):
99 return sitk.ReadImage(fp)
100
101def save_nifti(image, output_path, folder_path):
102 # Set metadata in the NIfTI file's header
103 output_dirpath = os.path.dirname(output_path)
104 if not os.path.exists(output_dirpath):
105 print(f"Creating directory {output_dirpath}")
106 os.makedirs(output_dirpath)
107 # Set metadata in the NIfTI file's header
108 image.SetMetaData("FolderPath", folder_path)
109 sitk.WriteImage(image, output_path)
110
111##modify by yanguoqing on 20250527
112def convert_windows_to_linux_path(windows_path):
113 # Replace backslashes with forward slashes and remove the drive letter
114 # Some meta files have windows paths, but the data is stored on a linux server
115 linux_path = windows_path.replace('\\', '/')
116 if ':' in linux_path:
117 linux_path = linux_path.split(':', 1)[1]
118 return linux_path
119
120def main(target_path, output_dir):
121 metadata_files = find_metadata_files(target_path)
122 pid_dirs=find_image_dirs(target_path)
123 pid_dirs=["Training","Testing","Validation"]
124 failed_files = []
125 if not os.path.isdir(output_dir):
126 os.makedirs(output_dir)
127 json_output_path = os.path.join(output_dir, 'nifti_mappings.json')
128 failed_files_path = os.path.join(output_dir, 'failed_files.json')
129 meta = meta_data()
130
131 # Initialize the JSON file
132 if not os.path.exists(json_output_path):
133 with open(json_output_path, 'w') as json_file:
134 json.dump({}, json_file)
135 meta_file=os.path.join(target_path,'211230_M&Ms_Dataset_information_diagnosis_opendataset.csv')
136 if os.path.isfile(meta_file):
137 mf_flag=True
138 df_meta=pd.read_csv(meta_file,sep=',')
139 else:
140 mf_flag=False
141
142 if pid_dirs:
143 for pid_dir in tqdm(pid_dirs, desc="Processing pid dirs"):
144 if not os.path.isdir(os.path.join(target_path,pid_dir)):
145 continue
146 if pid_dir =="Training":
147 tr_flag=True
148 else:
149 tr_flag=False
150 label_flag=False
151
152 if not tr_flag:
153 image_dirs=find_image_dirs(os.path.join(target_path,pid_dir))
154 unlabeled_list=image_dirs
155 else:
156 image_dir_1=find_image_dirs(os.path.join(target_path,pid_dir,'Labeled'))
157 image_dir_2=find_image_dirs(os.path.join(target_path,pid_dir,'Unlabeled'))
158 unlabeled_list=image_dir_2
159 image_dirs=image_dir_1+image_dir_2
160 for data_dir in tqdm(image_dirs, desc="Processing images files"):
161
162 location=data_dir
163 if not tr_flag:
164 full_path=os.path.join(target_path,pid_dir,data_dir)
165 else:
166 if data_dir in unlabeled_list:
167 full_path=os.path.join(target_path,pid_dir,"Unlabeled",data_dir)
168 else:
169 full_path=os.path.join(target_path,pid_dir,"Labeled",data_dir)
170 label_flag=True
171 data_info_row=df_meta[df_meta[meta_id_name]==data_dir]
172
173 if data_info_row.shape[0]>0:
174 data_info_row=data_info_row.reset_index()
175 #print(data_info_row[meta_id_name])
176 meta_image_id=data_info_row[meta_id_name][0]
177 meta_vendor=data_info_row[meta_vendor_name][0]
178 meta_centre=data_info_row[meta_centre_name][0]
179 meta_pathology=data_info_row[meta_pathology_name][0]
180 meta_age=data_info_row[meta_age_name][0]
181 meta_sex=data_info_row[meta_sex_name][0]
182 meta_height=data_info_row[meta_height_name][0]
183 meta_weigth=data_info_row[meta_weight_name][0]
184 meta_ed=data_info_row[meta_ed_name][0]
185 meta_es=data_info_row[meta_es_name][0]
186 else:
187 meta_image_id=data_dir
188 meta_vendor=''
189 meta_centre=''
190 meta_pathology=''
191 meta_age=''
192 meta_sex=''
193 meta_height=''
194 meta_weigth=''
195 meta_ed=''
196 meta_es=''
197 # full_path = convert_windows_to_linux_path(full_path)
198 if not os.path.isdir(full_path):
199 continue
200 try:
201 print(full_path)
202 full_path_image=os.path.join(full_path,"%s_sa.nii.gz"%data_dir)
203
204 if label_flag:
205 full_path_label=os.path.join(full_path,"%s_sa_gt.nii.gz"%data_dir)
206 if not os.path.isfile(full_path_label):
207 full_path_label=None
208 else:
209 full_path_label=None
210
211 sitk_img_original = util.load_nifti(full_path_image)
212 if sitk_img_original is None:
213 print(f" Failed to load image: {full_path_image}")
214 continue
215
216 modality="MRI"
217 study='MnMs'##Dataset_name
218 CIA_other_info = {
219 'metadata_file':''
220 # 'Series_Description':serise_desc
221 }
222 CIA_other_info['split'] = pid_dir
223 if mf_flag:
224 CIA_other_info['metadata_file']=meta_file
225
226 original_spacing = list(sitk_img_original.GetSpacing())
227 original_size = list(sitk_img_original.GetSize())
228 sitk_img_processed = sitk_img_original
229 # is_4d_image = msd_dataset_info.get("tensorImageSize", "3D").upper() == "4D" or sitk_img_original.GetDimension() == 4
230 is_4d_image = sitk_img_original.GetDimension() == 4
231
232 frame_flag=False
233 # --- Resampling Logic (Revised for 4D) ---
234 if is_4d_image:
235
236
237 # Always process 4D images channel-wise for resampling
238 # logging.info(f" Processing 4D image channel-wise: {original_img_full_path}") # Keep log for errors only
239 channels = []
240 num_channels = original_size[3] if len(original_size) == 4 and sitk_img_original.GetDimension() == 4 else 1
241 channel_target_spacing = TARGET_VOXEL_SPACING if TARGET_VOXEL_SPACING else original_spacing[:3] # Use 3D spacing
242
243
244 for i in range(num_channels):
245 extractor = sitk.ExtractImageFilter()
246 current_3d_channel_size = original_size[:3]
247
248 if sitk_img_original.GetDimension() == 4:
249 extractor.SetSize([current_3d_channel_size[0], current_3d_channel_size[1], current_3d_channel_size[2], 0])
250 extractor.SetIndex([0,0,0,i])
251 channel_3d_img = extractor.Execute(sitk_img_original)
252 else:
253 channel_3d_img = sitk_img_original
254 if i > 0: break
255
256 channel_resampler = util.get_unisize_resampler(
257 channel_3d_img, 'linear',
258 spacing=channel_target_spacing, size=current_3d_channel_size
259 )
260 if channel_resampler:
261 channels.append(channel_resampler.Execute(channel_3d_img))
262 else:
263 channels.append(channel_3d_img)
264
265 if channels:
266 if len(channels) > 1: # Only join if there are multiple channels
267 sitk_img_processed = sitk.JoinSeriesImageFilter().Execute(channels)
268 ##aded by yanguoqing on 2025-08-11
269 frame_flag=True
270 imgDict={}
271 for kf_idx in range(num_channels):
272 imgDict[str(kf_idx)]='none'
273 if str(meta_ed):imgDict[str(meta_ed)]='ed'
274 if str(meta_es):imgDict[str(meta_es)]='es'
275 meta.add_keyvalue('ImgDict',imgDict)
276 elif len(channels) == 1: # If only one channel resulted (e.g. original was 3D misidentified as 4D by tensorImageSize)
277 sitk_img_processed = channels[0]
278 elif TARGET_VOXEL_SPACING: # 3D image with target spacing
279 img_resampler_obj = util.get_unisize_resampler(sitk_img_original, 'linear',
280 spacing=TARGET_VOXEL_SPACING, size=original_size)
281 if img_resampler_obj: sitk_img_processed = img_resampler_obj.Execute(sitk_img_original)
282 else: # 3D image, no TARGET_VOXEL_SPACING
283 img_resampler_obj = util.get_unisize_resampler(sitk_img_original, 'linear',
284 spacing=original_spacing, size=original_size)
285 if img_resampler_obj: sitk_img_processed = img_resampler_obj.Execute(sitk_img_original)
286
287
288
289 ##
290 CIA_other_info['Image_id']=meta_image_id
291 CIA_other_info['Vendor']=meta_vendor
292 CIA_other_info['Centre']=str(meta_centre)
293 CIA_other_info['Pathology']=str(meta_pathology)
294 CIA_other_info['Age']=str(meta_age)
295 CIA_other_info['Sex']=meta_sex
296 CIA_other_info['Height']=str(meta_height)
297 CIA_other_info['Weight']=str(meta_weigth)
298 CIA_other_info['ED']=str(meta_ed)
299 CIA_other_info['ES']=str(meta_es)
300
301
302
303 # --- End Resampling Logic ---
304
305 is_processed_4d = sitk_img_processed.GetDimension() == 4
306 clamp_range_to_use=None
307 if clamp_range_to_use and is_processed_4d:
308 clamped_channels_final = []
309 num_channels_final = sitk_img_processed.GetSize()[3] if len(sitk_img_processed.GetSize()) == 4 else 1
310 for i in range(num_channels_final):
311 extractor = sitk.ExtractImageFilter()
312 proc_size_final = sitk_img_processed.GetSize()
313 extractor.SetSize([proc_size_final[0], proc_size_final[1], proc_size_final[2], 0])
314 extractor.SetIndex([0,0,0,i])
315 channel_3d_img_to_clamp = extractor.Execute(sitk_img_processed)
316 clamped_channels_final.append(util.clamp_image(channel_3d_img_to_clamp, clamp_range_to_use))
317 if clamped_channels_final:
318 if len(clamped_channels_final) > 1:
319 sitk_img_processed = sitk.JoinSeriesImageFilter().Execute(clamped_channels_final)
320 elif len(clamped_channels_final) == 1:
321 sitk_img_processed = clamped_channels_final[0]
322 elif clamp_range_to_use: # 3D image
323 sitk_img_processed = util.clamp_image(sitk_img_processed, clamp_range_to_use)
324
325
326 output_path = os.path.join(output_dir,data_dir, f"{data_dir}.nii.gz")
327 # output_path=convert_windows_to_linux_path(output_path)
328 save_nifti(sitk_img_processed, output_path, full_path_image)
329 print(f"Saved NIfTI file to {output_path}")
330
331 label_path_dict = {}
332
333 processed_lbl_full_path = os.path.join(output_dir, data_dir, TASK_VALUE, f"{data_dir}.nii.gz")
334 print(processed_lbl_full_path,full_path_label,tr_flag,label_flag)
335 if tr_flag and label_flag and os.path.exists(full_path_label):
336 sitk_lbl_original = util.load_nifti(full_path_label)
337 if not sitk_lbl_original:
338 print(f" Failed to load label: {full_path_label}")
339 processed_lbl_full_path = None
340 continue
341 if sitk_lbl_original:
342 label_resampler = sitk.ResampleImageFilter()
343 reference_for_label = sitk_img_processed # Default to processed image
344
345 if sitk_img_processed.GetDimension() == 4:
346 num_comp_proc = sitk_img_processed.GetSize()[3] if len(sitk_img_processed.GetSize()) == 4 else 1
347 if num_comp_proc > 0:
348 extractor = sitk.ExtractImageFilter()
349 proc_img_size_for_lbl_ref = sitk_img_processed.GetSize()
350 extractor.SetSize([proc_img_size_for_lbl_ref[0], proc_img_size_for_lbl_ref[1], proc_img_size_for_lbl_ref[2], 0])
351 extractor.SetIndex([0,0,0,0])
352 try:
353 reference_for_label = extractor.Execute(sitk_img_processed)
354 except Exception as ref_err:
355 print(f" Failed to extract 3D reference from 4D image: {output_path} for label alignment.")
356 # print(traceback.format_exc())
357 reference_for_label = None
358 else: # Fallback if extraction fails
359 print(f" Could not extract 3D reference for label from 4D image {output_path}. Label may not be correctly resampled.")
360 reference_for_label = None # This will cause an issue below if not handled
361
362 sitk_lbl_processed = None
363
364 if reference_for_label and reference_for_label.GetDimension() > 0:
365 label_resampler.SetInterpolator(sitk.sitkNearestNeighbor)
366 label_resampler.SetOutputPixelType(sitk_lbl_original.GetPixelID())
367
368 if sitk_lbl_original.GetDimension() == 4:
369 lbl_channels = []
370 lbl_size = list(sitk_lbl_original.GetSize())
371 for i in range(lbl_size[3]):
372 extractor = sitk.ExtractImageFilter()
373 extractor.SetSize([lbl_size[0], lbl_size[1], lbl_size[2], 0])
374 extractor.SetIndex([0, 0, 0, i])
375 single_channel = extractor.Execute(sitk_lbl_original)
376
377 label_resampler.SetReferenceImage(reference_for_label)
378 resampled_channel = label_resampler.Execute(single_channel)
379 lbl_channels.append(resampled_channel)
380
381 if len(lbl_channels) > 1:
382 sitk_lbl_processed = sitk.JoinSeriesImageFilter().Execute(lbl_channels)
383 elif len(lbl_channels) == 1:
384 sitk_lbl_processed = lbl_channels[0]
385 else:
386 label_resampler.SetReferenceImage(reference_for_label)
387 sitk_lbl_processed = label_resampler.Execute(sitk_lbl_original)
388 if processed_lbl_full_path:
389 if sitk_img_processed.GetSize()[:3] != sitk_lbl_processed.GetSize()[:3]:
390 print(f" Mismatch between image and label size (ignoring channels):")
391 print(f" Image size: {sitk_img_processed.GetSize()}")
392 print(f" Label size: {sitk_lbl_processed.GetSize()}")
393 util.save_nifti(sitk_lbl_processed, processed_lbl_full_path, full_path_label)
394 else:
395 print(f" Failed to set reference image for label resampling for {full_path_label}. Saving original label.")
396 util.save_nifti(sitk_lbl_original, processed_lbl_full_path, full_path_label) # Save original
397 # processed_lbl_full_path should still point to this saved original label
398 else:
399 processed_lbl_full_path = None
400 else:
401 processed_lbl_full_path = None
402
403 if processed_lbl_full_path:
404 label_path_dict['heart'] = processed_lbl_full_path
405
406 print('compare image and label size',sitk_img_original.GetSize(),sitk_lbl_original.GetSize())
407 print('compare image and label size',sitk_img_processed.GetSize(),sitk_lbl_processed.GetSize())
408 try:
409 assert sitk_img_processed.GetSize() == sitk_lbl_processed.GetSize()
410
411 except Exception as e:
412 failed_files.append(full_path_label)
413 continue
414
415 except RuntimeError:
416 failed_files.append(full_path_image)
417 print(f"Failed to load MnMs images from {full_path_image}")
418 continue
419
420
421
422
423 size_processed = list(sitk_img_processed.GetSize())
424 print('size_processed',size_processed)
425
426 # meta.add_keyvalue('Image_id',meta_image_id)
427 meta.add_keyvalue('Spacing_mm',min(original_spacing[:3]))##保留前三个x,y,z的最小spacing
428 meta.add_keyvalue('OriImg_path',full_path_image)
429 meta.add_keyvalue('Size',size_processed) # 这里用处理后的size -- YH Jachin
430 meta.add_keyvalue('Modality',modality)
431 meta.add_keyvalue('Dataset_name',study)
432 meta.add_keyvalue('ROI','chest')
433
434
435 if processed_lbl_full_path:
436 print(label_path_dict.keys())
437 meta.add_keyvalue('Task',TASK_VALUE)
438 # meta.add_keyvalue('Label_tissue',list(label_path_dict.keys()))
439 meta.add_keyvalue('Label_path',{TASK_VALUE:label_path_dict})
440 meta.add_keyvalue('Label_Dict',LABEL_DICT)
441 meta.add_extra_keyvalue('Metadata',CIA_other_info)
442
443
444
445
446 # Write the mapping to the JSON file on the fly
447 with open(json_output_path, 'r+') as json_file:
448 existing_mappings = json.load(json_file)
449 existing_mappings[output_path] = meta.get_meta_data()
450 json_file.seek(0)
451 print(existing_mappings)
452 json.dump(existing_mappings, json_file, indent=4)
453 json_file.truncate()
454 # else:
455 # print("No metadata.csv files found.")
456
457 with open(failed_files_path, "w") as json_file:
458 json.dump(failed_files, json_file)
459
460 print(f"The list has been written to {failed_files_path}")
461 print(f"Saved NIfTI mappings to {json_output_path}")
462
463if __name__ == "__main__":
464 parser = argparse.ArgumentParser(description="Process DICOM files and save as NIfTI.")
465 parser.add_argument("--target_path", type=str, help="Path to the target directory containing metadata files.", default="/home/data/Github/data/data_gen_def/DATASETS/MnMs/OpenDataset/")
466 parser.add_argument("--output_dir", type=str, help="Directory to save the NIfTI files.", default="/home/data/Github/data/data_gen_def/DATASETS_processed/MnMs/")
467 args = parser.parse_args()
468 print(args.target_path, args.output_dir)
469 main(args.target_path, args.output_dir)
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485